Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8142484347 | ||
|
|
0d91b82f86 | ||
|
|
90fc86d437 | ||
|
|
2fe58073e4 | ||
|
|
bc03e82687 | ||
|
|
3f983e6d76 | ||
|
|
134ec9a68e | ||
|
|
9cf177b198 | ||
|
|
a5fae489e0 | ||
|
|
8d0083d4e4 | ||
|
|
9f177557d2 | ||
|
|
46fce4a767 | ||
|
|
2d9595b0e5 | ||
|
|
a7d7456279 | ||
|
|
44a792c34f | ||
|
|
3987cd4260 | ||
|
|
95d9ed4aa6 | ||
|
|
8d0063d514 | ||
|
|
0ca0cac534 | ||
|
|
cf0c55ba80 | ||
|
|
fa4f823fc8 | ||
|
|
884060891e | ||
|
|
955c8c3445 | ||
|
|
99df44d7fb | ||
|
|
30fbdc52b2 | ||
|
|
046ff735cf | ||
|
|
7afaba0547 | ||
|
|
db8ed5a004 | ||
|
|
0ef25a8037 | ||
|
|
17a26894b0 | ||
|
|
946c8f26db | ||
|
|
8759b5860d | ||
|
|
dc902d2c0e | ||
|
|
cedcbb1434 | ||
|
|
4f72b193aa | ||
|
|
7b94e9a651 | ||
|
|
c6bf84a578 | ||
|
|
444f9b8166 | ||
|
|
7a47223a58 | ||
|
|
bc138e3b37 | ||
|
|
df7c3db085 | ||
|
|
6fda49369f | ||
|
|
8018cd4685 | ||
|
|
29875b7367 | ||
|
|
7ae5c57b6b | ||
|
|
dc4e8b9fcd | ||
|
|
6f9ba428d5 | ||
|
|
c60359bfed | ||
|
|
e6b76e156a | ||
|
|
e74986cc9a | ||
|
|
2df45b27d3 | ||
|
|
bacd6aec4f | ||
|
|
fad301e7a4 | ||
|
|
1cfb19bdc7 | ||
|
|
90c30d596f | ||
|
|
7ed6ccf171 | ||
|
|
ce30b99327 | ||
|
|
a7891d03c5 | ||
|
|
038da5374f | ||
|
|
92979fa920 | ||
|
|
2d5bd275fa | ||
|
|
49fe56b6e7 | ||
|
|
1fa6a0bea8 | ||
|
|
582f20fab6 | ||
|
|
6cb8231a9f | ||
|
|
6bbf532b70 | ||
|
|
2fa5c8c394 | ||
|
|
373415a2eb | ||
|
|
582c033e1c | ||
|
|
ef652ee5d8 | ||
|
|
250e3a9f90 | ||
|
|
6096dd6927 | ||
|
|
51e98bfc38 | ||
|
|
3ee00fb051 | ||
|
|
29094f5f51 | ||
|
|
16a44dec79 | ||
|
|
4e3e68d6e6 | ||
|
|
eb29f22eb7 | ||
|
|
8f6f89832f | ||
|
|
1c49870a7a | ||
|
|
142593083f | ||
|
|
8cfab498cf | ||
|
|
b5996a0e56 | ||
|
|
5e208353e3 | ||
|
|
66dbd0af81 | ||
|
|
59fd068711 | ||
|
|
8e12eee996 | ||
|
|
89c77ae6b5 | ||
|
|
0b672ed6e7 | ||
|
|
54d19cc386 | ||
|
|
04b58e5c6f | ||
|
|
6b0401b922 | ||
|
|
0cff964ff2 | ||
|
|
dc221338be | ||
|
|
8c10e57208 | ||
|
|
dd040adc96 | ||
|
|
4ab128ccab | ||
|
|
a28193a00c | ||
|
|
986bc8d01e | ||
|
|
07e3ea3737 | ||
|
|
b2eaa9e00b | ||
|
|
791ba5f507 | ||
|
|
7789db8240 | ||
|
|
f92cd9017f | ||
|
|
4bfba4aa86 | ||
|
|
86bc008a81 | ||
|
|
fac5c4465a | ||
|
|
9c9b1cf6d5 | ||
|
|
c3d88d9b20 | ||
|
|
569190480d | ||
|
|
5883ea509d | ||
|
|
a210992bfc | ||
|
|
0517de1f59 | ||
|
|
b59fd38a9c | ||
|
|
88a166e4aa | ||
|
|
07766c43e8 | ||
|
|
bb3c6dec45 | ||
|
|
3cf8f04d73 | ||
|
|
2b56c5c2eb | ||
|
|
c06a456a9a | ||
|
|
864f673ef5 | ||
|
|
6c63df6704 | ||
|
|
a5bcd9fca2 | ||
|
|
618ef812e6 | ||
|
|
e299c66121 | ||
|
|
11aef4ab7a | ||
|
|
4e52bdd4ca | ||
|
|
595490cb98 | ||
|
|
7537882235 | ||
|
|
47d968eeea | ||
|
|
e7f38396a2 | ||
|
|
f569b19c49 | ||
|
|
e795d191c5 | ||
|
|
4ad7183102 | ||
|
|
2c3aaec62d | ||
|
|
70598d47a3 | ||
|
|
5858e7b513 | ||
|
|
5346efcd10 | ||
|
|
6169e32ca5 | ||
|
|
90523284ab | ||
|
|
17bfea8754 | ||
|
|
2a9ddb78ac | ||
|
|
e887233210 | ||
|
|
b17d92d7c2 | ||
|
|
90ee34a1b4 | ||
|
|
118fc70e08 | ||
|
|
636372511b | ||
|
|
1354f5b503 | ||
|
|
98ec1ffcab | ||
|
|
18c11ca437 | ||
|
|
ca5e446fbd | ||
|
|
4c7ab12a3e | ||
|
|
c6b552e75f | ||
|
|
3cd0c525e2 | ||
|
|
f130a1c05a | ||
|
|
bbd31c7143 | ||
|
|
8731508645 | ||
|
|
b5d37ca142 | ||
|
|
eb0507f942 | ||
|
|
6d09b94bbb | ||
|
|
5274216981 | ||
|
|
75d89613d0 | ||
|
|
8fb276aa90 | ||
|
|
78a9324381 | ||
|
|
8a50538275 | ||
|
|
b4aea7fa5b | ||
|
|
92c305bab9 |
@@ -28,6 +28,14 @@ ALLOWED_EMAILS=
|
||||
# Admin emails (subset of the above) get the admin role.
|
||||
ADMIN_EMAILS=
|
||||
|
||||
# Logging (E22). LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# the worker/download child read; the API's running level is the admin Configuration page's
|
||||
# `log_level` knob, which takes effect without a restart. TRACE | DEBUG | INFO | WARNING | ERROR, in
|
||||
# any letter case; any other value stops the boot with an error. LOG_FORMAT is "json" (one object per
|
||||
# line — what a log collector ingests) or "text" (a readable line for a local terminal).
|
||||
LOG_LEVEL=INFO
|
||||
LOG_FORMAT=json
|
||||
|
||||
# Optional: origin of a separately-served frontend dev server (enables CORS). Leave empty in production.
|
||||
FRONTEND_ORIGIN=
|
||||
|
||||
|
||||
+234
@@ -7,6 +7,240 @@ edit the release notes and re-run the generator.
|
||||
|
||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||
|
||||
## 0.93.0 — 2026-09-20
|
||||
|
||||
A way to report a fault from inside the app: record what you did, mark the steps, attach screenshots — and the app writes the report for you, with the logs of exactly that session attached.
|
||||
|
||||
### Features
|
||||
|
||||
- Start a recording (the command palette, or Settings → Diagnostics) and a pill follows you: a timer, “Step” to mark what you just did, a screenshot button, and Stop. Everything you and the server do while it runs is collected under that one session, and nothing else is.
|
||||
- Stop opens the composer: a drafted title and description, the steps you marked interleaved with the pages you visited, a table of your browser and the app's version, and the attachments — each one reviewable, each one removable.
|
||||
- The log lines that would be sent can be read before they go, line by line, and any of them left out.
|
||||
- Screenshots come from a paste, a drop, a file, or capturing the tab — up to eight, and any of them can be dropped again before sending.
|
||||
- Send opens an issue on the project's tracker with the report and its logs attached, and the preview shows it exactly as the tracker will render it. Five reports an hour.
|
||||
- Without a tracker configured — or whenever you would rather keep it — the same report downloads as a zip instead.
|
||||
- An administrator sets the tracker up in Settings → System (address, repository, token) and can test the connection there. The server refuses to send to a public repository: a report carries its reporter's log.
|
||||
|
||||
### Fixes
|
||||
|
||||
- The Plex player keeps the keyboard inside itself: Tab no longer walks out of a playing film into the page behind it, not even while its own settings panel is open.
|
||||
- A report cannot be sent twice at once, from any tab: while one send runs the others say so, and a send that fails half-way retries only what is missing — never a second issue.
|
||||
- A recording ends by itself after an hour, and both Stop and that hour return your diagnostics level to whatever it was before you started.
|
||||
|
||||
### Internal
|
||||
|
||||
- A session's logs and screenshots are purged with the session after 7 days, like every other diagnostic line, and another account's session id is ignored rather than refused.
|
||||
- Four adversarial review rounds at the epic's end: 80 findings fixed across them, every regression test proven to fail with its own fix removed, and the keyboard and focus work measured in a real browser rather than argued.
|
||||
|
||||
## 0.92.0 — 2026-09-19
|
||||
|
||||
A debt burn: every item the roadmap had deferred is done or dropped — Plex playback per viewer, refusals said in your language, controls that keep the keyboard focus, and a screen reader that hears Hungarian.
|
||||
|
||||
### Features
|
||||
|
||||
- The command palette has a visible way in: a Ctrl K key cap in the header's search field, and a “Jump to…” pill on pages without one.
|
||||
- The Playlists rail says, under “Missing a list? Why?”, which lists YouTube's API does not hand out (Watch later, History, lists saved from other channels).
|
||||
- The last native drop-downs are the app's own controls: the live search's result count, rows per page, a download format's content, audio format and quality, and the download dialog's format.
|
||||
- Plex plays one session per viewer, so two accounts can watch the same film at once; an administrator sets the ceiling on concurrent sessions (Configuration → Plex).
|
||||
|
||||
### Fixes
|
||||
|
||||
- A YouTube action YouTube refuses says what to do — wait for tomorrow's quota, reconnect the account, or try again later — instead of a server error; a passing Google hiccup no longer asks you to reconnect.
|
||||
- Every refusal the server gives is said in your language, and every label a screen reader or tooltip reads is translated.
|
||||
- Dragging to reorder a list is announced in your language and with the right position, and a Plex playlist keeps a drag within the list it started in.
|
||||
- A button that turns itself off, or a row that removes itself, leaves the keyboard focus where it was; Back from a Plex title returns the focus to the tile you opened, even after using a button there.
|
||||
- With Plex turned off, the Home shelf shows your YouTube pins only, and reordering it leaves the hidden Plex pins in their places; two reorders at once can no longer put two pins on one slot.
|
||||
- The Home hero from a pinned show offers to unpin the show, and “Surprise me” from the command palette really asks for a surprise.
|
||||
- A finished Plex title scrobbles once and gets no resume point pushed after it; a subtitle pass that stalled resumes instead of starting over.
|
||||
- Tables keep their j/k keys after the window is resized across the phone breakpoint, and boxes grow with the text-size setting.
|
||||
- Whitelisting an address and activating its account happen in one step, and the audit log names the entry.
|
||||
|
||||
### Internal
|
||||
|
||||
- Every mutation answers `ok`, and route helpers leave the commit to the request — both held by tests; a request that leaves a write uncommitted now fails its test.
|
||||
- New lint rules and contracts: no native select, no hover that ignores aria-disabled, no untranslated accessible text, no unused translation key.
|
||||
- Two adversarial review rounds at the epic's end: 23 findings, all fixed, each regression test proven to fail with its own fix removed.
|
||||
|
||||
## 0.91.0 — 2026-09-19
|
||||
|
||||
What a test run on the live instance turned up, fixed: new videos keep arriving while YouTube's feeds are down, an open tab survives a release, and Back returns to what you left.
|
||||
|
||||
### Features
|
||||
|
||||
- A card's menu (right-click or long press) now offers the card's own actions — watched, save, hide, reset — beside the comfort-shelf pin.
|
||||
- Settings → Account shows your role and your last sign-in. 0.88 announced the last sign-in in your account panel; it is actually there now.
|
||||
|
||||
### Fixes
|
||||
|
||||
- When YouTube's feed for a channel stays dark for hours, as it did for 393 of 396 channels on 18–19 September, the channel's newest uploads are read through the YouTube API instead, at most once every 2 hours per channel (adjustable by an administrator, 0 turns it off).
|
||||
- A tab left open across a release no longer falls to the crash screen when it opens something it had not loaded yet: it reloads onto the new version once.
|
||||
- Back from a request's lines in the Server log returns to the view the pivot was made from, search text included, as 0.90 promised.
|
||||
- Back to Home keeps the question you asked (the time budget and the hero you rerolled to), and the keyboard lands on the card you opened, even when the rail's cards moved or several share a name.
|
||||
- Browser tabs are named after the page again: signed out, and on the privacy and terms pages.
|
||||
- On a phone, the feed and Plex toolbars wrap instead of running past the edge, and so do notifications.
|
||||
- The countdown of a raised diagnostics level counts from the moment it was raised, not from when the page was opened.
|
||||
- The download box's placeholder names every site it takes, Facebook included.
|
||||
|
||||
### Internal
|
||||
|
||||
- The screens' own polling reads (the unread badges, the download list, the scheduler) are logged at debug level in the server's log and in the browser's copy of it, where they made up about 95% of the lines.
|
||||
- An account with no YouTube channel no longer writes a warning every six hours, and a scheduler job that fails carries an event code.
|
||||
- Two adversarial review rounds: 15 findings raised, 11 survived, all fixed, and every regression test was proven to fail with its own fix removed.
|
||||
|
||||
## 0.90.0 — 2026-09-19
|
||||
|
||||
The server's log gets a screen of its own: an admin can now read what the API, the downloader and the browsers wrote down, filter it, and follow one request or job from start to end.
|
||||
|
||||
### Features
|
||||
|
||||
- Admin → Server log: the stored log, newest first, with a search, level chips, and source, account, event and period filters. A line opens in place to its full record; its request id or job id narrows the list to everything that request or job wrote.
|
||||
- Filters live in the address bar, so a filtered view is a link, and Back brings the previous view back.
|
||||
- Live follows the newest lines every few seconds; Copy puts the loaded lines on the clipboard, oldest first. A failed read says so instead of quietly freezing the list.
|
||||
- Both log viewers (Server log and My log) can be walked with j/k and a line opened with Enter.
|
||||
|
||||
### Fixes
|
||||
|
||||
- New videos no longer arrive late from channels whose YouTube feed answers 404 or 500 now and then: a feed is tried again before the channel counts as failed.
|
||||
- A % or _ typed into a search is searched for as itself, not as a wildcard.
|
||||
|
||||
### Internal
|
||||
|
||||
- The console's own reads are logged at debug level, so watching the log does not fill it.
|
||||
- Two adversarial review rounds: 19 findings raised, 12 survived, all fixed, and every regression test was proven to fail with its own fix removed.
|
||||
|
||||
## 0.89.0 — 2026-09-19
|
||||
|
||||
A release for the log rather than for the screen: what the server writes down when something fails is now complete, said once, and free of what you downloaded or searched for.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A download that failed because the server lost its database for a moment no longer shows the video's title and file path inside the database's own error message.
|
||||
|
||||
### Internal
|
||||
|
||||
- No log line carries what a person typed, watched or downloaded: search terms, titles, channel names, file paths, links and video ids are taken out — also from yt-dlp's and ffmpeg's own text, from error stacks, and from database errors. The one exception is Full diagnostics, and only under the user's own consent.
|
||||
- Every one of the backend's 114 swallowed exceptions was ruled on: 44 now leave a line with an event code, the other 70 say in the code why they need none — and a test keeps it that way. A failure is logged once, with its stack when it is our own bug, and a stalled or killed download no longer fails without a line.
|
||||
- yt-dlp's own explanations (“Requested format is not available”, a sign-in wall) reach the log instead of a stdout nobody read, and a burst of identical lines is summarised instead of flooding the stored log.
|
||||
- Three adversarial review rounds: 49 findings raised, 42 survived three independent attempts to refute them, all fixed, and every regression test was proven to fail with its own fix removed.
|
||||
|
||||
## 0.88.0 — 2026-09-18
|
||||
|
||||
When something goes wrong, you can now turn on detailed logging for yourself for 24 hours — and read your own log on the spot, before any of it is sent anywhere. Siftlode also has a privacy policy and terms at last, in both languages, saying exactly what is recorded and for how long.
|
||||
|
||||
### Features
|
||||
|
||||
- Settings → Diagnostics: choosing Detailed or Full changes nothing until you confirm a box that tells you what will be recorded, that the server keeps it, for how long, and that it switches itself off after 24 hours. Going back to Basic needs no confirmation and takes effect at once.
|
||||
- “My log”, on the same tab, shows what your browser has recorded — newest first, any line openable. Nothing leaves the browser before you have consented.
|
||||
- What you search for and filter by is recorded only at Full, and only under your own consent — never an administrator's. Passwords, tokens, cookies and email addresses are recorded at no level, by anyone.
|
||||
- Your last sign-in is shown in your account panel, and in the administrator's user list.
|
||||
- A privacy policy and terms of service, in Hungarian and English, covering what the server logs and what each diagnostics level adds.
|
||||
- Administrators get a Diagnostics section in Configuration: the log level takes effect without a restart, and stored logs are cleaned up on a schedule (7 days by default).
|
||||
- Every setting with a fixed set of values is now picked from those values instead of typed in, so a wrong one can no longer be saved.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Every message arrived twice if you left a conversation and came back quickly: two chat connections stayed open where there should have been one.
|
||||
- Settings was unusable on a phone — the tab rail took the width and left almost nothing for the panel beside it.
|
||||
- Dates and numbers follow the language you picked in Siftlode, not the one your computer happens to be set to. An English Siftlode on a Hungarian machine no longer shows Hungarian dates.
|
||||
- An administrator setting shown as a dropdown opened on its first option instead of the one actually in force.
|
||||
- In Hungarian, the audit log and the new log settings had names you could not tell apart.
|
||||
- The scheduled feed poll now covers only channels somebody actually follows — on this instance 582 of 979 polled channels had no follower at all. The trade-off is deliberate: new uploads from a channel nobody follows no longer appear in the shared Library.
|
||||
|
||||
### Internal
|
||||
|
||||
- One logging mechanism replaced four. The API, the worker, the downloader and the migration tool now write the same structured line; secrets are scrubbed where the line is written rather than at each of ~125 places that write one; and every request carries an id that joins a line from your browser to the server line of the same request. The browser's log uses the same shape and the same redaction rules, checked from both sides against one shared set of cases.
|
||||
- Two adversarial review rounds over the epic: 56 findings raised, 32 survived three independent attempts to refute each, all fixed. Every regression test was proven to fail with its own fix removed — which is how a test that passed only on leftover state from an earlier test was caught, and how a focus bug that jsdom could not reproduce at all was settled in a real browser.
|
||||
|
||||
## 0.87.1 — 2026-09-16
|
||||
|
||||
A YouTube video that YouTube will only show to a signed-in viewer can now be downloaded: paste your YouTube cookies in Settings the way the failure message already told you to — until now that list had no YouTube in it.
|
||||
|
||||
### Features
|
||||
|
||||
- YouTube joins the sites you can store download cookies for (Settings → Account → Download authentication). A signed-in YouTube download automatically uses the players that accept your login.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A YouTube download that stops at “Sign in to confirm you’re not a bot” says so in your language and points at a setting that now exists. If you had already added cookies, it tells you they were refused instead of asking for them again.
|
||||
- A video YouTube describes but then refuses to hand over is tried again with another player instead of failing at once, and it is no longer called a network error — nothing was wrong with the network.
|
||||
|
||||
### Internal
|
||||
|
||||
- The gated video was measured end to end on an isolated stack before and after the fix: every player client, two yt-dlp versions, and finally the real download (519 MB) with cookies attached. One deep-review round — 25 findings raised, none survived adversarial verification, and the one real defect came out of the verification itself: a comment of mine named an error yt-dlp does not raise there. Running it settled what reading it could not.
|
||||
|
||||
## 0.87.0 — 2026-09-15
|
||||
|
||||
A shared or library video keeps playing when you switch apps on an iPad, and a new tab opens as the account it belongs to without flashing another page first. Facebook reels keep their titles and an mp3 stays an mp3.
|
||||
|
||||
### Features
|
||||
|
||||
- Background play on iPad (Brave): the shared-video page, the library player and the Plex player keep playing in another app and on the locked screen.
|
||||
- The shared-video page has a Repeat toggle, on by default.
|
||||
- Switching modules no longer flashes a loading screen: the pages load quietly in the background, and a slow answer shows its spinner only after a moment.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A new tab honours your Startup choice and opens as its own account — no Home page flash, no other account's language or settings on the first frame.
|
||||
- A signed-in Facebook reel keeps its real title instead of “Video” or “Facebook”, and an mp3 download stays an mp3 (it used to be wrapped into an mp4 an iPad would not play).
|
||||
- A tab whose account was signed out, suspended or signed out everywhere acts as nobody and comes back as the account still signed in, instead of writing into another account or sticking on the sign-in page.
|
||||
- Switching or adding an account no longer saves the video you were watching into the other account's resume position.
|
||||
- A new tab keeps the account's notification history instead of replacing it with its own first notice.
|
||||
- The shared-video page tells a failed load, a wrong password, too many attempts and an unreachable server apart, with Retry where it helps. Its password limit counts per viewer, so one person guessing cannot lock everyone else out.
|
||||
- Form errors on sign-in, the message passphrase, password change and setup are read out by screen readers — again on a second identical failure — and the setup wizard names what is wrong instead of a generic error.
|
||||
|
||||
### Internal
|
||||
|
||||
- WebKit and iPad projects in the Playwright suite; the Vite dev proxy forwards websockets.
|
||||
- Four deep-review rounds (12 → 8 → 8 → 8, the last one targeted), every regression test mutation-checked; half of the later rounds' findings were older defects, fixed here.
|
||||
|
||||
## 0.86.0 — 2026-09-14
|
||||
|
||||
A failed download now tells you why it failed and what you can do about it, instead of repeating yt-dlp's raw error. Facebook comes first: downloads that used to die on “Cannot parse data” go through, and a Facebook group post downloads its own video instead of a different one from the page.
|
||||
|
||||
### Features
|
||||
|
||||
- A failure says why, in your language: the row names the cause (“Facebook needs you to be signed in”, “Facebook did not accept your cookies”, “No video found in this post”) and its details window says what to do next. The raw error text is still there below it.
|
||||
- A Facebook download with your cookies that stops at “Cannot parse data” is tried again as Chrome, which gets past it.
|
||||
- A Facebook group post link downloads the post's own video. Before, yt-dlp either found nothing or quietly downloaded another video from the page; a post without a video now says so.
|
||||
- Retry and Re-download use a cookie you added after the failure, so the Retry the message asks for actually goes out signed in.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A Re-download of a download that needed your cookies used to rebuild without them.
|
||||
- A job queued behind a download that already failed shows that failure's explanation instead of a bare “Download failed”.
|
||||
- A download yt-dlp reports no length for takes its length from the file, so its Edit / trim action appears.
|
||||
- Pressing Pause, Stop or Remove twice, or in two tabs, can no longer free a file another download still uses.
|
||||
- The confirm buttons in Downloads name the action (Stop download, Re-download, Remove) instead of repeating the question.
|
||||
|
||||
### Internal
|
||||
|
||||
- Four sprints, each Facebook behaviour measured on the development server with a real cookie before it was built. Two epic-end review rounds (20 → 17 fixed, then a targeted 4 → 4), every regression test mutation-checked.
|
||||
|
||||
## 0.85.0 — 2026-09-13
|
||||
|
||||
Siftlode can now be driven from the keyboard, and Back, dialogs and menus stop losing your place. j/k and the arrow keys walk every list and grid, Enter opens what you are on, and Back puts you on the item you opened a view from instead of at the top of the page.
|
||||
|
||||
### Features
|
||||
|
||||
- j/k and the arrow keys walk the feed, the library, a channel page, the Home rails, every Plex grid and rail, the Channels, Playlists and Downloads tables and the conversation list; Enter opens the item you are on. After you hide or delete an item, the next j goes on from where it was.
|
||||
- Back returns you to the item a view was opened from — the Plex title you opened from a Home rail, the channel you opened from its row — with the keyboard still on it.
|
||||
- Shortcuts to get around: s jumps to the search field, g-chords jump to a module, and the cheat sheet (h) names every key in the language you are reading. PageUp/PageDown move a page at a time in the command palette and in every dropdown list.
|
||||
- The command palette ranks what you typed first: “theme” now puts the Theme rows above a channel whose handle happens to contain it.
|
||||
- A card reached with Tab or j shows one ring around the whole card, and a screen reader calls it by its title.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Back and overlays keep their books straight: Back after navigating over About also closes About, a late clean-up no longer undoes the navigation you just made, and Shift+W on Home closes the player.
|
||||
- Dialogs and menus hand the keyboard back where it came from. Escape from a card’s context menu returns to the card, not its thumbnail link; closing the player under another dialog no longer leaves the page unscrollable; a button that is busy with a request keeps the focus instead of dropping it to the page.
|
||||
- A write that fails says so, with the reason — several saves, share links and admin actions used to fail silently or in English under the Hungarian interface.
|
||||
- A split video export that fails part-way (the job limit) tells you how many clips were added and leaves only the rest selected, so trying again does not queue the first ones twice.
|
||||
- “Feed” is called Feed everywhere, in both languages.
|
||||
|
||||
### Internal
|
||||
|
||||
- Five sprints: history integrity, one dialog/focus contract, palette ranking and accessibility, the keyboard layer, and a Hungarian/English keyboard-only pass. Focus and event-order questions were measured in a real browser before each fix, not argued.
|
||||
- Three epic-end review rounds — 19 → 14, 18 → 12 and a targeted 14 → 14 — each regression test mutation-checked. The last round's findings all sat inside the previous round's own fixes, which is where the loop was closed.
|
||||
|
||||
## 0.84.0 — 2026-09-12
|
||||
|
||||
A bug-fix release, from a list you wrote while using Siftlode. Two of these were destroying your own data: a YouTube video rewound itself when you reopened it, and clearing a resume point on a half-watched Plex item told Plex nothing. Subtitles now appear in seconds instead of minutes, the Plex player's bitrate is a real measurement rather than a guess, and a channel you turn full history on for fills in front of you instead of looking frozen.
|
||||
|
||||
@@ -56,15 +56,16 @@ how it gets built. Concretely:
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`security`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||
feature component or a domain API slice (`no-restricted-imports` in `eslint.config.js`, with its own
|
||||
test). A new generic component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
feature component or a domain API slice, and one layer down `lib/` may not import from `components/` at
|
||||
all, not even a type (`no-restricted-imports` in `eslint.config.js`, with its own test). A new generic
|
||||
component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
Data via react-query (`qk` factory), per-account `localStorage`, and server prefs; API through
|
||||
`lib/api/*` slices over `core.req<T>`.
|
||||
|
||||
@@ -81,16 +82,21 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`) | ad-hoc `{"unsubscribed"}`/`{"cleared": n}`/echo |
|
||||
| Transaction commit | the handler/job-wrapper commits; helpers `flush` | helpers that commit (only `quota` owns its session) |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE control | the chip/segmented primitives (`Select` only for >6 options) | a native `<select>` |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN adduser --disabled-password --gecos "" appuser && chown -R appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["sh", "entrypoint.sh"]
|
||||
@@ -1,6 +1,8 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
# Split the path list the way the OS does (alembic's legacy space/comma/colon split is deprecated).
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from app.config import settings
|
||||
from app.db import Base
|
||||
from app.logs import configure_logging
|
||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
||||
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
# Migrations run at container start (entrypoint.sh), so their output is part of the API's log
|
||||
# stream — one format with everything else, not alembic.ini's own fileConfig shape.
|
||||
configure_logging("alembic")
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""add download_jobs.error_detail — the domain-specific entry of the failure catalog (E11.5 S1)
|
||||
|
||||
Revision ID: 0084_dl_job_error_detail
|
||||
Revises: 0083_comfort_pins
|
||||
Create Date: 2026-09-13
|
||||
|
||||
`error_code` is the generic category (it decides whether a Retry is offered). `error_detail` names
|
||||
the host-specific catalog entry `downloads.errors.classify` matched on top of it — `facebook.login_wall`
|
||||
and its siblings — so the UI can say what went wrong ON THAT SITE and what to do about it, instead of
|
||||
the category's generic sentence. Nullable: most failures have no host-specific entry.
|
||||
|
||||
Backfill: a job's code is written once, at failure time, so a new rule never reaches the rows that
|
||||
failed before it. The one shape measured on prod (2026-09-13, jobs 9/66/75/145/156/157/182/256/257) is
|
||||
yt-dlp's `[facebook] <id>: Cannot parse data`, stored as `unknown`. Without a cookie that is Facebook's
|
||||
login wall. Whether a cookie was attached at the time is not recorded anywhere, so every old row gets
|
||||
the no-cookie entry — on prod no account had a Facebook cookie, so that is what they all were.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0084_dl_job_error_detail"
|
||||
down_revision: str | None = "0083_comfort_pins"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("error_detail", sa.String(length=48), nullable=True))
|
||||
op.execute(
|
||||
sa.text(
|
||||
"UPDATE download_jobs SET error_code = 'login_required', "
|
||||
"error_detail = 'facebook.login_wall' "
|
||||
"WHERE (error_code IS NULL OR error_code = 'unknown') "
|
||||
"AND error ILIKE '%[facebook]%cannot parse data%'"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "error_detail")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""add users.last_login_at — when the account last signed in (E22 S1)
|
||||
|
||||
Revision ID: 0085_users_last_login_at
|
||||
Revises: 0084_dl_job_error_detail
|
||||
Create Date: 2026-09-16
|
||||
|
||||
Stamped by the one sign-in path (`routes/auth._establish_session`), which the Google callback and
|
||||
password login share. No backfill is possible: the signal exists only at sign-in, so existing rows
|
||||
read NULL until each account signs in again, and the admin UI says so rather than inventing a date.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0085_users_last_login_at"
|
||||
down_revision: str | None = "0084_dl_job_error_detail"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "last_login_at")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""add log_events — stored log events for diagnostics (E22 S3)
|
||||
|
||||
Revision ID: 0086_log_events
|
||||
Revises: 0085_users_last_login_at
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Written by app/logsink.py's background writer (WARNING and above, request lines), pruned by the log_gc
|
||||
job after `log_retention_days`. Sized against prod on 2026-09-16: ~2k rows on a normal day, ~14k on a
|
||||
YouTube-RSS-outage day before flood control — a few MB over the 7-day window. `user_id` carries no
|
||||
foreign key on purpose (see models.LogEvent).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0086_log_events"
|
||||
down_revision: str | None = "0085_users_last_login_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"log_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("level", sa.String(10), nullable=False),
|
||||
sa.Column("sev", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("logger", sa.String(128), nullable=False),
|
||||
sa.Column("event", sa.String(64), nullable=True),
|
||||
sa.Column("msg", sa.Text(), nullable=False),
|
||||
sa.Column("attrs", postgresql.JSONB(), nullable=True),
|
||||
sa.Column("exc", sa.Text(), nullable=True),
|
||||
sa.Column("request_id", sa.String(64), nullable=True),
|
||||
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||
sa.Column("debug_session_id", sa.String(64), nullable=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True),
|
||||
sa.Column("version", sa.String(64), nullable=True),
|
||||
)
|
||||
op.create_index("ix_log_events_ts", "log_events", ["ts"])
|
||||
op.create_index("ix_log_events_request_id", "log_events", ["request_id"])
|
||||
op.create_index("ix_log_events_user_ts", "log_events", ["user_id", "ts"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("log_events")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add users.diagnostics_level + diagnostics_until — a user's consented diagnostics level (E22 S5)
|
||||
|
||||
Revision ID: 0087_users_diagnostics_level
|
||||
Revises: 0086_log_events
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Both NULL = the base level (Alap). A raised level is ALWAYS paired with its expiry, computed by the
|
||||
server at consent time (`app/diagnostics.py`), so no client can ask for a level that never ends.
|
||||
Columns rather than `users.preferences`: that JSON is written wholesale by the client, and a consent
|
||||
with a lifetime must not be something a client can write.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0087_users_diagnostics_level"
|
||||
down_revision: str | None = "0086_log_events"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("diagnostics_level", sa.String(8), nullable=True))
|
||||
op.add_column(
|
||||
"users", sa.Column("diagnostics_until", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "diagnostics_until")
|
||||
op.drop_column("users", "diagnostics_level")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""add channels.feed_fallback_at — when a channel's uploads were last read through the API because
|
||||
its RSS feed would not answer
|
||||
|
||||
Revision ID: 0088_channels_feed_fallback_at
|
||||
Revises: 0087_users_diagnostics_level
|
||||
Create Date: 2026-09-19
|
||||
|
||||
YouTube's RSS feeds go dark for hours at a time, per channel (2026-09-18/19: 393 of 396 answered 404
|
||||
for four hours; the same channel answers 404 to every retry). The RSS poll then reads the newest
|
||||
uploads through the Data API instead, at a quota cost, and at most once per `rss_fallback_hours` per
|
||||
channel: this column is that clock. NULL = never needed it.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0088_channels_feed_fallback_at"
|
||||
down_revision: str | None = "0087_users_diagnostics_level"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"channels", sa.Column("feed_fallback_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("channels", "feed_fallback_at")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""add debug_sessions + debug_screenshots, and index log_events.debug_session_id (E16)
|
||||
|
||||
Revision ID: 0089_debug_sessions
|
||||
Revises: 0088_channels_feed_fallback_at
|
||||
Create Date: 2026-09-19
|
||||
|
||||
A debug session is one recording: Start → reproduce → Stop (app/debugsession.py). Every log line
|
||||
of that user's requests while it runs carries its id in `log_events.debug_session_id` — the column
|
||||
E22 created and left NULL — and the issue it ends in is built from exactly those rows, so the column
|
||||
gets its index now. Partial: nearly every row has no session, and an index of NULLs would be most
|
||||
of the table for nothing.
|
||||
|
||||
At most one OPEN session per user, held by a partial unique index rather than by the start handler
|
||||
alone: two tabs pressing Start at once would otherwise both pass a "none open" check.
|
||||
|
||||
Screenshots are rows, not files: bounded (size and count per session, app/debugsession.py), gone
|
||||
with their session by cascade, and a self-host has no file volume to configure for them.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0089_debug_sessions"
|
||||
down_revision: str | None = "0088_channels_feed_fallback_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"debug_sessions",
|
||||
sa.Column("id", sa.String(16), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("level", sa.String(8), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("until", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("ended_reason", sa.String(16), nullable=True),
|
||||
sa.Column("issue_url", sa.String(512), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"debug_sessions",
|
||||
["user_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("ended_at IS NULL"),
|
||||
)
|
||||
op.create_table(
|
||||
"debug_screenshots",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"session_id",
|
||||
sa.String(16),
|
||||
sa.ForeignKey("debug_sessions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("mime", sa.String(16), nullable=False),
|
||||
sa.Column("size", sa.Integer, nullable=False),
|
||||
sa.Column("data", sa.LargeBinary, nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_log_events_debug_session",
|
||||
"log_events",
|
||||
["debug_session_id"],
|
||||
postgresql_where=sa.text("debug_session_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_log_events_debug_session", table_name="log_events")
|
||||
op.drop_table("debug_screenshots")
|
||||
op.drop_index("uq_debug_sessions_open_per_user", table_name="debug_sessions")
|
||||
op.drop_table("debug_sessions")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""add debug_sessions.draft — the issue being written from a session (E16 S3)
|
||||
|
||||
Revision ID: 0090_debug_session_draft
|
||||
Revises: 0089_debug_sessions
|
||||
Create Date: 2026-09-19
|
||||
|
||||
The composer saves as it goes: the title, the rendered markdown body that an export or an issue
|
||||
carries, the log rows the user took out, and the form's own fields so reopening it later (Settings →
|
||||
Diagnosztika lists past sessions) shows what was written. NULL until the composer first saves.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0090_debug_session_draft"
|
||||
down_revision: str | None = "0089_debug_sessions"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("debug_sessions", sa.Column("draft", JSONB, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("debug_sessions", "draft")
|
||||
@@ -13,16 +13,18 @@ Three layers, by *who* runs the code — not by feature:
|
||||
response models, status-code mapping, serialization. It owns logic that runs *only* on a request.
|
||||
Business logic that a request happens to trigger does **not** live here if anything else runs it too.
|
||||
|
||||
2. **`plex/`, `downloads/`, `sync/` — domain logic shared by more than one caller.** Anything the
|
||||
2. **`plex/`, `downloads/`, `sync/`, `issues/` — domain logic shared by more than one caller.** Anything the
|
||||
**scheduler or the worker** runs (not just a request) lives in one of these domain packages, so the
|
||||
route, the worker, and the scheduler share one definition. Examples from this codebase:
|
||||
`downloads/service.py` owns enqueue + the job state transitions (routes call it); `plex/sync.py` is
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline.
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, … are single-file primitives many
|
||||
layers import. They depend on nothing above them (a core module must never import a `routes/…`
|
||||
module).
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
*second* caller needs it (a worker as well as a route; a scheduler as well as a route), extract it down
|
||||
|
||||
@@ -58,6 +58,9 @@ class AuditAction:
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
# E16: a user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||
ISSUE_CREATE = "issue_create"
|
||||
|
||||
|
||||
def record(
|
||||
|
||||
+94
-29
@@ -1,19 +1,20 @@
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import BackgroundTasks, Depends, HTTPException, Request
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app import debugsession
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.models import Invite, LogEvent, User
|
||||
from app.security import decrypt_optional
|
||||
from app.utils import canonical_email
|
||||
from app.refusal import refused
|
||||
|
||||
PASSWORD_MIN_LEN = 10
|
||||
|
||||
@@ -37,7 +38,7 @@ READ_SCOPES = f"{BASE_SCOPES} {READ_SCOPE}"
|
||||
WRITE_SCOPES = f"{BASE_SCOPES} {READ_SCOPE} {WRITE_SCOPE}"
|
||||
_YOUTUBE_SCOPES = frozenset({READ_SCOPE, WRITE_SCOPE})
|
||||
|
||||
log = logging.getLogger("siftlode.auth")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def google_enabled(db: Session) -> bool:
|
||||
@@ -220,7 +221,7 @@ def revoke_google_token(token: str | None) -> None:
|
||||
timeout=10,
|
||||
)
|
||||
log.info("Google token revoke: status=%s", resp.status_code)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a revoke we cannot make must not fail the sign-out
|
||||
log.warning("Google token revoke failed", exc_info=True)
|
||||
|
||||
|
||||
@@ -245,6 +246,12 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
)
|
||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||
db.execute(delete(User).where(User.id == user_id))
|
||||
# Stored log events name the account by id with no foreign key to cascade from (models.LogEvent),
|
||||
# so they go explicitly — they are request history, i.e. personal data. Forgotten FIRST: this very
|
||||
# request's access line, a queued row, a later background task would otherwise put rows back
|
||||
# under the id after this DELETE (logs.forget_user).
|
||||
logs.forget_user(user_id)
|
||||
db.execute(delete(LogEvent).where(LogEvent.user_id == user_id))
|
||||
# Every canonically-equal invite goes, not just the literal spelling: is_allowed matches on the
|
||||
# canonical form, so leaving a sibling row behind would keep granting access to a mailbox whose
|
||||
# account was just erased — and prod really does carry approved pairs for one mailbox.
|
||||
@@ -269,52 +276,85 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
ACTIVE_ACCOUNT_HEADER = "x-siftlode-account"
|
||||
|
||||
|
||||
def resolved_user_id(conn: HTTPConnection) -> tuple[int | None, bool]:
|
||||
def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]:
|
||||
"""Which account this connection acts as, and whether that's the session's default account.
|
||||
Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this
|
||||
exact per-tab resolution instead of re-implementing it.
|
||||
|
||||
The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser)
|
||||
plus a default `user_id`. A caller may override the default with the X-Siftlode-Account header,
|
||||
but ONLY for an account already in the wallet — so a tab can't impersonate an account that never
|
||||
authenticated here. Everything else (WebSocket, plain navigations) keeps using the cookie default.
|
||||
plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or
|
||||
`?account=`), but ONLY an account in the wallet — so a tab can't impersonate an account that
|
||||
never authenticated here. A connection that names no account acts as the cookie default.
|
||||
|
||||
A connection naming an account NOT in the wallet acts as nobody (`None`) — typically a tab still
|
||||
pinned to an account that was signed out in another tab. It must not fall back to the default:
|
||||
that tab would go on acting as an account it never showed, its player checkpoints and pref saves
|
||||
landing there and its Sign out signing THAT account out. Only the bootstrap /api/me probe passes
|
||||
`strict=False` and gets the default, which is how such a tab learns who it now is (the frontend
|
||||
re-pins it and reloads). An account whose session ended while it stayed in the wallet (deleted,
|
||||
suspended, revoked) leaves the wallet when it is refused (`_session_user`), so a tab pinned to it
|
||||
takes the same way back.
|
||||
"""
|
||||
default_id = conn.session.get("user_id")
|
||||
wallet = conn.session.get("account_ids") or []
|
||||
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
|
||||
# plain <a> file download). Both are wallet-gated below, so neither can impersonate an
|
||||
# account that never signed into this browser.
|
||||
# plain <a> file download). Both are wallet-gated below.
|
||||
hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account")
|
||||
if hdr:
|
||||
try:
|
||||
hid = int(hdr)
|
||||
except ValueError:
|
||||
# Under `strict` this becomes a 401 and the tab authenticates as nobody, which reads
|
||||
# in the browser exactly like a session that expired. The value itself stays out —
|
||||
# it came from a request, and `wallet` says which accounts were actually on offer.
|
||||
log.debug(
|
||||
"the active-account header could not be read as an id",
|
||||
extra=logs.event("auth.account_header.invalid", accounts=len(wallet)),
|
||||
)
|
||||
hid = None
|
||||
if hid is not None and hid in wallet:
|
||||
return hid, hid == default_id
|
||||
if strict:
|
||||
return None, False
|
||||
return default_id, True
|
||||
|
||||
|
||||
def drop_from_wallet(session: dict, user_id: int) -> None:
|
||||
"""Take one account out of this browser's wallet (`account_ids`); the session default is untouched."""
|
||||
session["account_ids"] = [a for a in (session.get("account_ids") or []) if a != user_id]
|
||||
|
||||
|
||||
def _end_account_session(request: Request, user_id: int, is_default: bool) -> None:
|
||||
"""An account whose session here is no longer valid. Only nuke the whole session when it is the
|
||||
SESSION DEFAULT; a per-tab account just leaves the wallet (other tabs' default may still be valid).
|
||||
Leaving is what lets a tab pinned to it recover: its next /api/me names an account no longer signed
|
||||
in here and gets the default. Left in, it answered "signed out" — the landing page, with the default
|
||||
still signed in, and no way off it but closing the tab."""
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
else:
|
||||
drop_from_wallet(request.session, user_id)
|
||||
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
user_id, is_default = resolved_user_id(request)
|
||||
return _session_user(request, db, *resolved_user_id(request))
|
||||
|
||||
|
||||
def _session_user(request: Request, db: Session, user_id: int | None, is_default: bool) -> User:
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
user = db.get(User, user_id)
|
||||
if user is None or not user.is_active or user.is_suspended:
|
||||
# Suspended/deactivated mid-session → drop the session so the block takes effect at once.
|
||||
# But only nuke the whole session when the SESSION DEFAULT identity is gone; a stale
|
||||
# per-tab header account just 401s that one tab (other tabs' default may still be valid).
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
# Suspended/deactivated mid-session → end it so the block takes effect at once.
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise refused(401, "not_authenticated")
|
||||
# SA4: reject a cookie whose recorded epoch is behind the account's current one (revoked by a
|
||||
# password reset/change or "log out everywhere"). A cookie with no recorded epoch (pre-SA4, or an
|
||||
# account signed in before this shipped) is treated as epoch 0 — so the FIRST bump revokes it too.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(user_id), 0)) != (user.session_epoch or 0):
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise refused(401, "not_authenticated")
|
||||
# Always keep the session default in the switchable list — covers sessions created before
|
||||
# multi-session existed (their account_ids was never seeded), so the switcher isn't empty.
|
||||
default_id = request.session.get("user_id")
|
||||
@@ -322,16 +362,44 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
if default_id and default_id not in accounts:
|
||||
accounts.append(default_id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
# Attribute this request's log lines to the account (E22). Here, not in the request middleware:
|
||||
# this is the one place a session becomes an account, and the middleware runs outside both the
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
# So does a debug session the request names (E16): only the caller's own open one counts, and a
|
||||
# header naming anything else is ignored rather than refused — a stale tab must keep working.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id if session else None)
|
||||
request.state.user_id = user.id
|
||||
return user
|
||||
|
||||
|
||||
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
||||
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it).
|
||||
The one lenient resolution (`strict=False`): a tab naming an account no longer signed in here gets
|
||||
the default account, which is how it learns who it now is — see `resolved_user_id`. That includes an
|
||||
account refused right here, which `_session_user` has just taken out of the wallet."""
|
||||
user_id, is_default = resolved_user_id(request, strict=False)
|
||||
try:
|
||||
return current_user(request, db)
|
||||
return _session_user(request, db, user_id, is_default)
|
||||
# silent-ok: this is the anonymous case as much as the refused one — every landing page hit
|
||||
# comes through here. The second attempt below is the handling, and it reports.
|
||||
except HTTPException:
|
||||
if is_default:
|
||||
return None
|
||||
try:
|
||||
return _session_user(request, db, *resolved_user_id(request, strict=False))
|
||||
except HTTPException:
|
||||
# Both the pinned account AND the default one were refused, so the tab is signed out
|
||||
# without being told why — and it got here from a session that named a real account.
|
||||
log.info(
|
||||
"a session was resolved to nobody: neither the named nor the default account held",
|
||||
extra=logs.event("auth.session.probe_rejected", user_id=user_id),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -355,10 +423,7 @@ def require_write_scope(user: User = Depends(require_human)) -> User:
|
||||
checks that had drifted into three different 403 messages across channels/playlists. Layers on
|
||||
`require_human` because writing to YouTube needs a real, non-demo identity."""
|
||||
if not has_write_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Enable YouTube editing in Settings first.",
|
||||
)
|
||||
raise refused(403, "youtube_editing_off")
|
||||
return user
|
||||
|
||||
|
||||
@@ -366,7 +431,7 @@ def admin_user(user: User = Depends(current_user)) -> User:
|
||||
"""Dependency: require the signed-in user to be an admin. Single source of truth for the
|
||||
admin gate — every admin route/action depends on this rather than re-checking the role."""
|
||||
if user.role != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin only")
|
||||
raise refused(403, "admin_only")
|
||||
return user
|
||||
|
||||
|
||||
|
||||
+81
-6
@@ -1,6 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic import ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -15,11 +16,40 @@ def _version_from_file() -> str:
|
||||
v = p.read_text(encoding="utf-8").strip()
|
||||
if v:
|
||||
return v
|
||||
# silent-ok: this runs at import, and `app.logs` imports THIS module — a log call here
|
||||
# would be a circular import. The loop tries the other location, then falls back to "dev".
|
||||
except OSError:
|
||||
continue
|
||||
return "dev"
|
||||
|
||||
|
||||
# Closed value sets. A setting typed with one of these is a CHOICE: pydantic rejects anything else at
|
||||
# boot, and sysconfig derives the admin picker's options from the same Literal — so the list lives in
|
||||
# exactly one place and a consumer never has to guess what a typo should fall back to.
|
||||
SearchSource = Literal["scrape", "api"]
|
||||
DownloadLayout = Literal["plex", "flat"]
|
||||
LogLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR"]
|
||||
LogFormat = Literal["json", "text"]
|
||||
TranscodeHw = Literal["auto", "on", "off"]
|
||||
# libx264's presets, fastest first (`placebo` left out on purpose: no measurable gain for its cost).
|
||||
TranscodePreset = Literal[
|
||||
"ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"
|
||||
]
|
||||
|
||||
|
||||
def choices_of(annotation: object) -> tuple[str, ...]:
|
||||
"""The allowed values of a choice-typed setting, or () for any other type."""
|
||||
return tuple(get_args(annotation)) if get_origin(annotation) is Literal else ()
|
||||
|
||||
|
||||
def canonical_choice(options: tuple[str, ...], raw: object) -> str | None:
|
||||
"""The option `raw` names, in its canonical spelling (any letter case, surrounding space
|
||||
ignored), or None. The ONE matching rule for a choice value from the env and from the admin
|
||||
API, so a spelling one accepts the other cannot refuse."""
|
||||
folded = str(raw).strip().casefold()
|
||||
return next((option for option in options if option.casefold() == folded), None)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
@@ -57,7 +87,7 @@ class Settings(BaseSettings):
|
||||
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
||||
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
||||
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
||||
# deploy. Comma-separated. See routes.auth._client_ip + the deploy docs.
|
||||
# deploy. Comma-separated. See ratelimit.client_key + the deploy docs.
|
||||
trusted_proxy_ips: str = ""
|
||||
|
||||
# --- Outbound email (onboarding: access-request + approval notices) ---
|
||||
@@ -85,10 +115,13 @@ class Settings(BaseSettings):
|
||||
search_daily_limit_per_user: int = 70
|
||||
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
||||
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
||||
search_source: str = "scrape"
|
||||
search_source: SearchSource = "scrape"
|
||||
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
||||
backfill_recent_max_videos: int = 100
|
||||
backfill_recent_max_days: int = 365
|
||||
# A channel whose RSS feed has not answered for this many hours gets its newest uploads read
|
||||
# through the Data API instead (1 unit), at most once per this many hours. 0 = never.
|
||||
rss_fallback_hours: int = 2
|
||||
|
||||
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
||||
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
||||
@@ -155,6 +188,9 @@ class Settings(BaseSettings):
|
||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
||||
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
||||
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
||||
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
||||
download_player_clients: str = "android_vr"
|
||||
download_player_clients_fallback: str = "web_safari,web,android"
|
||||
|
||||
@@ -221,10 +257,27 @@ class Settings(BaseSettings):
|
||||
audit_retention_days: int = 180
|
||||
audit_gc_minutes: int = 1440
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: str = "plex"
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging (E22) ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
# startup and on change without a restart. This is what a process logs at before it can reach
|
||||
# the database — and the only knob the worker/runner children read, since they are short-lived.
|
||||
# TRACE (below DEBUG) is the "Teljes" diagnostics level; see app/logs.py.
|
||||
log_level: LogLevel = "INFO"
|
||||
# "json" (one object per line — what Loki and the log_events sink ingest) or "text" (a readable
|
||||
# line for a local terminal). Redaction applies to both.
|
||||
log_format: LogFormat = "json"
|
||||
# Also write events to the `log_events` table (app/logsink.py): WARNING and above, the request
|
||||
# lines, and — once they exist — client batches. Off only where no table is wanted (the tests).
|
||||
log_db_sink: bool = True
|
||||
# Days a `log_events` row is kept (the privacy policy promises this), and how often the log_gc job
|
||||
# enforces it — hourly, so a row never outlives the promise by more than an hour.
|
||||
log_retention_days: int = 7
|
||||
log_gc_minutes: int = 60
|
||||
|
||||
# --- Plex integration (optional) ---
|
||||
# The Plex module plays the LOCAL physical media file directly (our own ffmpeg/direct-serve);
|
||||
# Plex is used only for metadata + optional watch-state sync. Requires the Plex server to be
|
||||
@@ -262,11 +315,15 @@ class Settings(BaseSettings):
|
||||
# re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve
|
||||
# and remux (stream-copy) are not capped by this.
|
||||
plex_max_transcodes: int = 4
|
||||
# Max concurrent playback sessions of ANY kind — remux (stream-copy) included. Cheap on the CPU but
|
||||
# each one is a whole segmented copy of the film on the scratch volume, written faster than
|
||||
# realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`).
|
||||
plex_max_sessions: int = 8
|
||||
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
||||
plex_transcode_hw: str = "auto"
|
||||
plex_transcode_hw: TranscodeHw = "auto"
|
||||
# Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf`
|
||||
# trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output
|
||||
# height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always
|
||||
@@ -276,7 +333,7 @@ class Settings(BaseSettings):
|
||||
# call, not a fix. Only affects files that go through remux/transcode — a direct-played file is
|
||||
# served untouched, but those are browser-native stereo anyway, not the 5.1 mixes this is for.
|
||||
plex_audio_loudnorm: bool = False
|
||||
plex_transcode_preset: str = "veryfast"
|
||||
plex_transcode_preset: TranscodePreset = "veryfast"
|
||||
plex_transcode_crf: int = 20
|
||||
plex_transcode_max_height: int = 1080
|
||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||
@@ -294,6 +351,13 @@ class Settings(BaseSettings):
|
||||
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
|
||||
# --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the
|
||||
# composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's
|
||||
# log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). ---
|
||||
issues_gitea_url: str = ""
|
||||
issues_gitea_repo: str = ""
|
||||
issues_gitea_token: str = ""
|
||||
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
@@ -325,6 +389,17 @@ class Settings(BaseSettings):
|
||||
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
||||
return self.oauth_redirect_url.lower().startswith("https://")
|
||||
|
||||
@field_validator("*", mode="before")
|
||||
@classmethod
|
||||
def _canonical_choice(cls, value: object, info: ValidationInfo) -> object:
|
||||
"""A choice setting accepts its value in any letter case (`LOG_LEVEL=debug`) and stores the
|
||||
canonical spelling; an unknown value still fails the boot with pydantic's own message."""
|
||||
field = cls.model_fields.get(info.field_name or "")
|
||||
options = choices_of(field.annotation) if field else ()
|
||||
if options and isinstance(value, str):
|
||||
return canonical_choice(options, value) or value
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _enforce_production_secrets(self) -> "Settings":
|
||||
"""Fail fast rather than boot a public instance with the shipped placeholder
|
||||
|
||||
@@ -20,6 +20,10 @@ engine = create_engine(
|
||||
max_overflow=30,
|
||||
pool_timeout=30,
|
||||
future=True,
|
||||
# A database error renders its bound parameters into its message — a title, an uploader, a
|
||||
# file path, a URL — and that message is logged and stored as a job's error text. The SQL
|
||||
# stays; the values are data, and the ids an operator needs are on the line already.
|
||||
hide_parameters=True,
|
||||
)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
|
||||
While a session is open its user's browser names it in `X-Debug-Session` on every request. The auth
|
||||
dependency accepts the header only for the caller's OWN open session (anything else is ignored, never
|
||||
an error — a stale tab must not break the app) and then logs that request at the session's level with
|
||||
every line tagged by its id; the browser tags its own events the same way. The issue is later built
|
||||
from exactly the rows carrying the tag, so what the review list shows is what gets sent.
|
||||
|
||||
Starting a session IS the consent for its level, for its duration: the start dialog carries the
|
||||
consent text. It needs no change to the user's own diagnostics level, and none is made — when the
|
||||
session ends the level in force is simply the user's own again.
|
||||
|
||||
A session is capped at `MAX_MINUTES`, set by the server at Start. An open one past its cap is over
|
||||
whether or not anything has closed it yet: it reads as over at once, a new Start closes it, and the
|
||||
hourly log job sweeps the rest. Sessions go with the logs they point at, after `log_retention_days`.
|
||||
|
||||
Core layer: imports `models`, `diagnostics`.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from typing import Literal, get_args
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, delete, func, not_, select, update
|
||||
from sqlalchemy.orm import Session, defer
|
||||
|
||||
from app import diagnostics
|
||||
from app.models import DebugScreenshot, DebugSession, LogEvent, User
|
||||
|
||||
HEADER = "x-debug-session"
|
||||
MAX_MINUTES = diagnostics.SESSION_MINUTES
|
||||
SCREENSHOT_MAX_BYTES = 10 * 1024 * 1024
|
||||
SCREENSHOTS_MAX = 8
|
||||
# What one user may keep in screenshots across ALL their sessions (until retention takes them): the
|
||||
# per-session cap alone would let Start → 8 images → Stop repeat without end.
|
||||
SCREENSHOT_USER_MAX_BYTES = 100 * 1024 * 1024
|
||||
# The screenshot types kept, and the extension each is exported and sent under — one table, so an
|
||||
# accepted type always has a name.
|
||||
IMAGE_TYPES = {"image/png": "png", "image/jpeg": "jpg", "image/webp": "webp"}
|
||||
# What a screenshot is called in the zip and on the issue. `issues/bundle.py` builds the names from
|
||||
# this and `IMAGE_TYPES`; this module and `issues/gitea.py` read them back to tell a screenshot's
|
||||
# link from a log's. One definition, because three copies of a name is how the next rename breaks
|
||||
# `images_pending` into silence.
|
||||
SCREENSHOT_PREFIX = "screenshot-"
|
||||
# Rows an issue can carry. A 60-minute session at Teljes is a few thousand rows; past this the
|
||||
# bundle says how many it left out rather than growing without bound.
|
||||
BUNDLE_MAX = 5_000
|
||||
|
||||
SessionLevel = Literal["DEBUG", "TRACE"]
|
||||
SESSION_LEVELS: tuple[str, ...] = get_args(SessionLevel)
|
||||
# Detail order of the diagnostics levels: a session raises the level in force, never lowers it.
|
||||
_DETAIL = {"INFO": 0, "DEBUG": 1, "TRACE": 2}
|
||||
_ID_LEN = 16
|
||||
|
||||
|
||||
class EndReason(StrEnum):
|
||||
STOPPED = "stopped"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
def is_live(session: DebugSession, now: datetime | None = None) -> bool:
|
||||
"""Open and within its cap — THE liveness rule: an open session past its cap is over."""
|
||||
return session.ended_at is None and session.until > (now or datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def open_session(db: Session, user: User) -> DebugSession | None:
|
||||
"""The user's live session, or None. Read-only: one past its cap is over and reads as None;
|
||||
`start` and the hourly sweep are what close it."""
|
||||
session = _unclosed(db, user)
|
||||
if session is None or not is_live(session, datetime.now(timezone.utc)):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def _unclosed(db: Session, user: User) -> DebugSession | None:
|
||||
return db.scalar(
|
||||
select(DebugSession).where(DebugSession.user_id == user.id, DebugSession.ended_at.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def start(db: Session, user: User, level: SessionLevel) -> DebugSession:
|
||||
"""Open a session, or return the live one — a second tab pressing Start joins the recording
|
||||
rather than ending it. One past its cap is closed first: the one-open-per-user index would
|
||||
refuse the new row beside it. Flushes; the caller commits. Two concurrent Starts can still both
|
||||
get here: the index refuses the second, and the caller joins the one that won."""
|
||||
now = datetime.now(timezone.utc)
|
||||
current = _unclosed(db, user)
|
||||
if current is not None:
|
||||
if is_live(current, now):
|
||||
return current
|
||||
end(current, EndReason.EXPIRED)
|
||||
db.flush()
|
||||
session = DebugSession(
|
||||
id=secrets.token_hex(_ID_LEN // 2),
|
||||
user_id=user.id,
|
||||
level=level,
|
||||
started_at=now,
|
||||
until=now + timedelta(minutes=MAX_MINUTES),
|
||||
)
|
||||
db.add(session)
|
||||
db.flush()
|
||||
return session
|
||||
|
||||
|
||||
def stop(session: DebugSession) -> None:
|
||||
"""The user's Stop. Past its cap the session had already ended on its own, and a late Stop must
|
||||
not rename that."""
|
||||
end(session, EndReason.STOPPED if is_live(session) else EndReason.EXPIRED)
|
||||
|
||||
|
||||
def end(session: DebugSession, reason: EndReason) -> None:
|
||||
"""Close a session. An expired one is dated at its cap, not at whenever it was noticed."""
|
||||
if session.ended_at is not None:
|
||||
return
|
||||
session.ended_at = (
|
||||
session.until
|
||||
if reason is EndReason.EXPIRED
|
||||
else min(datetime.now(timezone.utc), session.until)
|
||||
)
|
||||
session.ended_reason = reason
|
||||
|
||||
|
||||
def from_header(db: Session, user: User, value: str | None) -> DebugSession | None:
|
||||
"""The session a request names, if it is the caller's own and still live; else None."""
|
||||
if not value or len(value) != _ID_LEN or not all(c in "0123456789abcdef" for c in value):
|
||||
return None
|
||||
session = db.get(DebugSession, value)
|
||||
if (
|
||||
session is None
|
||||
or session.user_id != user.id
|
||||
or not is_live(session, datetime.now(timezone.utc))
|
||||
):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def level_for(user: User, session: DebugSession | None) -> tuple[str, datetime | None]:
|
||||
"""The diagnostics level (and its expiry) a request is logged at: the user's own, raised to the
|
||||
session's while one is named — whichever records more.
|
||||
|
||||
A named session also caps the expiry at its own: this pair is what the process tracks under the
|
||||
session (app/logs.py), and an expiry outliving the recording would hold the root logger down
|
||||
after it ended."""
|
||||
own = diagnostics.effective_level(user)
|
||||
until = user.diagnostics_until if own != diagnostics.BASE else None
|
||||
if session is None:
|
||||
return own, until
|
||||
if _DETAIL[session.level] <= _DETAIL[own]:
|
||||
return own, min(until, session.until) if until else session.until
|
||||
return session.level, session.until
|
||||
|
||||
|
||||
def lock(db: Session, session: DebugSession) -> None:
|
||||
"""Re-read the session under a row lock, held until the caller commits. Everything that changes
|
||||
what an issue is made of — the draft, the screenshots, the send itself — takes it first: a send
|
||||
in flight sees none of them change under it, and each of them sees a send that committed."""
|
||||
db.refresh(session, with_for_update=True)
|
||||
|
||||
|
||||
def save_draft(session: DebugSession, draft: dict) -> None:
|
||||
"""Keep the composer's draft (the caller validated, locked and commits). Shape: `title`, `body`
|
||||
(the rendered markdown an export or an issue carries), `excluded` (log row ids taken out of the
|
||||
evidence) and `form` (the composer's own fields, to reopen it as it was). A delivery's progress
|
||||
is not the composer's to replace, and is kept."""
|
||||
progress = (session.draft or {}).get("delivery")
|
||||
session.draft = {**draft, "delivery": progress} if progress else draft
|
||||
|
||||
|
||||
# --- a delivery's progress -----------------------------------------------------------------
|
||||
# Written by app/issues/gitea.py into the draft; read HERE, so the send's decision and what the
|
||||
# composer shows are one reading of it.
|
||||
def delivery(session: DebugSession) -> dict:
|
||||
return dict((session.draft or {}).get("delivery") or {})
|
||||
|
||||
|
||||
def not_uploaded(session: DebugSession) -> list[str]:
|
||||
"""The files a send that stopped half-way has still to upload — by name, as the issue carries
|
||||
them. What the composer names to the user."""
|
||||
return list(delivery(session).get("pending") or [])
|
||||
|
||||
|
||||
def images_pending(session: DebugSession) -> bool:
|
||||
"""Whether the issue's body still has to be edited to show the screenshots it already carries
|
||||
(every upload went, the edit that shows them did not)."""
|
||||
state = delivery(session)
|
||||
shown = state.get("images_shown")
|
||||
return not shown and any(n.startswith(SCREENSHOT_PREFIX) for n in state.get("links") or {})
|
||||
|
||||
|
||||
def unfinished(session: DebugSession) -> bool:
|
||||
"""Whether a send owes the issue anything — the retry's question."""
|
||||
return bool(not_uploaded(session)) or images_pending(session)
|
||||
|
||||
|
||||
# A send claims the session while it talks to the tracker, instead of holding the row's lock across
|
||||
# those calls: a Stop, an autosave or a screenshot would queue behind a lock for the whole send.
|
||||
# They see the claim and are refused at once instead. A claim not renewed for this long was left by
|
||||
# a request that died mid-send (the process restarted), and the next send takes it over — which is
|
||||
# why a live send RENEWS it every time it reports progress (`set_delivery`). A claim is a lease, and
|
||||
# a send slower than the lease must not lose it while it is still running.
|
||||
SEND_CLAIM_SECONDS = 600
|
||||
|
||||
|
||||
def _claim(session: DebugSession) -> dict:
|
||||
claim = delivery(session).get("sending")
|
||||
return claim if isinstance(claim, dict) else {}
|
||||
|
||||
|
||||
def _stamp(token: str) -> dict:
|
||||
return {"at": datetime.now(timezone.utc).isoformat(), "by": token}
|
||||
|
||||
|
||||
def sending(session: DebugSession) -> bool:
|
||||
"""Whether a send is under way right now (a live claim)."""
|
||||
at = _claim(session).get("at")
|
||||
if not at:
|
||||
return False
|
||||
return (datetime.now(timezone.utc) - datetime.fromisoformat(at)).total_seconds() < (
|
||||
SEND_CLAIM_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def claim_send(session: DebugSession) -> str:
|
||||
"""Mark the session as being sent, and return the token that OWNS this claim. The caller holds
|
||||
the lock and commits, which is what makes the claim visible to everyone else."""
|
||||
token = uuid4().hex
|
||||
_write_delivery(session, {**delivery(session), "sending": _stamp(token)})
|
||||
return token
|
||||
|
||||
|
||||
def release_send(session: DebugSession, token: str) -> None:
|
||||
"""Give back our OWN claim, and only ours: one that aged out and was taken over belongs to the
|
||||
send holding it now, and handing that one back would let a third send in beside it."""
|
||||
if _claim(session).get("by") != token:
|
||||
return
|
||||
_write_delivery(session, {k: v for k, v in delivery(session).items() if k != "sending"})
|
||||
|
||||
|
||||
class SendClaimLost(Exception):
|
||||
"""This send's claim is no longer on the row: it aged out and another send took it over. The
|
||||
send must stop — what it uploads from here it can no longer record, so a retry would send it
|
||||
again — and it must not write, because the row now holds the other send's progress."""
|
||||
|
||||
|
||||
def claim_owner(db: Session, session: DebugSession) -> str | None:
|
||||
"""Who holds the row's claim RIGHT NOW, read from the ROW and not from the object.
|
||||
|
||||
`expire_on_commit` is off (app/db.py), so a checkpoint does not re-read: a send that stalled
|
||||
past its lease still carries its own dead token in memory long after another send took over.
|
||||
A targeted scalar, not `lock`/`db.refresh` — a refresh here would discard the `issue_url` that
|
||||
`gitea.deliver` assigns one line before its first progress report, and that link is the one
|
||||
thing the send must not lose."""
|
||||
return db.scalar(
|
||||
select(DebugSession.draft["delivery"]["sending"]["by"].as_string()).where(
|
||||
DebugSession.id == session.id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def set_delivery(db: Session, session: DebugSession, state: dict, token: str) -> None:
|
||||
"""Write a delivery's progress (app/issues/gitea.py keeps it; this module reads it), and renew
|
||||
the claim while doing it — a send still reporting progress is still alive, so one slower than
|
||||
the lease cannot expire while it is still running.
|
||||
|
||||
The claim is not the caller's to carry: `gitea.deliver` threads ONE snapshot of this dict
|
||||
through a whole send, so a progress report that carried the claim wrote a stale one back at
|
||||
every step. It is stamped from `token` here instead, and only after the ROW confirms the claim
|
||||
is still ours. Without that check the write is a full-column overwrite with no WHERE: a send
|
||||
that lost its lease came back and stomped the take-over send's committed links, then put its
|
||||
own dead token back on an idle session (deep-review round 4). `release_send` has always made
|
||||
the same check; this is the other half of it."""
|
||||
if claim_owner(db, session) != token:
|
||||
raise SendClaimLost(session.id)
|
||||
state = {k: v for k, v in state.items() if k != "sending"}
|
||||
state["sending"] = _stamp(token)
|
||||
_write_delivery(session, state)
|
||||
|
||||
|
||||
def _write_delivery(session: DebugSession, state: dict) -> None:
|
||||
# A new dict, so the JSONB column sees the change.
|
||||
session.draft = {**(session.draft or {}), "delivery": state}
|
||||
|
||||
|
||||
def serialize(session: DebugSession | None, screenshots: int = 0) -> dict | None:
|
||||
if session is None:
|
||||
return None
|
||||
return {
|
||||
"id": session.id,
|
||||
"level": session.level,
|
||||
"started_at": session.started_at.isoformat(),
|
||||
"until": session.until.isoformat(),
|
||||
"ended_at": session.ended_at.isoformat() if session.ended_at else None,
|
||||
"ended_reason": session.ended_reason,
|
||||
"issue_url": session.issue_url,
|
||||
# A send that stopped half-way: the files it has still to upload, and whether the issue's
|
||||
# body has still to be edited to show the screenshots it already carries. Both are true of a
|
||||
# send that is RUNNING too — every step commits its progress — so `sending` goes with them:
|
||||
# without it a reload or a second tab renders a live send as a failed one, and offers a
|
||||
# retry that is certain to be refused.
|
||||
"pending_files": not_uploaded(session),
|
||||
"images_pending": images_pending(session),
|
||||
"sending": sending(session),
|
||||
"screenshots": screenshots,
|
||||
"draft": session.draft,
|
||||
}
|
||||
|
||||
|
||||
def view(db: Session, session: DebugSession) -> dict:
|
||||
"""The session as the API serves it — one shape wherever it is served."""
|
||||
return serialize(session, screenshot_count(db, session)) or {}
|
||||
|
||||
|
||||
def screenshot_count(db: Session, session: DebugSession) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count())
|
||||
.select_from(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
# By content, not by the declared type: what is stored is served back to a browser as an image.
|
||||
_SIGNATURES = (
|
||||
(b"\x89PNG\r\n\x1a\n", "image/png"),
|
||||
(b"\xff\xd8\xff", "image/jpeg"),
|
||||
)
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str | None:
|
||||
"""The image type `data` really is — one of IMAGE_TYPES — or None."""
|
||||
for magic, mime in _SIGNATURES:
|
||||
if data.startswith(magic):
|
||||
return mime
|
||||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
|
||||
def lock_user(db: Session, user_id: int) -> None:
|
||||
"""Hold this user's own lock for the rest of the transaction. A session's row lock bounds what
|
||||
THAT session may keep; the byte cap below is the user's across all of them, and a lock on one
|
||||
session cannot serialize a check that reads them all — two uploads to two sessions both read the
|
||||
total before either commits, and both pass."""
|
||||
db.execute(select(func.pg_advisory_xact_lock(user_id)))
|
||||
|
||||
|
||||
def stored_screenshot_bytes(db: Session, user_id: int) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.sum(DebugScreenshot.size), 0))
|
||||
.join(DebugSession, DebugSession.id == DebugScreenshot.session_id)
|
||||
.where(DebugSession.user_id == user_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
class ScreenshotRefused(Exception):
|
||||
"""A screenshot that cannot be kept; `code` is the refusal's machine word."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def add_screenshot(db: Session, session: DebugSession, data: bytes) -> DebugScreenshot:
|
||||
"""Keep one screenshot. Locks the user and the session, and flushes; the caller commits.
|
||||
Refused while a send is under way or once the issue was sent (it can no longer carry it), past
|
||||
the per-session count or the user's storage, or when it is not an image we can show."""
|
||||
if not data:
|
||||
raise ScreenshotRefused("empty")
|
||||
mime = sniff(data)
|
||||
if mime is None:
|
||||
raise ScreenshotRefused("not_an_image")
|
||||
# Under both locks: two uploads at once cannot both pass the count (the session's), nor both
|
||||
# pass the user's byte cap from two different sessions (the user's). Always in this order.
|
||||
lock_user(db, session.user_id)
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if session.issue_url is not None:
|
||||
raise ScreenshotRefused("already_sent")
|
||||
if screenshot_count(db, session) >= SCREENSHOTS_MAX:
|
||||
raise ScreenshotRefused("too_many")
|
||||
if stored_screenshot_bytes(db, session.user_id) + len(data) > SCREENSHOT_USER_MAX_BYTES:
|
||||
raise ScreenshotRefused("storage_full")
|
||||
shot = DebugScreenshot(
|
||||
session_id=session.id,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
mime=mime,
|
||||
size=len(data),
|
||||
data=data,
|
||||
)
|
||||
db.add(shot)
|
||||
db.flush()
|
||||
return shot
|
||||
|
||||
|
||||
def remove_screenshot(db: Session, session: DebugSession, shot: DebugScreenshot) -> None:
|
||||
"""Delete one screenshot, under the lock. Refused while a send is under way or unfinished: a
|
||||
delivery names the screenshots by their order, and removing one would renumber what it still
|
||||
has to send. Once the issue has everything, this only drops OUR copy — the issue keeps its
|
||||
own. The caller commits."""
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if unfinished(session):
|
||||
raise ScreenshotRefused("already_sent")
|
||||
db.delete(shot)
|
||||
db.flush()
|
||||
|
||||
|
||||
def screenshots(db: Session, session: DebugSession) -> list[DebugScreenshot]:
|
||||
"""The session's screenshots, oldest first, WITHOUT their bytes loaded."""
|
||||
return list(
|
||||
db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.options(defer(DebugScreenshot.data))
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- the evidence --------------------------------------------------------------------------
|
||||
# The browser's own log upload (`POST` here, every few seconds while recording) is how the evidence
|
||||
# arrives, not part of it: its server lines would fill the bundle with the recording's plumbing. The
|
||||
# browser leaves its copies out for the same reason (frontend lib/diagnostics `isOwnRequest`).
|
||||
INGEST_PATH = "/api/diagnostics/events"
|
||||
|
||||
|
||||
def _evidence(session: DebugSession):
|
||||
upload_line = and_(
|
||||
LogEvent.event == "http.request", LogEvent.attrs["path"].astext == INGEST_PATH
|
||||
)
|
||||
# `IS TRUE`, not the bare condition: for a row with no event or no attrs it is NULL, and
|
||||
# `NOT NULL` would drop exactly the rows that are evidence.
|
||||
return (LogEvent.debug_session_id == session.id, not_(upload_line.is_(True)))
|
||||
|
||||
|
||||
def bundle(
|
||||
db: Session, session: DebugSession, *, drop_excluded: bool = False
|
||||
) -> tuple[list[LogEvent], int]:
|
||||
"""The session's log rows, browser and server interleaved in time, capped at BUNDLE_MAX — and
|
||||
how many rows it has in all, so a cut bundle can say so. `drop_excluded` leaves out the rows
|
||||
the user took out in the composer: what an export or an issue carries."""
|
||||
where = list(_evidence(session))
|
||||
excluded = (session.draft or {}).get("excluded") or [] if drop_excluded else []
|
||||
if excluded:
|
||||
where.append(LogEvent.id.not_in(excluded))
|
||||
rows = list(
|
||||
db.scalars(
|
||||
select(LogEvent).where(*where).order_by(LogEvent.ts, LogEvent.id).limit(BUNDLE_MAX)
|
||||
)
|
||||
)
|
||||
total = len(rows)
|
||||
if total == BUNDLE_MAX:
|
||||
total = db.scalar(select(func.count()).select_from(LogEvent).where(*where)) or total
|
||||
return rows, total
|
||||
|
||||
|
||||
# --- housekeeping (the hourly log job) -----------------------------------------------------
|
||||
def sweep(db: Session) -> int:
|
||||
"""Stage closing every open session past its cap; returns how many."""
|
||||
now = datetime.now(timezone.utc)
|
||||
result = db.execute(
|
||||
update(DebugSession)
|
||||
.where(DebugSession.ended_at.is_(None), DebugSession.until <= now)
|
||||
.values(ended_at=DebugSession.until, ended_reason=EndReason.EXPIRED.value)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage deleting sessions (and by cascade their screenshots) started before the log retention
|
||||
window — the rows they would bundle are gone by then too. Returns how many."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(DebugSession).where(DebugSession.started_at < cutoff))
|
||||
return result.rowcount or 0
|
||||
@@ -0,0 +1,65 @@
|
||||
"""A user's diagnostics level (E22 S5): what they consented to, and for how long.
|
||||
|
||||
Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
* INFO — the base, for everyone, no consent. The browser keeps its log in memory; nothing extra
|
||||
is sent or stored.
|
||||
* DEBUG — consented. The server logs this user's requests in detail and stores every line of
|
||||
them, and the browser streams its own log to `POST /api/diagnostics/events`.
|
||||
* TRACE — consented. As DEBUG, plus what the user typed into a search.
|
||||
|
||||
A raised level always lapses after `ELEVATION_HOURS`: the expiry is computed HERE at consent time,
|
||||
never taken from the client, and an expired level reads as INFO without anything having to clear it.
|
||||
The shared demo account cannot raise its level — its log would mix every visitor's session.
|
||||
|
||||
Core layer: imports `models` and `sysconfig`.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal, get_args
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import User
|
||||
|
||||
DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
# How long a debug session records at most (E16, app/debugsession.py). Here, beside the other
|
||||
# consent window, because the start dialog's consent text states it and reads it from `state`.
|
||||
SESSION_MINUTES = 60
|
||||
|
||||
|
||||
def effective_level(user: User, now: datetime | None = None) -> DiagnosticsLevel:
|
||||
"""The level in force for `user` right now: the consented one until it expires, else BASE."""
|
||||
level, until = user.diagnostics_level, user.diagnostics_until
|
||||
if level not in LEVELS or level == BASE or until is None:
|
||||
return BASE
|
||||
if until <= (now or datetime.now(timezone.utc)):
|
||||
return BASE
|
||||
return level # type: ignore[return-value] — narrowed by the LEVELS check above
|
||||
|
||||
|
||||
def state(db: Session, user: User) -> dict:
|
||||
"""The `/api/me` view of it: the level in force, when a raised one lapses, and how long stored
|
||||
log lines are kept — the consent text states that number, so it must be the instance's own."""
|
||||
level = effective_level(user)
|
||||
return {
|
||||
"level": level,
|
||||
"until": user.diagnostics_until.isoformat() if level != BASE else None,
|
||||
"retention_days": sysconfig.get_int(db, "log_retention_days"),
|
||||
"session_minutes": SESSION_MINUTES,
|
||||
}
|
||||
|
||||
|
||||
def set_level(user: User, level: DiagnosticsLevel) -> None:
|
||||
"""Record a consent (or its withdrawal). The caller commits. Raising the level again restarts
|
||||
the clock — each consent is for the next ELEVATION_HOURS, not an extension of an older one."""
|
||||
if level == BASE:
|
||||
user.diagnostics_level = None
|
||||
user.diagnostics_until = None
|
||||
return
|
||||
user.diagnostics_level = level
|
||||
user.diagnostics_until = datetime.now(timezone.utc) + timedelta(hours=ELEVATION_HOURS)
|
||||
@@ -81,7 +81,11 @@ def pick(
|
||||
return None
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
for card in pins.playables(pins.shelf(db, user) if shelf is None else shelf):
|
||||
if shelf is None:
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
else:
|
||||
own = shelf
|
||||
for card in pins.playables(own):
|
||||
if cards.key(card) not in skip and fits(card["remaining_seconds"], budget):
|
||||
return card
|
||||
return None
|
||||
|
||||
+109
-15
@@ -8,14 +8,16 @@ second one.
|
||||
|
||||
Every shelf card carries a resolved `next`: the thing that actually plays when you click the tile —
|
||||
a show's in-progress-or-next episode, a channel's newest unwatched video, a playlist's first
|
||||
unwatched item. It costs a query per non-leaf pin, which is affordable precisely because this shelf is
|
||||
short and hand-curated, and it buys the property the whole surface exists for: one click, no
|
||||
intermediate page, no second decision.
|
||||
unwatched item. That buys the property the whole surface exists for: one click, no intermediate page,
|
||||
no second decision. Channels and playlists are resolved for the whole shelf at once (`_Batch`), so the
|
||||
cost does not grow a query or two per pin.
|
||||
|
||||
Domain module, so nothing here raises HTTP. `resolve_ref` returns None for "no such thing, or not
|
||||
yours" and the route turns that into its 404 — core never imports routes.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -62,6 +64,8 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
return (col, sh.id) if sh is not None else None
|
||||
try:
|
||||
pid = int(ref)
|
||||
# silent-ok: validating a reference the client supplied. "Not an integer" is answered as "no
|
||||
# such pin target", which is the same answer this function gives for one that does not exist.
|
||||
except ValueError:
|
||||
return None
|
||||
pl = db.get(Playlist, pid)
|
||||
@@ -71,6 +75,44 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
return (col, pl.id) if pl is not None and pl.user_id == user.id else None
|
||||
|
||||
|
||||
def reorder(db: Session, user: User, ids: list[int]) -> None:
|
||||
"""Put the listed pins in the listed order, WITHIN the slots they already hold.
|
||||
|
||||
The client sends the order it can SEE, which is not always the whole shelf: with Plex off the Home
|
||||
rail hides the Plex pins, and a stale tab may not know a pin added elsewhere. Numbering only the
|
||||
listed ids 0..n-1 collided with every pin left out — two tiles on one position, and the hidden
|
||||
ones jumped about when they came back. So the pins left out keep their slots, and the listed ones
|
||||
are dealt into the slots the listed set occupies. Ids that are not this user's, or repeated, are
|
||||
ignored rather than failing a stale replay wholesale. Flushes; the route commits."""
|
||||
# Locked: the rows it leaves alone are part of the answer too. Two overlapping reorders each
|
||||
# skipped the rows that already matched in their own snapshot and interleaved into two pins on
|
||||
# one slot; the second now waits and deals from the first one's result. Locked in id order and
|
||||
# sorted HERE: with `ORDER BY position … FOR UPDATE` Postgres sorts before it locks, so the one
|
||||
# that waited got the fresh rows in its stale order and dealt the pins it left out back to their
|
||||
# old slots — and one global lock order means two reorders cannot deadlock.
|
||||
locked = db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.order_by(ComfortPin.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalars()
|
||||
current = sorted(locked, key=lambda p: (p.position, p.id))
|
||||
owned = {p.id for p in current}
|
||||
listed: list[int] = []
|
||||
for i in ids:
|
||||
if i in owned and i not in listed:
|
||||
listed.append(i)
|
||||
chosen = set(listed)
|
||||
queue = iter(listed)
|
||||
order = [next(queue) if p.id in chosen else p.id for p in current]
|
||||
by_id = {p.id: p for p in current}
|
||||
for position, pin_id in enumerate(order):
|
||||
if by_id[pin_id].position != position:
|
||||
by_id[pin_id].position = position
|
||||
db.flush()
|
||||
|
||||
|
||||
def list_pins(db: Session, user: User) -> list[ComfortPin]:
|
||||
return list(
|
||||
db.execute(
|
||||
@@ -101,7 +143,34 @@ def next_position(db: Session, user: User) -> int:
|
||||
)
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
@dataclass
|
||||
class _Batch:
|
||||
"""What the shelf resolves for every pinned channel and playlist in one go."""
|
||||
|
||||
channels: dict[str, Channel] = field(default_factory=dict)
|
||||
channel_summary: dict[str, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
playlists: dict[int, Playlist] = field(default_factory=dict)
|
||||
playlist_summary: dict[int, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def load(cls, db: Session, user: User, pins: list[ComfortPin]) -> "_Batch":
|
||||
ch_ids = [p.channel_id for p in pins if p.channel_id is not None]
|
||||
pl_ids = [p.playlist_id for p in pins if p.playlist_id is not None]
|
||||
b = cls()
|
||||
if ch_ids:
|
||||
b.channels = {
|
||||
c.id: c for c in db.scalars(select(Channel).where(Channel.id.in_(ch_ids)))
|
||||
}
|
||||
b.channel_summary = queries.channels_summary(db, user, ch_ids)
|
||||
if pl_ids:
|
||||
b.playlists = {
|
||||
p.id: p for p in db.scalars(select(Playlist).where(Playlist.id.in_(pl_ids)))
|
||||
}
|
||||
b.playlist_summary = queries.playlists_summary(db, user, pl_ids)
|
||||
return b
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
@@ -115,11 +184,11 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
nxt = dict(card) # a video plays itself
|
||||
|
||||
elif pin.channel_id is not None:
|
||||
ch = db.get(Channel, pin.channel_id)
|
||||
ch = batch.channels.get(pin.channel_id)
|
||||
if ch is None:
|
||||
return None
|
||||
card = cards.from_channel(ch, queries.channel_unwatched_count(db, user, ch.id))
|
||||
row = queries.channel_next_video(db, user, ch.id)
|
||||
count, row = batch.channel_summary.get(ch.id, (0, None))
|
||||
card = cards.from_channel(ch, count)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
elif pin.plex_item_id is not None:
|
||||
@@ -140,10 +209,10 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
pl = db.get(Playlist, pin.playlist_id)
|
||||
pl = batch.playlists.get(pin.playlist_id)
|
||||
if pl is None:
|
||||
return None
|
||||
count, row = queries.playlist_summary(db, user, pl.id)
|
||||
count, row = batch.playlist_summary.get(pl.id, (0, None))
|
||||
card = cards.from_playlist(pl, count, row.get("thumbnail_url") if row else None)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
@@ -157,11 +226,24 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
return card
|
||||
|
||||
|
||||
def shelf(db: Session, user: User) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order."""
|
||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||
|
||||
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order.
|
||||
|
||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||
2026-09-19): the rails beside it go YouTube-only, and a Plex tile there would offer a Play that
|
||||
lands on a gated module. The pins themselves are kept — the pin menus still read the full shelf
|
||||
(`GET /pins`), and switching Plex back on brings the tiles back in their old slots
|
||||
(`reorder` keeps a hidden pin's place)."""
|
||||
out = []
|
||||
for pin in list_pins(db, user):
|
||||
card = _shelf_card(db, user, pin)
|
||||
pins = list_pins(db, user)
|
||||
if not include_plex:
|
||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||
batch = _Batch.load(db, user, pins)
|
||||
for pin in pins:
|
||||
card = _shelf_card(db, user, pin, batch)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
@@ -169,5 +251,17 @@ def shelf(db: Session, user: User) -> list[dict]:
|
||||
|
||||
def playables(shelf_cards: list[dict]) -> list[dict]:
|
||||
"""The `next` of every shelf entry that has one — what the hero may draw from. A pinned channel
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now."""
|
||||
return [c["next"] for c in shelf_cards if c.get("next")]
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now.
|
||||
|
||||
Each carries `pinned_from`: the pin it was resolved from. A show's next EPISODE is not itself
|
||||
pinned, so without it the hero's menu looked the episode up, missed, and offered "Pin" under a
|
||||
kicker that says "From your shelf" — and taking it inserted a second pin. With it, the hero's
|
||||
pin row acts on the shelf entry the card came from (the owner's call, 2026-09-19: "Unpin")."""
|
||||
return [
|
||||
{
|
||||
**c["next"],
|
||||
"pinned_from": {k: c[k] for k in ("source", "kind", "id", "title")},
|
||||
}
|
||||
for c in shelf_cards
|
||||
if c.get("next")
|
||||
]
|
||||
|
||||
@@ -43,8 +43,9 @@ from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry. Matches `routes/plex/discover.py`'s RAIL_LIMIT on purpose —
|
||||
# the two surfaces should feel like the same app, and a rail is a rail.
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
# (`routes/plex/discover.py` imports it): the two surfaces should feel like the same app, and a rail
|
||||
# is a rail. One definition, because a comment was all that held two copies in step.
|
||||
RAIL_LIMIT = 20
|
||||
|
||||
# The feed parameters a Home rail runs with. Spelled out rather than borrowed from the route's
|
||||
@@ -129,18 +130,6 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) ->
|
||||
return [r for r in out if not r["position_seconds"]][:limit]
|
||||
|
||||
|
||||
def channel_next_video(db: Session, user: User, channel_id: str) -> dict | None:
|
||||
"""The newest unwatched video of a pinned channel — what "put on that channel" resolves to."""
|
||||
# `library_source="all"`: a pinned channel means the whole channel, including videos that only
|
||||
# ever arrived through a search. The feed's "organic" default is a feed-scoping choice, not a
|
||||
# statement about what belongs to a channel.
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("newest", 0, rank_expr)
|
||||
row = db.execute(query.order_by(*order_by(keys)).limit(1)).first()
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
"""One video as a feed row, with this user's watch state.
|
||||
|
||||
@@ -158,31 +147,55 @@ def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def channel_unwatched_count(db: Session, user: User, channel_id: str) -> int:
|
||||
"""How many unwatched videos a pinned channel is holding — the tile's only honest promise, since a
|
||||
channel has no runtime of its own."""
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, _ = filtered_query(db, user, **params)
|
||||
return int(db.execute(select(func.count()).select_from(query.subquery())).scalar() or 0)
|
||||
def channels_summary(
|
||||
db: Session, user: User, channel_ids: list[str]
|
||||
) -> dict[str, tuple[int, dict | None]]:
|
||||
"""For each pinned channel: (unwatched count, its newest unwatched video) — the tile's only honest
|
||||
promise, since a channel has no runtime of its own, and what "put on that channel" resolves to.
|
||||
|
||||
ONE query for the whole shelf, not two per channel: the count and the first row per channel are
|
||||
window functions over the same filtered set. `library_source="all"`: a pinned channel means the
|
||||
whole channel, including videos that only ever arrived through a search — the feed's "organic"
|
||||
default is a feed-scoping choice, not a statement about what belongs to a channel."""
|
||||
if not channel_ids:
|
||||
return {}
|
||||
params = dict(_FEED_BASE, channel_ids=list(channel_ids), scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
newest = order_by(sort_keys("newest", 0, rank_expr))
|
||||
per_channel = {"partition_by": Video.channel_id}
|
||||
sub = query.add_columns(
|
||||
func.row_number().over(**per_channel, order_by=newest).label("_rn"),
|
||||
func.count().over(**per_channel).label("_total"),
|
||||
).subquery()
|
||||
rows = db.execute(select(sub).where(sub.c._rn == 1)).all()
|
||||
return {r.channel_id: (int(r._total), serialize(r)) for r in rows}
|
||||
|
||||
|
||||
def playlist_summary(db: Session, user: User, playlist_id: int) -> tuple[int, dict | None]:
|
||||
"""(item count, the row to play next) for a pinned playlist — its first not-yet-watched item, or
|
||||
its first item once the whole list is watched. The same row doubles as the tile's artwork, so a
|
||||
shelf of playlists is scannable instead of a row of blank boxes."""
|
||||
def playlists_summary(
|
||||
db: Session, user: User, playlist_ids: list[int]
|
||||
) -> dict[int, tuple[int, dict | None]]:
|
||||
"""For each pinned playlist: (item count, the row to play next) — its first not-yet-watched item,
|
||||
or its first item once the whole list is watched. The same row doubles as the tile's artwork, so
|
||||
a shelf of playlists is scannable instead of a row of blank boxes. One query for the whole shelf."""
|
||||
if not playlist_ids:
|
||||
return {}
|
||||
st = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, st), PlaylistItem.position)
|
||||
select(*feed_columns(user, st), PlaylistItem.playlist_id.label("_playlist"))
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
.order_by(PlaylistItem.position.asc())
|
||||
.where(PlaylistItem.playlist_id.in_(playlist_ids))
|
||||
.order_by(PlaylistItem.playlist_id, PlaylistItem.position.asc())
|
||||
).all()
|
||||
if not rows:
|
||||
return 0, None
|
||||
nxt = next((r for r in rows if (r.status or "new") not in ("watched", "hidden")), rows[0])
|
||||
return len(rows), serialize(nxt)
|
||||
grouped: dict[int, list] = {}
|
||||
for r in rows:
|
||||
grouped.setdefault(r._playlist, []).append(r)
|
||||
out: dict[int, tuple[int, dict | None]] = {}
|
||||
for pid, items in grouped.items():
|
||||
nxt = next((r for r in items if (r.status or "new") not in ("watched", "hidden")), items[0])
|
||||
out[pid] = (len(items), serialize(nxt))
|
||||
return out
|
||||
|
||||
|
||||
# ── Plex ──────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -11,9 +11,22 @@ Resolution works both before extraction (from source_kind / URL host) and after
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||||
FACEBOOK_PARSE_WALL = "cannot parse data"
|
||||
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
|
||||
# them from here because this module must not import errors (errors → credentials → adapters).
|
||||
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
|
||||
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
@@ -22,9 +35,21 @@ class SourceAdapter:
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
|
||||
download's own options (cookie, timeouts, address family) for any page this needs to read.
|
||||
|
||||
Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
|
||||
download cannot go ahead: the content is there but holds no single video
|
||||
(`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
|
||||
refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
|
||||
is raised as itself, so it is classified like any other download error. Generic: `url`."""
|
||||
return url
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
|
||||
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
|
||||
`authed` is whether the acting user's cookie file goes with the download."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
@@ -37,6 +62,10 @@ class SourceAdapter:
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def title(self, info: dict) -> str | None:
|
||||
"""The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
|
||||
return info.get("title")
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
@@ -65,12 +94,46 @@ class YouTubeAdapter(SourceAdapter):
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
# Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
|
||||
# refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
|
||||
# between the player response and the stream, so this is the same per-client flakiness one
|
||||
# step later — and it used to end the job on attempt 1, because the text classifies as
|
||||
# `network` and nothing here matched it. The format URLs belong to the client that minted
|
||||
# them, so the next client set is a real second chance.
|
||||
"unable to download video data",
|
||||
)
|
||||
# Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
|
||||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
||||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||||
# that moves the table turns the gate red instead of silently un-authing every YouTube download.
|
||||
_COOKIE_CAPABLE = frozenset(
|
||||
{
|
||||
"mweb",
|
||||
"tv",
|
||||
"tv_downgraded",
|
||||
"web",
|
||||
"web_creator",
|
||||
"web_embedded",
|
||||
"web_music",
|
||||
"web_safari",
|
||||
}
|
||||
)
|
||||
# Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
|
||||
# Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
|
||||
# the per-client flakiness that the two-tier ladder gives a signed-out download.
|
||||
_AUTHED_DEFAULT = ("web_safari", "web")
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
|
||||
if authed:
|
||||
kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
|
||||
return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
|
||||
return sets or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
@@ -93,6 +156,63 @@ class YouTubeAdapter(SourceAdapter):
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||||
# a cookie the same error is the login wall, which no disguise gets past.
|
||||
_IMPERSONATE = "impersonate"
|
||||
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
|
||||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||||
post_id = facebook_group_post_id(url)
|
||||
if post_id is None or not ydl_opts.get("cookiefile"):
|
||||
return url
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||||
page = raw.decode("utf-8", "replace")
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||||
video_id = facebook_post_video_id(page, post_id)
|
||||
if video_id is None:
|
||||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||||
raise StatedFailure(POST_WITHOUT_VIDEO_MESSAGE)
|
||||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
return [None, self._IMPERSONATE] if authed else [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
if clients == self._IMPERSONATE:
|
||||
return {"impersonate": ImpersonateTarget.from_str("chrome")}
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return FACEBOOK_PARSE_WALL in error_text.lower()
|
||||
|
||||
# Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
|
||||
# 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
|
||||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||||
|
||||
def title(self, info: dict) -> str | None:
|
||||
raw = info.get("title")
|
||||
if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
|
||||
return raw
|
||||
line = next(
|
||||
(ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
|
||||
)
|
||||
# The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
|
||||
# "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
|
||||
# stored title is unbounded; display_name and the file name clip it themselves.
|
||||
return line or raw # no description either (the 18+ reel): nothing better on the page
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
@@ -107,6 +227,39 @@ class FacebookAdapter(SourceAdapter):
|
||||
return True
|
||||
|
||||
|
||||
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
|
||||
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
|
||||
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
|
||||
|
||||
|
||||
def _quoted(post_id: str) -> str:
|
||||
# How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
|
||||
return f'"{post_id}"'
|
||||
|
||||
|
||||
def facebook_group_post_id(url: str) -> str | None:
|
||||
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
|
||||
parsed = urlparse(url)
|
||||
if not _host_is(parsed.netloc.lower(), "facebook.com"):
|
||||
return None
|
||||
match = _FB_GROUP_POST_PATH.match(parsed.path)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def facebook_post_video_id(page: str, post_id: str) -> str | None:
|
||||
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
|
||||
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
|
||||
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
|
||||
quoted = _quoted(post_id)
|
||||
ids = {
|
||||
vid
|
||||
for block in _SCRIPT_BLOCK.findall(page)
|
||||
if quoted in block
|
||||
for vid in _VIDEO_NODE.findall(block)
|
||||
}
|
||||
return ids.pop() if len(ids) == 1 else None
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
@@ -11,11 +11,12 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
@@ -23,12 +24,17 @@ from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
with formats.hiding(url):
|
||||
return _extract_meta_scoped(url, adapter, asset_id)
|
||||
|
||||
|
||||
def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
@@ -40,7 +46,19 @@ def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
if info:
|
||||
return info
|
||||
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
||||
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
|
||||
# The asset id, not the URL. A source URL is a row of this user's download
|
||||
# history, and `redact_text` only takes credential params out of it — the rest
|
||||
# of it goes in whole. The id names the same asset to anyone holding the
|
||||
# database and nothing at all to anyone who is not; the host is kept because it
|
||||
# is what decides which client set was tried. yt-dlp's message goes through
|
||||
# `log_safe` for the same reason: it opens with the URL or the video id.
|
||||
log.info(
|
||||
"meta extract failed for asset %s on %s (%s): %s",
|
||||
asset_id,
|
||||
urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query
|
||||
clients,
|
||||
formats.log_safe(exc)[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -60,12 +78,14 @@ def run() -> dict:
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url, adapter)
|
||||
info = _extract_meta(url, adapter, a.id)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
log.info("asset %s uploader_url <- %s", a.id, uu)
|
||||
log.info(
|
||||
"asset %s uploader_url filled", a.id
|
||||
) # the URL names what they downloaded
|
||||
if a.poster_path is None:
|
||||
# Record a self-hosted poster for every download (the reliable og:image source):
|
||||
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
||||
@@ -73,7 +93,7 @@ def run() -> dict:
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
log.info("asset %s poster <- %s", a.id, rel)
|
||||
log.info("asset %s poster filled", a.id) # the path carries the title
|
||||
db.commit()
|
||||
result = {
|
||||
"assets": len(assets),
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||
cookie apply to this download, and how does that keep the result private".
|
||||
|
||||
Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
Two sides depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
in prose):
|
||||
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user.
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user. Retry
|
||||
(`service.resume`) and Re-download (`service.redownload`) ask the same question again, so a
|
||||
cookie the user added after a failure goes with the next attempt.
|
||||
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||
asset (the load-bearing cross-user privacy fix).
|
||||
|
||||
Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift.
|
||||
Both key the private identity on `cookie_salt` / `host_key_for_url` here, so they can't drift.
|
||||
|
||||
Security controls enforced here:
|
||||
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
||||
@@ -21,19 +23,19 @@ Security controls enforced here:
|
||||
logged, or written to a job's error/snapshot.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.credentials")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class HostSpec:
|
||||
@@ -53,6 +55,11 @@ class HostSpec:
|
||||
# all derive from this, so there is nothing else to wire up.
|
||||
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
||||
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273: every player client fails on that one video while its neighbours download fine, and
|
||||
# the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
|
||||
# row here the `login_required` copy sent the user to a dropdown that could not help them.
|
||||
HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
|
||||
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
||||
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
||||
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
||||
@@ -99,11 +106,38 @@ def cookie_salt(user_id: int, host_key: str) -> str:
|
||||
|
||||
|
||||
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to — the private asset
|
||||
identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE."""
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to under the current
|
||||
signature version — the private asset enqueue creates, and one of the identities `is_authed_asset`
|
||||
recognises. Neither calls it; the tests use it to hold the two to each other."""
|
||||
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||
|
||||
|
||||
def rebuild_salt(asset_id: int, cookie: str | None) -> str:
|
||||
"""The salt of a Re-download's fresh asset: its OWN row id (so every rebuild is a new identity no
|
||||
enqueue can ever compute), prefixed with the user's `cookie_salt` when a cookie applies. The id is
|
||||
what makes it recognisable — the worker has the asset row, so `is_authed_asset` can recompute this
|
||||
one, where a salt built from the job's previous asset id could not be."""
|
||||
return f"{cookie}:rebuild:{asset_id}" if cookie else f"rebuild:{asset_id}"
|
||||
|
||||
|
||||
def is_authed_asset(asset: MediaAsset, spec: dict, user_id: int, url: str | None) -> bool:
|
||||
"""Whether an asset is `user_id`'s private, cookie-salted identity for `url`'s host — the one
|
||||
kind of asset a cookie is ever attached to: the one enqueue creates (`authed_format_sig`), or a
|
||||
Re-download's rebuild of it (`rebuild_salt`). The worker's cookie gate asks it before a download,
|
||||
and the failure diagnosis asks it after one (a failure on this asset happened WITH the cookie).
|
||||
Either signature version counts (`formats.format_sigs`): a job queued before a deploy that bumped
|
||||
the version still downloads into the asset it was given."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return False
|
||||
salt = cookie_salt(user_id, host_key)
|
||||
if asset.format_sig in formats.format_sigs(spec, salt=salt):
|
||||
return True
|
||||
return asset.id is not None and asset.format_sig in formats.format_sigs(
|
||||
spec, salt=rebuild_salt(asset.id, salt)
|
||||
)
|
||||
|
||||
|
||||
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
||||
normal shared download). Returns the salt string so the caller passes it straight to
|
||||
@@ -199,20 +233,19 @@ def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
# --- worker: resolve + write the cookie file --------------------------------------------------
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset_format_sig: str,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download, or None.
|
||||
def usable_cookie(
|
||||
db: Session, user_id: int, url: str | None, asset: MediaAsset, spec: dict
|
||||
) -> str | None:
|
||||
"""The user's cookie text a download of `url` into `asset` goes with, or None — the ONE decision
|
||||
whether a download is signed in. The worker child writes it out for yt-dlp (`resolve_cookiefile`);
|
||||
the worker asks it again for a job that inherits a failed asset. That second ask sees the
|
||||
credential as it is when the job is claimed: a cookie the user removed or added since the failed
|
||||
download changes this job's explanation to fit the cookie its Retry would go with.
|
||||
|
||||
The double gate that prevents cross-user leakage:
|
||||
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||
2. the asset being downloaded into must BE this user's private, salted identity
|
||||
(`asset_format_sig == authed_format_sig(...)`).
|
||||
(`is_authed_asset`).
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
@@ -224,18 +257,31 @@ def resolve_cookiefile(
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if asset_format_sig != authed_format_sig(spec, user_id, host_key):
|
||||
if not is_authed_asset(asset, spec, user_id, url):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
log.warning(
|
||||
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
||||
"proceeding without it",
|
||||
"treated as no cookie",
|
||||
user_id,
|
||||
host_key,
|
||||
)
|
||||
return None
|
||||
return _write_cookiefile(raw, staging)
|
||||
return raw
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset: MediaAsset,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download (`usable_cookie`), or None."""
|
||||
raw = usable_cookie(db, user_id, url, asset, spec)
|
||||
return _write_cookiefile(raw, staging) if raw else None
|
||||
|
||||
|
||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||
|
||||
@@ -23,7 +23,6 @@ for a player hover-preview later.
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
@@ -33,9 +32,11 @@ import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
log = logging.getLogger("siftlode.edit")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def encode_thread_cap() -> int:
|
||||
@@ -71,7 +72,13 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
if crop:
|
||||
try:
|
||||
c = {k: int(round(float(crop[k]))) for k in ("x", "y", "w", "h")}
|
||||
except (KeyError, TypeError, ValueError):
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
# The clip still renders — uncropped. The user asked for a crop, got a whole frame,
|
||||
# and nothing anywhere called it a failure; this line is the only trace there is.
|
||||
log.warning(
|
||||
"the crop in an edit spec could not be read and was dropped",
|
||||
extra=logs.event("edit.spec.crop_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
c = None
|
||||
if c and c["w"] > 0 and c["h"] > 0 and c["x"] >= 0 and c["y"] >= 0:
|
||||
out["crop"] = c
|
||||
@@ -80,6 +87,7 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
segs = spec.get("segments")
|
||||
if isinstance(segs, list) and segs:
|
||||
norm: list[dict] = []
|
||||
unreadable = 0
|
||||
for s in segs:
|
||||
try:
|
||||
st = max(0.0, float((s or {}).get("start_s") or 0.0))
|
||||
@@ -88,9 +96,18 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
continue
|
||||
en = float(end_raw)
|
||||
except (TypeError, ValueError):
|
||||
# silent-ok: counted, not logged here — one cut-list can drop several, and the
|
||||
# join that comes out is simply shorter than the one the user built. Reported
|
||||
# once after the loop.
|
||||
unreadable += 1
|
||||
continue
|
||||
if en > st:
|
||||
norm.append({"start_s": round(st, 3), "end_s": round(en, 3)})
|
||||
if unreadable:
|
||||
log.warning(
|
||||
"some segments of a cut list could not be read and were dropped",
|
||||
extra=logs.event("edit.spec.segments_dropped", dropped=unreadable, kept=len(norm)),
|
||||
)
|
||||
norm.sort(key=lambda x: x["start_s"])
|
||||
# SEC-13: an accurate join builds one ffmpeg `filter_complex` with two filters per segment,
|
||||
# so an unbounded list becomes an unbounded command line and an ffmpeg that never finishes.
|
||||
@@ -114,7 +131,13 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
start = max(0.0, float(trim.get("start_s") or 0.0))
|
||||
end_raw = trim.get("end_s")
|
||||
end = float(end_raw) if end_raw is not None else None
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError) as exc:
|
||||
# Either a 400 the user cannot act on, or — beside a crop — a full-length clip where
|
||||
# they asked for a cut. Both are worth one line saying the spec was the problem.
|
||||
log.warning(
|
||||
"the trim in an edit spec could not be read and was dropped",
|
||||
extra=logs.event("edit.spec.trim_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
start, end = 0.0, None
|
||||
t: dict = {"start_s": round(start, 3)}
|
||||
if end is not None and end > start:
|
||||
@@ -306,11 +329,14 @@ def _parse_out_time(line: str) -> float | None:
|
||||
if line.startswith("out_time_us="):
|
||||
try:
|
||||
return int(line.split("=", 1)[1]) / 1_000_000
|
||||
# silent-ok: one line of ffmpeg's `-progress` stream, several per second. None means "this
|
||||
# line carried no usable time", which the caller already treats as "no progress yet".
|
||||
except ValueError:
|
||||
return None
|
||||
if line.startswith("out_time_ms="): # (some builds mislabel this as microseconds)
|
||||
try:
|
||||
return int(line.split("=", 1)[1]) / 1_000_000
|
||||
# silent-ok: as above, the other spelling of the same line.
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -329,7 +355,9 @@ def _pump(stream, sink: "queue.Queue[str | None]") -> None:
|
||||
try:
|
||||
for line in stream:
|
||||
sink.put(line)
|
||||
except (ValueError, OSError): # pipe closed under us by the kill path
|
||||
# silent-ok: the pipe was closed under us by our OWN kill path — the expected end of a
|
||||
# cancelled or timed-out run, and the code that did the killing is the code that reports it.
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
finally:
|
||||
sink.put(None)
|
||||
@@ -347,6 +375,8 @@ def _drain_tail(stream, sink: list[str]) -> None:
|
||||
sink.append(line)
|
||||
if len(sink) > 200:
|
||||
del sink[:100]
|
||||
# silent-ok: same closed pipe as `_pump` above, and the tail collected so far is still handed
|
||||
# to whoever reports the failure.
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
@@ -362,13 +392,17 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
return
|
||||
# silent-ok: the escalation below IS the handling of this timeout, and surviving THAT is what
|
||||
# the line under the second wait reports.
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
kill()
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid)
|
||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||
|
||||
|
||||
def _stop(proc: subprocess.Popen) -> None:
|
||||
@@ -404,6 +438,8 @@ def run_ffmpeg(
|
||||
while True:
|
||||
try:
|
||||
line = lines.get(timeout=1.0)
|
||||
# silent-ok: not a failure — the one-second timeout is how this loop gets a turn to
|
||||
# poll for cancellation while ffmpeg is quiet.
|
||||
except queue.Empty:
|
||||
line = ""
|
||||
if line is None: # stdout hit EOF — ffmpeg is done (or dying)
|
||||
@@ -416,11 +452,11 @@ def run_ffmpeg(
|
||||
if should_cancel():
|
||||
raise EditAborted
|
||||
if time.monotonic() - last_output > stall_timeout_s:
|
||||
raise RuntimeError(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
raise StatedFailure(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
try:
|
||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||
raise RuntimeError("ffmpeg did not exit after closing its output")
|
||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
||||
except BaseException:
|
||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||
@@ -435,7 +471,7 @@ def run_ffmpeg(
|
||||
if pipe:
|
||||
pipe.close()
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
|
||||
|
||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
||||
@@ -471,7 +507,13 @@ def run_ffprobe_json(
|
||||
try:
|
||||
out = subprocess.run(args, capture_output=True, text=True, timeout=30, check=True).stdout
|
||||
return json.loads(out)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError) as exc:
|
||||
# None makes every field it would have filled read as absent, and a file with no streams
|
||||
# and no duration is indistinguishable here from one that is not media at all.
|
||||
log.warning(
|
||||
"ffprobe could not read a file",
|
||||
extra=logs.event("media.ffprobe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -525,7 +567,13 @@ def probe_duration(path: Path) -> int:
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
return int(float(out)) if out else 0
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError) as exc:
|
||||
# Zero is the denominator of the re-encode progress bar, so a failed probe leaves the user
|
||||
# watching a bar that never moves rather than a job that says it went wrong.
|
||||
log.info(
|
||||
"the duration probe failed; progress for this job has no total",
|
||||
extra=logs.event("media.duration_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -670,15 +718,28 @@ def _run_ffmpeg_best_effort(cmd: list[str], timeout_s: int) -> Exception | None:
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, timeout=timeout_s, check=True)
|
||||
return None
|
||||
# silent-ok: the exception is the RETURN VALUE. This helper exists so the three callers share
|
||||
# one run/timeout policy, and each logs the failure with the context only it has.
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError) as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _step_failure(err: BaseException) -> str:
|
||||
"""What a best-effort image step's failure line says: the kind and, for ffmpeg, its exit
|
||||
status. Never `str(err)`: a CalledProcessError or TimeoutExpired quotes ffmpeg's argv, and an
|
||||
OSError its file name — the source path either way, which is built from the channel and the
|
||||
title."""
|
||||
code = getattr(err, "returncode", None)
|
||||
return f"{type(err).__name__} {code}" if code is not None else type(err).__name__
|
||||
|
||||
|
||||
def _is_jpeg(path: Path) -> bool:
|
||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
return f.read(3) == b"\xff\xd8\xff"
|
||||
# silent-ok: False only means "do not copy this straight through" — the caller then goes the
|
||||
# ffmpeg route, which opens the same file and reports it properly if it really is unreadable.
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
@@ -699,13 +760,15 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
shutil.copyfile(src, dest)
|
||||
return dest
|
||||
except OSError as exc:
|
||||
log.info("thumbnail copy failed (%s); poster falls back to a cut frame", str(exc)[:120])
|
||||
log.info(
|
||||
"thumbnail copy failed (%s); poster falls back to a cut frame", _step_failure(exc)
|
||||
)
|
||||
return None
|
||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||
if err is not None:
|
||||
log.info(
|
||||
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||
str(err)[:120],
|
||||
_step_failure(err),
|
||||
)
|
||||
return None
|
||||
return dest if dest.exists() else None
|
||||
@@ -730,7 +793,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
at = min(3.0, (asset.duration_s or 10) * 0.1)
|
||||
err = _run_ffmpeg_best_effort(build_poster_cmd(src, dest, at), timeout_s)
|
||||
if err is not None:
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
return rel if dest.exists() else None
|
||||
|
||||
@@ -761,7 +824,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
||||
cmd = build_storyboard_cmd(src, dest, asset.duration_s, geom)
|
||||
err = _run_ffmpeg_best_effort(cmd, timeout_s)
|
||||
if err is not None:
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
if not dest.exists():
|
||||
return None
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code.
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code — and, where a site's
|
||||
failure has been measured, into that site's entry of the failure catalog.
|
||||
|
||||
The worker stores the raw exception text (truncated) for technical detail, but a raw
|
||||
`ERROR: [youtube] <id>: Private video. Sign in…` is noise to a user. `classify()` maps that text
|
||||
@@ -6,21 +7,51 @@ to one of a small set of category codes; the frontend turns the code into a loca
|
||||
explanation (and decides whether a retry makes sense). The raw text stays available behind a
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
|
||||
**The failure catalog** (`_CATALOG`, E11.5). A category is generic: "Sign-in required" cannot tell a
|
||||
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
||||
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
||||
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
||||
its own sentence for — why it failed on that site, and what the user can do. `diagnose()` tries the
|
||||
catalog first and falls back to the category. How an entry is added, every time a failure is reported:
|
||||
1. reproduce it on dev (with and without the user's cookie) and keep the verbatim yt-dlp text;
|
||||
2. add the entry below, scoped as narrowly as the measurement allows (a marker that is yt-dlp's
|
||||
generic wording for every site must never match without its host; a marker another module also
|
||||
decides on — the Facebook Chrome retry's FACEBOOK_PARSE_WALL — is imported, never retyped);
|
||||
3. pin the real text in `tests/test_download_errors.py`;
|
||||
4. add `downloads.errors.fail.catalog.<detail>.{short,body}` in EN and HU (a contract test checks it;
|
||||
the group is not called `detail`, which `actionErrors.contract` reads as a raw `HttpError.detail`).
|
||||
"""
|
||||
|
||||
# Two worker-authored strings that this module owns because both are coupled to `_RULES`.
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
|
||||
from app.downloads.adapters import (
|
||||
FACEBOOK_PARSE_WALL,
|
||||
POST_NOT_SHOWN_MESSAGE,
|
||||
POST_WITHOUT_VIDEO_MESSAGE,
|
||||
facebook_group_post_id,
|
||||
)
|
||||
from app.downloads.credentials import host_key_for_url
|
||||
|
||||
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
||||
#
|
||||
# QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule
|
||||
# below matches a fragment of it, and test_download_errors pins the pair, because a message and its
|
||||
# rule drifting apart is exactly how a category silently becomes unreachable.
|
||||
#
|
||||
# NETWORK_REDACTED_MESSAGE is the opposite direction: it REPLACES a message after classification, so
|
||||
# it deliberately has no rule. Nothing re-classifies a stored `job.error` today; if anything ever
|
||||
# does, it must carry the original code rather than re-deriving one from this text.
|
||||
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
||||
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
||||
# text (`worker._process_job`), so it has a rule of its own that gives the category back.
|
||||
#
|
||||
# Other worker-raised texts are still inline literals at their raise sites. The ffmpeg ones and
|
||||
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
||||
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
||||
#
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`, E11.5 S4) are adapters.py's
|
||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||
# that gives the site's sentence.
|
||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||
|
||||
@@ -30,6 +61,12 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
||||
# first — it is our wording, so it can never be a false match on an upstream message.
|
||||
("download storage is full", "quota_exceeded"),
|
||||
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
||||
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
||||
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
||||
# or the site answered the signed-in read with something that is not the post at all.
|
||||
(POST_WITHOUT_VIDEO_MESSAGE.lower(), "format_unavailable"),
|
||||
(POST_NOT_SHOWN_MESSAGE.lower(), "login_required"),
|
||||
# Login / age / members / bot-check — the user must be signed in (we deliberately are not).
|
||||
("sign in to confirm your age", "login_required"),
|
||||
("confirm your age", "login_required"),
|
||||
@@ -101,6 +138,120 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
)
|
||||
|
||||
|
||||
class Diagnosis(NamedTuple):
|
||||
"""What a failure was: the generic `code` (category) and, when a catalog entry matched, its
|
||||
host-specific `detail`."""
|
||||
|
||||
code: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Entry:
|
||||
"""One measured failure of one site. It matches when the download URL belongs to `host`
|
||||
(`credentials.SUPPORTED_HOSTS`), every one of `markers` is in the message, the cookie state is
|
||||
`cookie` (None: either), and — when `link` is given — it recognises the URL (a truthy result). A
|
||||
link predicate is the adapter's own, so the catalog and the download agree on what a link is."""
|
||||
|
||||
host: str
|
||||
markers: tuple[str, ...]
|
||||
code: str
|
||||
detail: str
|
||||
cookie: bool | None = None
|
||||
link: Callable[[str], object] | None = None
|
||||
|
||||
|
||||
_CATALOG: tuple[_Entry, ...] = (
|
||||
# Facebook answers a signed-out client with a page yt-dlp cannot parse — an 18+ reel, a closed
|
||||
# group's post (measured 2026-09-13: prod jobs 145, 156, 256 download once the owner's cookie is
|
||||
# attached). "Cannot parse data" is yt-dlp's generic wording for a broken extractor on EVERY site,
|
||||
# which is why the entry is scoped to Facebook and never becomes a general rule.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.login_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same text WITH a cookie: an expired / signed-out cookie, or a page that also needs
|
||||
# impersonation (prod job 257). Telling this user to add cookies would be wrong — they have.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post read with a cookie that Facebook answered with its login page (measured with a
|
||||
# refused cookie: no post id anywhere on the page) — the same refused cookie, found by the adapter.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_NOT_SHOWN_MESSAGE.lower(),),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post with no single video of its own. With a cookie the adapter reads the post's page
|
||||
# and says so itself (POST_WITHOUT_VIDEO_MESSAGE); without one yt-dlp is given the post link and
|
||||
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
|
||||
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
|
||||
# that video, not the post — the category says it. Only for group post links (the adapter's own
|
||||
# test of one): a reel with no formats is a different failure.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", "no video formats found"),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
cookie=False,
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
|
||||
# neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
|
||||
# identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
|
||||
# The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
|
||||
# "you're not a bot" with a straight quote never matches the real text.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.bot_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
|
||||
# one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
|
||||
# and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
|
||||
# longer valid. They have likely been rotated in the browser as a security measure."
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# The same gate one step later: the player response comes back, and then the CDN refuses the
|
||||
# stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
|
||||
# very same video, ~90 minutes after it was answering with the bot message — signed out it
|
||||
# 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
|
||||
# the category this text carries generically, `network`, is a lie here: nothing was wrong with
|
||||
# the network. Scoped to a cookie-less YouTube download, where that is what we measured.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("unable to download video data", "403"),
|
||||
code="login_required",
|
||||
detail="youtube.media_forbidden",
|
||||
cookie=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def classify(msg: str | None) -> str:
|
||||
"""Return a category code for a raw failure message (``"unknown"`` if nothing matches)."""
|
||||
if not msg:
|
||||
@@ -112,6 +263,27 @@ def classify(msg: str | None) -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def diagnose(
|
||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||
) -> Diagnosis:
|
||||
"""The catalog entry for this failure of this link, else just its category (see the module doc).
|
||||
|
||||
`url` is the address yt-dlp was given; `cookie_attached` whether the acting user's cookie went
|
||||
with it. Without a URL only the category can be known."""
|
||||
host = host_key_for_url(url) if msg and url else None
|
||||
if host is not None:
|
||||
low = msg.lower() # type: ignore[union-attr] — `host` is only set when `msg` is truthy
|
||||
for e in _CATALOG:
|
||||
if (
|
||||
e.host == host
|
||||
and all(m in low for m in e.markers)
|
||||
and (e.cookie is None or e.cookie == cookie_attached)
|
||||
and (e.link is None or e.link(url)) # type: ignore[arg-type] — as `msg` above
|
||||
):
|
||||
return Diagnosis(e.code, e.detail)
|
||||
return Diagnosis(classify(msg))
|
||||
|
||||
|
||||
def user_error_text(source_kind: str, msg: str, code: str) -> str:
|
||||
"""The failure text to STORE on the job, given the raw message and its category.
|
||||
|
||||
|
||||
@@ -17,8 +17,134 @@ spec shape (all keys optional; defaults + browser-playability clamp applied by `
|
||||
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from collections.abc import Iterator
|
||||
|
||||
from app import logs
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# yt-dlp names what it is working on in these shapes, and each one is a row of someone's download
|
||||
# history: the source URL, the `[extractor] <id>:` prefix on nearly every warning and error, the
|
||||
# file path it writes to (`storage.rel_path` builds it from the channel and the title), a
|
||||
# playlist's title, and an id quoted mid-sentence. The job id is already on every line
|
||||
# (`logs.set_job` in the download child), so none of them adds anything an operator needs. What
|
||||
# stays: the host (it says which extractor is failing) and the extractor tag. Every pattern below
|
||||
# is a shape measured in yt-dlp's own source (2026.07.04), pinned in test_ytdlp_logging.py.
|
||||
_URL_RE = re.compile(r"https?://([^/\s]+)\S*")
|
||||
# `Destination:` is yt-dlp's own word in `[download] Destination: <path>`, not an id — the path
|
||||
# after it is what `_PATH_RE` takes out.
|
||||
_ID_PREFIX_RE = re.compile(r"(\[[\w:.-]+\] )(?!Destination:)[^\s:\[\]]+:")
|
||||
# The generic extractor's "id" is the URL's basename, percent-DECODED: `My%20Holiday.mp4` is
|
||||
# `My Holiday` — spaces, brackets, colons and all, which no pattern can bound (`Our Trip: Day 1`
|
||||
# ends at which `: `?). The job knows its own URL, so the exact name is taken out as a literal
|
||||
# (`hiding`). This pattern is the fallback for a line logged outside that scope: up to the first
|
||||
# `: `. Only this extractor: every other one names a site id, and letting the prefix run to `: `
|
||||
# everywhere would take the words out of lines like `[VideoConvertor] Converting …; Destination:`.
|
||||
_GENERIC_ID_RE = re.compile(r"(\[generic\] )(?!Destination:)[^\[\]\n]+?:(?= |$)", re.M)
|
||||
# `[youtube:tab] Playlist <title>: Downloading 5 items` — the title is not an id, and has spaces.
|
||||
_PLAYLIST_TITLE_RE = re.compile(r"(\] Playlist ).+?(?=: Downloading )")
|
||||
# `[download] Finished downloading playlist: <title>`, `[download] Downloading playlist: <title>`,
|
||||
# `… Skipping already downloaded playlist: <title>` — the title runs to the end of the line.
|
||||
_TITLE_TAIL_RE = re.compile(r"(playlist: |\[download\] Downloading \w+: ).*", re.I)
|
||||
# An id quoted mid-sentence, not as a prefix: `… not available for <id>`, and `--no-playlist`'s
|
||||
# `Downloading just the video <id>` (`_yes_playlist`) / `Downloading just video <id>` (multifeed).
|
||||
_BARE_ID_RE = re.compile(
|
||||
r"((?:subtitles not available for|Downloading just (?:the )?\w+) )[^\s,;]+"
|
||||
)
|
||||
# Two segments or more, and not glued to a word: `www.youtube.com/…` is what the URL pattern
|
||||
# leaves behind, and it must survive this one.
|
||||
_PATH_RE = re.compile(r"(?<![\w.…])/(?:[^/\s\"']+/)+[^/\s\"']*")
|
||||
|
||||
|
||||
_HIDDEN: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
|
||||
"formats_hidden", default=()
|
||||
)
|
||||
|
||||
|
||||
def generic_id(url: str) -> str:
|
||||
"""The name yt-dlp's generic extractor gives a URL: its basename, extension off, percent-
|
||||
decoded — `InfoExtractor._generic_id` (2026.07.04), pinned against it in the tests."""
|
||||
return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hiding(url: str | None) -> Iterator[None]:
|
||||
"""For the work on one URL: `log_safe` also takes out the name yt-dlp's generic extractor
|
||||
gives it, exactly, where yt-dlp puts it (`[generic] <name>:`). A context variable, so it
|
||||
holds for yt-dlp's logger calls too — they run on the same thread as the extraction."""
|
||||
name = generic_id(url) if url else ""
|
||||
token = _HIDDEN.set((*_HIDDEN.get(), name) if name else _HIDDEN.get())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_HIDDEN.reset(token)
|
||||
|
||||
|
||||
def log_safe(text: object) -> str:
|
||||
"""yt-dlp's text (its own lines, or a DownloadError's message) with the parts that name a
|
||||
person's download taken out. Every log line that carries yt-dlp's words goes through here,
|
||||
not just the adapter's: yt-dlp raises the same text it has just logged, so a line that skipped
|
||||
it would put back one line later exactly what the adapter took out."""
|
||||
text = str(text)
|
||||
for name in _HIDDEN.get():
|
||||
text = text.replace(f"[generic] {name}:", "[generic] …:")
|
||||
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", text)
|
||||
text = _ID_PREFIX_RE.sub(r"\1…:", text)
|
||||
text = _GENERIC_ID_RE.sub(r"\1…:", text)
|
||||
text = _PLAYLIST_TITLE_RE.sub(r"\1…", text)
|
||||
text = _TITLE_TAIL_RE.sub(r"\1…", text)
|
||||
text = _BARE_ID_RE.sub(r"\1…", text)
|
||||
return _PATH_RE.sub("…", text)
|
||||
|
||||
|
||||
class _YdlLog:
|
||||
"""yt-dlp's logger protocol, pointed at ours.
|
||||
|
||||
Without it yt-dlp explains itself to nobody: `quiet` sends its output to a stdout we do not
|
||||
read, so "Requested format is not available", a signature-extraction failure or a POT refusal
|
||||
reached us only as whatever text ended up in the raised exception. Measured against yt-dlp
|
||||
2026.07.04: with `logger` set it calls `debug` for extraction and progress lines and
|
||||
`warning`/`error` for the rest, and `quiet`/`no_warnings` do NOT gate those calls.
|
||||
|
||||
`debug` goes to TRACE: it is several lines per second during a download, so it shows only
|
||||
when the whole process runs at TRACE. (Not under a user's own Teljes consent — that lives in
|
||||
the API process's request scope, and the download child that runs yt-dlp has none.)
|
||||
|
||||
`error` goes to WARNING, not ERROR. yt-dlp calls it from `trouble()` just before it raises,
|
||||
on EVERY failed attempt — including the ones the pipeline then recovers from with its fallback
|
||||
client set. Whether the job failed is the pipeline's verdict (`download.job.failed`), and that
|
||||
is the one line whose level should say so."""
|
||||
|
||||
def _emit(self, level: int, msg: str) -> None:
|
||||
# The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
|
||||
# `[download] 53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
|
||||
# a record that has args, this way the question never arises — and it is the shape ruff's
|
||||
# `G` rules ask for. (The sink's flood limiter groups a bare `%s` by the text itself, so
|
||||
# distinct yt-dlp warnings do not share one bucket — `logsink._template`.)
|
||||
log.log(level, "%s", log_safe(msg), extra=logs.event("download.ytdlp"))
|
||||
|
||||
def debug(self, msg):
|
||||
self._emit(logs.TRACE, msg)
|
||||
|
||||
def info(self, msg):
|
||||
self._emit(logging.DEBUG, msg)
|
||||
|
||||
def warning(self, msg):
|
||||
self._emit(logging.WARNING, msg)
|
||||
|
||||
def error(self, msg):
|
||||
self._emit(logging.WARNING, msg)
|
||||
|
||||
|
||||
_YDL_LOG = _YdlLog()
|
||||
|
||||
# Fields that affect the produced file — and therefore the cache identity.
|
||||
_SIG_KEYS = (
|
||||
@@ -57,6 +183,10 @@ _DEFAULTS = {
|
||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||
_SIG_VERSION = 4
|
||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||
_AUDIO_SIG_VERSION = 5
|
||||
|
||||
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
||||
|
||||
@@ -74,7 +204,13 @@ def normalize(spec: dict) -> dict:
|
||||
# rather than raising an unhandled 500 at enqueue.
|
||||
try:
|
||||
s["max_height"] = int(s["max_height"])
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError) as exc:
|
||||
# "best" is the opposite of what a height cap asks for: the user set a ceiling to save
|
||||
# bandwidth or disk, and the download quietly takes the largest rendition there is.
|
||||
log.warning(
|
||||
"the height cap on a download could not be read and was dropped",
|
||||
extra=logs.event("download.format.max_height_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
s["max_height"] = None
|
||||
for b in ("embed_subs", "embed_chapters", "embed_thumbnail", "sponsorblock"):
|
||||
s[b] = bool(s[b])
|
||||
@@ -96,20 +232,34 @@ def normalize(spec: dict) -> dict:
|
||||
return s
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None) -> str:
|
||||
def _sig_version(s: dict) -> int:
|
||||
return _AUDIO_SIG_VERSION if s["mode"] == "a" else _SIG_VERSION
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None, *, version: int | None = None) -> str:
|
||||
"""Stable short hash of the output-affecting spec — the MediaAsset cache key component.
|
||||
|
||||
`salt` forks a distinct, non-dedup identity for the same spec (used by force-redownload so it
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs)."""
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs).
|
||||
`version` hashes under a signature version other than the current one (see `format_sigs`)."""
|
||||
s = normalize(spec)
|
||||
data = {k: s[k] for k in _SIG_KEYS}
|
||||
data["_v"] = _SIG_VERSION
|
||||
data["_v"] = _sig_version(s) if version is None else version
|
||||
if salt is not None:
|
||||
data["_salt"] = salt
|
||||
payload = json.dumps(data, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def format_sigs(spec: dict, salt: str | None = None) -> tuple[str, str]:
|
||||
"""`format_sig` under the current signature version and under the one before it. An asset keeps
|
||||
the signature it was created with, so a job queued before a deploy that bumped the version and
|
||||
run after it downloads into an asset carrying the previous one — which must still be recognised
|
||||
as that user's private asset (`credentials.is_authed_asset`), or it downloads without its cookie."""
|
||||
previous = _sig_version(normalize(spec)) - 1
|
||||
return format_sig(spec, salt), format_sig(spec, salt, version=previous)
|
||||
|
||||
|
||||
def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
@@ -163,6 +313,12 @@ def build_ydl_opts(
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
# With a logger attached, `quiet` and `no_warnings` stop deciding whether yt-dlp explains itself
|
||||
# and only decide where — so both stay on above and the explanation comes here instead of to a
|
||||
# stdout nothing reads. AFTER the adapter merge, like `cookiefile` and the size cap: an adapter
|
||||
# must not be able to silence the one account of what went wrong that a failed job leaves.
|
||||
opts["logger"] = _YDL_LOG
|
||||
|
||||
# SEC-13: hard ceiling on a single download. AFTER the adapter merge, for the same reason
|
||||
# `cookiefile` is — a source adapter must not be able to raise a limit the quota layer set.
|
||||
#
|
||||
|
||||
@@ -16,7 +16,6 @@ Five passes each run:
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -24,14 +23,13 @@ from pathlib import Path
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.gc")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
@@ -99,6 +97,7 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
if pause_s and touched % batch == 0:
|
||||
time.sleep(pause_s)
|
||||
|
||||
unreclaimed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
continue
|
||||
@@ -111,7 +110,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
f.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
# silent-ok: counted, not logged per file — a broken mount would put one line
|
||||
# under every orphan. The total is reported once at the end of the sweep.
|
||||
unreclaimed += 1
|
||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||
for d in sorted(
|
||||
(p for p in entry.rglob("*") if p.is_dir()),
|
||||
@@ -120,12 +121,20 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
):
|
||||
try:
|
||||
d.rmdir()
|
||||
# silent-ok: `rmdir` refusing means the directory still holds files, which is
|
||||
# the ordinary answer here — this prune only takes the ones that emptied.
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
entry.rmdir()
|
||||
# silent-ok: the same prune one level up, at the channel directory.
|
||||
except OSError:
|
||||
pass
|
||||
if unreclaimed:
|
||||
log.warning(
|
||||
"some orphaned download files could not be reclaimed",
|
||||
extra=logs.event("download.gc.unreclaimed", files=unreclaimed, removed=removed),
|
||||
)
|
||||
return removed
|
||||
|
||||
|
||||
|
||||
@@ -49,6 +49,8 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
||||
exp_s, sig = grant.split(".", 1)
|
||||
try:
|
||||
exp = int(exp_s)
|
||||
# silent-ok: an untrusted grant that does not parse is simply not a valid grant, and the
|
||||
# honest answer to it — a password prompt — is the same as for one that expired.
|
||||
except ValueError:
|
||||
return False
|
||||
if exp < datetime.now(timezone.utc).timestamp():
|
||||
|
||||
@@ -66,6 +66,8 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
||||
width = (data[3] << 8) + data[4]
|
||||
return width, height
|
||||
f.seek(length - 2, 1) # skip this segment's payload
|
||||
# silent-ok: None omits width/height from the unfurl card, which is what the card does for
|
||||
# every poster whose dimensions were never asked for. Nothing is claimed that is not true.
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
@@ -8,9 +7,8 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
@@ -25,12 +23,13 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
_touch_job,
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class _Aborted(Exception):
|
||||
@@ -99,6 +98,8 @@ def _parse_upload_date(raw) -> date | None:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(str(raw), "%Y%m%d").date()
|
||||
# silent-ok: a date we cannot read becomes a date the card does not show, which is what the
|
||||
# card shows for every source that never supplies one. Absent is the honest rendering.
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@@ -114,9 +115,10 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
# `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty
|
||||
# (e.g. a title that was only a social engagement prefix); the completion path later sets
|
||||
# the final title with its own source_ref fallback.
|
||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||
adapter = adapters.resolve_adapter(None, None, info)
|
||||
asset.title = normalize_title(adapter.title(info)) or info.get("id")
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
@@ -138,7 +140,14 @@ def _make_progress_hook(job_id: int, asset_id: int):
|
||||
try:
|
||||
_fill_asset_meta_early(asset_id, d["info_dict"])
|
||||
except Exception: # noqa: BLE001 — metadata is best-effort, never fail a download
|
||||
pass
|
||||
# Best-effort is about the DOWNLOAD, not about knowing. What this swallows is
|
||||
# anything from a shape change in yt-dlp's info dict to the database being down,
|
||||
# and all the user sees is a card with an id on it until the job finishes.
|
||||
log.info(
|
||||
"early metadata could not be filled in for this download",
|
||||
extra=logs.event("download.meta.early_fill_failed", asset_id=asset_id),
|
||||
exc_info=True,
|
||||
)
|
||||
if d.get("status") != "downloading":
|
||||
return
|
||||
if not allow_write():
|
||||
@@ -299,7 +308,7 @@ def _produced_file(info: dict, staging: Path) -> Path:
|
||||
and not _YTDLP_TEMP_RE.search(p.name)
|
||||
]
|
||||
if not cands:
|
||||
raise RuntimeError("yt-dlp produced no output file")
|
||||
raise StatedFailure("yt-dlp produced no output file")
|
||||
return max(cands, key=lambda p: p.stat().st_size)
|
||||
|
||||
|
||||
@@ -323,14 +332,33 @@ def _extract_with_fallback(
|
||||
):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other
|
||||
sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
|
||||
Before the first attempt the adapter may resolve `url` to the link that really holds the video (a
|
||||
Facebook group post → its video's watch link); when the page shows no single video there, or is not
|
||||
the post at all (a refused cookie's login page), the adapter raises its own verdict
|
||||
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
||||
download as itself.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
client_sets = adapter.client_sets()
|
||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||
_touch_job(job_id)
|
||||
target = adapter.resolve_url(
|
||||
url,
|
||||
formats.build_ydl_opts(
|
||||
spec,
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
cookiefile=cookiefile,
|
||||
max_bytes=max_bytes,
|
||||
),
|
||||
)
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
@@ -348,7 +376,7 @@ def _extract_with_fallback(
|
||||
# killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.)
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
return ydl.extract_info(url, download=True)
|
||||
return ydl.extract_info(target, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
@@ -356,13 +384,22 @@ def _extract_with_fallback(
|
||||
"job %s: clients %s failed (%s); retrying with fallback",
|
||||
job_id,
|
||||
clients,
|
||||
str(exc)[:70],
|
||||
formats.log_safe(exc)[:70],
|
||||
)
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
# A partial left behind is not inert: the next client set resumes into it,
|
||||
# so the retry this loop exists to make clean starts dirty instead.
|
||||
log.info(
|
||||
"a partial file could not be removed before the retry",
|
||||
extra=logs.event(
|
||||
"download.retry.partial_unlink_failed",
|
||||
job_id=job_id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
_set_job(job_id, progress=0, phase="downloading")
|
||||
continue
|
||||
raise
|
||||
@@ -385,8 +422,12 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk."""
|
||||
if spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
||||
|
||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
||||
which iPad's WebKit refuses to play (prod job 268)."""
|
||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
return produced
|
||||
vcodec, acodec = editmod.probe_codecs(produced)
|
||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||
@@ -425,6 +466,7 @@ def _download(
|
||||
) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
cookiefile: Path | None = None
|
||||
url: str | None = None # read by the failure branch, which can run before it is resolved
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
@@ -440,17 +482,15 @@ def _download(
|
||||
try:
|
||||
urlguard.check_url(url)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise RuntimeError(f"Refusing to download from this address ({e.reason}).") from e
|
||||
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
asset_sig = cdb.execute(
|
||||
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||
).scalar_one_or_none()
|
||||
if asset_sig is not None:
|
||||
target = cdb.get(MediaAsset, asset_id)
|
||||
if target is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, asset_sig, spec, staging
|
||||
cdb, user_id, url, target, spec, staging
|
||||
)
|
||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||
@@ -484,13 +524,19 @@ def _download(
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||
# media_kind from those nulls misclassified a real H.264 video as a download-only 'file', and a
|
||||
# missing duration hid the trim action. Probed before the sidecars so the .nfo runtime has it
|
||||
# too. Same probe the upload ingest uses.
|
||||
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
|
||||
meta = storage.MediaMeta(
|
||||
video_id=info.get("id") or source_ref,
|
||||
title=normalize_title(info.get("title")) or source_ref,
|
||||
title=normalize_title(adapter.title(info)) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||
duration_s=duration_s,
|
||||
description=info.get("description"),
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
@@ -516,12 +562,11 @@ def _download(
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = size
|
||||
asset.duration_s = meta.duration_s
|
||||
# Codecs/dims come from a PROBE of the finished file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose codec fields are absent for some sources (Facebook among them):
|
||||
# deriving media_kind from those nulls misclassified a real H.264 video as a
|
||||
# download-only 'file' with no in-app play/edit. Same probe the upload ingest uses.
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = probe.resolve_media_facts(
|
||||
final, info
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||
vcodec,
|
||||
acodec,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
asset.container = ext
|
||||
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
||||
@@ -559,13 +604,23 @@ def _download(
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||
log.info("job %s done: %s", job_id, rel)
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("download.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
# Paused/canceled mid-download or mid-transcode: free the asset for a later retry; keep the
|
||||
# job's status (the route set it to paused/canceled). ref_count is adjusted by the route.
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("job %s aborted (pause/cancel)", job_id)
|
||||
log.info(
|
||||
"job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("download.job.aborted", job_id=job_id),
|
||||
)
|
||||
except _OwnerRefused as exc:
|
||||
# Fail this job only. The asset is never marked `error`; it goes back to `pending` (below),
|
||||
# so another user's job for the same video is claimed and downloaded normally instead of
|
||||
@@ -578,6 +633,7 @@ def _download(
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_detail=None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
@@ -585,29 +641,41 @@ def _download(
|
||||
# it, and this one must already be out of the way. (The staging tree is still torn down in
|
||||
# `finally` below — the same narrow window the pause/cancel path above has always had.)
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("job %s refused for its owner: %s", job_id, msg)
|
||||
log.info(
|
||||
"job %s refused for its owner: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event("download.job.refused", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||
msg = str(exc)[:500]
|
||||
code = dlerrors.classify(msg)
|
||||
# The catalog needs the link and whether the user's cookie went with it: the same yt-dlp
|
||||
# text is Facebook's login wall without one and a refused cookie with one.
|
||||
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
|
||||
msg = dlerrors.user_error_text(source_kind, msg, code)
|
||||
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never
|
||||
# overwritten with `error` (error also holds the asset → would re-open the
|
||||
# double-release class).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=code,
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
# After the write: a job the write could not fail is not a failed job. When it raises,
|
||||
# the runner reports this failure with it (`runner._run`) and the worker fails the job.
|
||||
# silent-ok: not silent — `_log_download_failed` is the report.
|
||||
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
@@ -616,7 +684,50 @@ def _download(
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
def _failure_stack(exc: BaseException) -> dict[str, str]:
|
||||
"""The `extra=` fragment that puts a failure's traceback on its line — or nothing.
|
||||
|
||||
Nothing for yt-dlp's DownloadError and for our own `StatedFailure` (`ffmpeg failed (1): …`):
|
||||
their message is the whole story, and their stack is only the `except` that caught them.
|
||||
Everything else keeps it — a TypeError on an odd info dict, a plain RuntimeError, an OSError,
|
||||
a database error: that is our code or our platform failing, and the stack is the lead.
|
||||
|
||||
Scrubbed, link by link (`logs.scrubbed_stack`): a stack's messages are data — an OSError names
|
||||
its file, and a database error raised inside `except DownloadError` carries yt-dlp's
|
||||
unscrubbed text as its context."""
|
||||
if isinstance(exc, (yt_dlp.utils.DownloadError, StatedFailure)):
|
||||
return {}
|
||||
return logs.scrubbed_stack(exc, formats.log_safe)
|
||||
|
||||
|
||||
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
|
||||
"""The child's `download.job.failed` line. `msg` is the user's copy and keeps their URL; the
|
||||
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
|
||||
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
|
||||
place the failure: a recognised one is fully described by its code."""
|
||||
unplaced = code in (None, "", "unknown")
|
||||
log.warning(
|
||||
"job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
error_code=code,
|
||||
error_type=type(exc).__name__,
|
||||
),
|
||||
**(_failure_stack(exc) if unplaced else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
|
||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||
Logged after the write, as on the download error path."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
@@ -632,7 +743,22 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"edit.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
),
|
||||
**(_failure_stack(exc) if exc is not None else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
@@ -684,7 +810,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
if not dest_staging.exists():
|
||||
raise RuntimeError("ffmpeg produced no output file")
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
@@ -720,13 +846,24 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||
log.info("edit job %s done: %s", job_id, rel)
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"edit job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("edit.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: free the asset for a later retry (ref_count is the route's job).
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("edit job %s aborted (pause/cancel)", job_id)
|
||||
log.info(
|
||||
"edit job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("edit.job.aborted", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500])
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
@@ -64,6 +64,8 @@ def _facts_from_probe(
|
||||
raw = fmt.get("duration")
|
||||
if raw is not None:
|
||||
duration = int(float(raw))
|
||||
# silent-ok: an unusable duration is stored as no duration, which the UI renders as unknown
|
||||
# rather than as zero — the honest state, and the one a re-probe can still fill in.
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
return vcodec, acodec, width, height, duration
|
||||
@@ -81,10 +83,12 @@ def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | N
|
||||
|
||||
def resolve_media_facts(
|
||||
path: Path, info: dict
|
||||
) -> tuple[str | None, str | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height) for the stored download asset: PROBE the finished file on disk
|
||||
(the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources (Facebook among
|
||||
them), so deriving media_kind from it there labelled a real H.264 video a download-only 'file'.
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) for the stored download asset: PROBE the finished
|
||||
file on disk (the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources
|
||||
(Facebook among them), so deriving media_kind from it there labelled a real H.264 video a
|
||||
download-only 'file'. It omits the duration too (a Facebook reel, an Instagram reel), and a NULL
|
||||
duration hides the Library row's trim action although the file plays for minutes.
|
||||
|
||||
Precedence is by whether the probe RAN, not per-field: if ffprobe read the file, its codec verdict
|
||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||
@@ -92,16 +96,23 @@ def resolve_media_facts(
|
||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
||||
dimensions still gap-fill from `info` in either case."""
|
||||
dimensions and the duration still gap-fill from `info` in either case."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
info_w = int(info["width"]) if info.get("width") else None
|
||||
info_h = int(info["height"]) if info.get("height") else None
|
||||
info_d = int(info["duration"]) if info.get("duration") else None
|
||||
if data is None:
|
||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
||||
return info_v, info_a, info_w, info_h
|
||||
pv, pa, pw, ph, _dur = _facts_from_probe(data)
|
||||
return info_v, info_a, info_w, info_h, info_d
|
||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||
# gap-fills the cosmetic width/height the probe couldn't read.
|
||||
return pv, pa, pw if pw is not None else info_w, ph if ph is not None else info_h
|
||||
# gap-fills the width/height/duration the probe couldn't read.
|
||||
return (
|
||||
pv,
|
||||
pa,
|
||||
pw if pw is not None else info_w,
|
||||
ph if ph is not None else info_h,
|
||||
pd if pd is not None else info_d,
|
||||
)
|
||||
|
||||
@@ -11,18 +11,20 @@ writes the terminal job/asset state itself (done / error, or a cooperative pause
|
||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from app import logs
|
||||
from app import logsink
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logging.getLogger("siftlode.worker.runner")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _deprioritize() -> None:
|
||||
@@ -52,12 +54,21 @@ def _deprioritize() -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logs.configure_logging("runner")
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||
# noqa T201: a usage line is for the person who typed the command. Logging is already
|
||||
# configured here, and in a deployment it writes JSON records — the one reader of this
|
||||
# line wants the bare sentence, not a record with a request id they do not have.
|
||||
print( # noqa: T201
|
||||
"usage: python -m app.downloads.runner <job_id>", file=sys.stderr
|
||||
)
|
||||
raise SystemExit(2)
|
||||
job_id = int(sys.argv[1])
|
||||
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
||||
# output joins up with the worker's and the API's view of the same download.
|
||||
logs.set_job(job_id)
|
||||
logsink.start() # flushed at exit (atexit) — this process has no shutdown hook of its own
|
||||
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
@@ -71,14 +82,40 @@ def main() -> None:
|
||||
if asset_id is None:
|
||||
return
|
||||
|
||||
_run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id)
|
||||
|
||||
|
||||
def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) -> None:
|
||||
"""Run the job, and be the last stop before Python's own exception hook.
|
||||
|
||||
What gets past the pipeline is a failure it could not even record (the database refused the
|
||||
write that fails the job). Python's hook would print it to stderr raw — and its context is the
|
||||
download's own failure, whose text is the URL or ffmpeg's stderr naming the file. Here it is
|
||||
one line with the whole chain scrubbed; the non-zero exit then has the worker fail the job
|
||||
(`cause=child_exit`)."""
|
||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||
from app.downloads.pipeline import _download, _process_edit
|
||||
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
with formats.hiding(source_ref):
|
||||
try:
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||
# noqa TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||
log.error( # noqa: TRY400
|
||||
"the download child for job %s failed",
|
||||
job_id,
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.child.failed", job_id=job_id, error_type=type(exc).__name__
|
||||
),
|
||||
**logs.scrubbed_stack(exc, formats.log_safe),
|
||||
},
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -7,6 +7,7 @@ handles everything else. State transitions used by the routes (pause/resume/canc
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
@@ -80,11 +81,15 @@ def get_or_create_asset(
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
try:
|
||||
db.flush()
|
||||
# A SAVEPOINT: losing the race undoes only this insert, never the caller's own pending writes
|
||||
# (Retry clears the job's error before it gets here; a whole-session rollback threw that away).
|
||||
with db.begin_nested():
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
# silent-ok: the expected outcome of the race this savepoint exists for — two jobs wanting the
|
||||
# same asset. Losing it means the other insert won, and re-selecting is the whole handling.
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
asset = db.execute(stmt).scalar_one()
|
||||
return asset
|
||||
|
||||
@@ -341,13 +346,20 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
"""Requeue a paused/errored job for a fresh attempt, clearing its error. If the SHARED asset itself
|
||||
failed, reset it to pending so the worker actually re-downloads — otherwise the requeued job would
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset)."""
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset).
|
||||
|
||||
A cookie the user added AFTER the job was queued moves the job onto their private asset first
|
||||
(`_follow_the_cookie`): the worker only ever attaches a cookie to that identity, so without the move
|
||||
the Retry a login-wall message asks for would fail on the same wall."""
|
||||
if job.status in (JobStatus.PAUSED, JobStatus.ERROR):
|
||||
job.progress = 0
|
||||
job.error = None
|
||||
job.error_code = None
|
||||
job.error_detail = None
|
||||
if job.asset_id:
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is not None:
|
||||
asset = _follow_the_cookie(db, job, asset)
|
||||
if asset and asset.status == "error": # MediaAsset.status (no enum), not a JobStatus
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
@@ -355,6 +367,32 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
"""Re-point a requeued job at the asset enqueue would pick NOW, when the user has a cookie for the
|
||||
link's host but the job still holds an asset the worker won't attach it to (queued before the
|
||||
cookie existed). Returns the asset the job holds afterwards.
|
||||
|
||||
Only a `pending` / `error` asset moves — neither has a file nor a download in flight. A `ready` one
|
||||
already has the file, and a `downloading` one may be this job's own child still being stopped by
|
||||
the pause; the next Retry after it settles moves the job then. Removing a cookie never moves a job
|
||||
back: the worker attaches nothing without a credential, so the private asset is as good as any."""
|
||||
if asset.status not in ("pending", "error"):
|
||||
return asset
|
||||
url = source_url(job.source_kind, job.source_ref)
|
||||
salt = credentials.salt_for_enqueue(db, job.user_id, url)
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
if salt is None or credentials.is_authed_asset(asset, spec, job.user_id, url):
|
||||
return asset
|
||||
target = get_or_create_asset(
|
||||
db, job.source_kind, job.source_ref, formats.format_sig(spec, salt=salt)
|
||||
)
|
||||
populate_from_catalog(db, target)
|
||||
release_asset(db, job) # drop the old hold BEFORE re-pointing — it keys off job.asset_id
|
||||
incr_ref_count(db, target.id)
|
||||
job.asset_id = target.id
|
||||
return target
|
||||
|
||||
|
||||
def cancel(db: Session, job: DownloadJob) -> None:
|
||||
"""Cancel a job unless it's already done/canceled, releasing its asset hold WHILE it still counts
|
||||
as holding — `canceled` ∉ HOLDING, so a later delete won't double-release."""
|
||||
@@ -373,13 +411,31 @@ def delete(db: Session, job: DownloadJob) -> None:
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
"""Rebuild a finished/failed job's file into a FRESH, private asset (the cache sig is salted with
|
||||
the job id + current asset id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
the new asset's own id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
so any public share link (DownloadLink -> job_id) keeps resolving to the newly-downloaded file.
|
||||
The old hold is dropped first (freeing the old file iff this job was its last holder). The caller
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409)."""
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409).
|
||||
|
||||
When the user has a cookie for the link's host, the salt carries it (`credentials.rebuild_salt`),
|
||||
so the worker recognises the rebuild as their private identity and attaches the cookie — a cookie
|
||||
download, or one the user added after a failure, rebuilds with the cookie."""
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}")
|
||||
asset = get_or_create_asset(db, job.source_kind, job.source_ref, sig)
|
||||
cookie = credentials.salt_for_enqueue(
|
||||
db, job.user_id, source_url(job.source_kind, job.source_ref)
|
||||
)
|
||||
# The salt names the row's own id, which exists only once the row does: insert it under a unique
|
||||
# placeholder, then write the real signature in the same transaction (nobody sees the placeholder).
|
||||
asset = MediaAsset(
|
||||
source_kind=job.source_kind,
|
||||
source_ref=job.source_ref,
|
||||
format_sig=f"rebuild:{uuid.uuid4().hex}",
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
asset.format_sig = formats.format_sig(spec, salt=credentials.rebuild_salt(asset.id, cookie))
|
||||
db.flush()
|
||||
populate_from_catalog(db, asset)
|
||||
# Drop the old hold BEFORE re-pointing the job — release_asset keys off the job's current asset.
|
||||
release_asset(db, job)
|
||||
@@ -389,6 +445,8 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
job.progress = 0
|
||||
job.phase = None
|
||||
job.error = None
|
||||
job.error_code = None # a rebuild that fails again gets its own diagnosis, never the old one
|
||||
job.error_detail = None
|
||||
job.speed_bps = None
|
||||
job.eta_s = None
|
||||
# Re-derive the display name from the freshly cleaned title on completion (else an older, un-cleaned
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""The failures the download pipeline raises itself, with the whole story in the message.
|
||||
|
||||
`ffmpeg failed (1): …`, `yt-dlp produced no output file`, a post that shows no video: the message
|
||||
says what happened, and the stack is only the `except` that caught it. A failure line leaves that
|
||||
stack off (`pipeline._failure_stack`) — but ONLY for these. A plain `RuntimeError` is also what
|
||||
Python raises for a bug of ours (a dict changed size during iteration, a generator's
|
||||
StopIteration, `RecursionError`, `NotImplementedError`), and that one's stack is the only lead.
|
||||
"""
|
||||
|
||||
|
||||
class StatedFailure(RuntimeError):
|
||||
"""A failure we raised on purpose, whose message is its full explanation. A `RuntimeError` so
|
||||
every `except RuntimeError` that caught these before still does."""
|
||||
@@ -19,8 +19,11 @@ from datetime import date
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from app import logs
|
||||
from app.downloads import mediakind
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
|
||||
_MAX_TITLE = 120
|
||||
# User-facing (Content-Disposition) names can be longer than an on-disk component.
|
||||
@@ -294,7 +297,13 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
|
||||
if thumb_src and thumb_src.exists():
|
||||
shutil.copyfile(str(thumb_src), str(_sidecar(media, ".jpg")))
|
||||
return True
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# The download itself still completes, so the job reads as done while the library it was
|
||||
# downloaded FOR has no .nfo to read and no poster to show.
|
||||
log.warning(
|
||||
"the sidecars beside a download could not be written",
|
||||
extra=logs.event("download.sidecars.write_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -304,14 +313,23 @@ def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
pass # silent-ok: already gone is the outcome this function wanted
|
||||
except OSError as exc:
|
||||
# The row goes and the bytes stay. Nothing else ever revisits this path, so the disk
|
||||
# quietly keeps a file no account has any way to see, let alone delete.
|
||||
log.info(
|
||||
"a deleted download left a file behind",
|
||||
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
||||
)
|
||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||
root = Path(download_root).resolve()
|
||||
parent = media.parent
|
||||
while parent != root and root in parent.parents:
|
||||
try:
|
||||
parent.rmdir() # only succeeds if empty
|
||||
# silent-ok: this `rmdir` failing is the loop's stop condition — the parent still
|
||||
# holds another download's files, so the prune has gone as far up as it may.
|
||||
except OSError:
|
||||
break
|
||||
parent = parent.parent
|
||||
|
||||
+15
-7
@@ -11,7 +11,6 @@ to be a person telling them nothing arrived.
|
||||
"""
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import smtplib
|
||||
import ssl
|
||||
@@ -21,15 +20,14 @@ from email.utils import formatdate
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import mailtext, sysconfig
|
||||
from app import logs, mailtext, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.notifications import create_notification
|
||||
from app.ratelimit import RateLimiter
|
||||
|
||||
log = logging.getLogger("siftlode.email")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# One admin notice an hour per reason. A relay that is down fails every send, and the inbox must
|
||||
# stay readable; the log keeps the full record either way.
|
||||
_alert_throttle = RateLimiter(max_events=1, window_seconds=3600)
|
||||
@@ -155,7 +153,15 @@ def _send(to: list[str], subject: str, body: str, reply_to: str | None = None) -
|
||||
return False
|
||||
c = _smtp()
|
||||
if not (c["host"] and c["user"] and c["password"]):
|
||||
log.info("SMTP not configured; skipping email %r to %s", subject, recipients)
|
||||
# The count, not the addresses: `redact_text` rewrites every one of them to `[email]`
|
||||
# before either sink sees the line, so the list only ever printed as noise. The subject
|
||||
# stays — the templates are static, and the one that interpolates (`admin_new_request`)
|
||||
# interpolates an address, which the same redactor covers.
|
||||
log.info(
|
||||
"SMTP not configured; skipping email %r to %d recipient(s)",
|
||||
subject,
|
||||
len(recipients), # log-pii-ok: how many, not who
|
||||
)
|
||||
_alert_admins("not_configured", subject, recipients)
|
||||
return False
|
||||
msg = EmailMessage()
|
||||
@@ -176,10 +182,12 @@ def _send(to: list[str], subject: str, body: str, reply_to: str | None = None) -
|
||||
s.starttls(context=ssl.create_default_context())
|
||||
s.login(c["user"], c["password"])
|
||||
s.send_message(msg)
|
||||
log.info("Sent email %r to %s", subject, recipients)
|
||||
# log-pii-ok: how many, not who
|
||||
log.info("Sent email %r to %d recipient(s)", subject, len(recipients))
|
||||
return True
|
||||
except Exception:
|
||||
log.exception("SMTP send failed (%r to %s)", subject, recipients)
|
||||
# log-pii-ok: how many, not who
|
||||
log.exception("SMTP send failed (%r to %d recipient(s))", subject, len(recipients))
|
||||
_alert_admins("send_failed", subject, recipients)
|
||||
return False
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ from app.models import (
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.utils import iso, to_tsquery_str
|
||||
from app.utils import iso, like_escape, to_tsquery_str
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
@@ -272,8 +272,9 @@ def filtered_query(
|
||||
# `unaccent(channels.title) ILIKE '%q%'` straight into the per-video predicate defeated the
|
||||
# search_vector GIN — the planner detoasted every non-matching tsvector to test the `@@`
|
||||
# (~2.6s/search on the prod-sized set); channel_id.in_(...) keeps the GIN as a BitmapOr branch
|
||||
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone.
|
||||
like = func.unaccent(f"%{q}%")
|
||||
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone. The query
|
||||
# is a literal inside the pattern: `_` in `my_channel` is not a wildcard.
|
||||
like = func.unaccent(f"%{like_escape(q)}%")
|
||||
matching_channel_ids = (
|
||||
db.execute(select(Channel.id).where(func.unaccent(Channel.title).ilike(like)))
|
||||
.scalars()
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Bug reports built from debug sessions (E16): the files an issue carries, and where they go —
|
||||
downloaded as a zip, or (S4) sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
and tagged — is the core module `app/debugsession.py`."""
|
||||
@@ -0,0 +1,79 @@
|
||||
"""What an issue from a debug session carries, as files — one definition for the zip export and the
|
||||
Gitea upload, so the two cannot drift apart.
|
||||
|
||||
* `issue.md` — the title and the body the user wrote and reviewed in the composer (rendered there,
|
||||
in their language; the server does not rewrite it);
|
||||
* `client.log` / `server.log` — the session's evidence (`debugsession.bundle`) minus the rows the
|
||||
user took out, one JSON object per line, browser rows and server rows apart. `.log`, not
|
||||
`.jsonl`: Gitea refuses that extension (measured, plans/e16 S0);
|
||||
* the screenshots, in the order they were taken.
|
||||
|
||||
Every row is already redacted twice (at capture, and again when stored), and nothing here adds a
|
||||
field the review list did not show.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logsink
|
||||
from app.models import DebugScreenshot, DebugSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IssueFile:
|
||||
name: str
|
||||
data: bytes
|
||||
mime: str
|
||||
|
||||
|
||||
def title_of(session: DebugSession) -> str:
|
||||
# Not a sentence of ours: the composer always saves a title in the user's language, and a
|
||||
# session exported before it ever saved is named by its date alone, in no language at all.
|
||||
title = ((session.draft or {}).get("title") or "").strip()
|
||||
return title or f"{session.started_at:%Y-%m-%d %H:%M} · {session.id}"
|
||||
|
||||
|
||||
def body_of(session: DebugSession) -> str:
|
||||
return ((session.draft or {}).get("body") or "").strip()
|
||||
|
||||
|
||||
def _lines(rows) -> bytes:
|
||||
return "".join(
|
||||
json.dumps(logsink.serialize(r), ensure_ascii=False, default=str) + "\n" for r in rows
|
||||
).encode()
|
||||
|
||||
|
||||
def files(db: Session, session: DebugSession) -> list[IssueFile]:
|
||||
"""The session's issue as files: the text, the two logs, the screenshots."""
|
||||
rows, _ = debugsession.bundle(db, session, drop_excluded=True)
|
||||
text = f"# {title_of(session)}\n\n{body_of(session)}\n"
|
||||
out = [
|
||||
IssueFile("issue.md", text.encode(), "text/markdown"),
|
||||
IssueFile("client.log", _lines(r for r in rows if r.source == "web"), "text/plain"),
|
||||
IssueFile("server.log", _lines(r for r in rows if r.source != "web"), "text/plain"),
|
||||
]
|
||||
shots = db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
for n, shot in enumerate(shots, start=1):
|
||||
ext = debugsession.IMAGE_TYPES[shot.mime]
|
||||
name = f"{debugsession.SCREENSHOT_PREFIX}{n}.{ext}"
|
||||
out.append(IssueFile(name, shot.data, shot.mime))
|
||||
return out
|
||||
|
||||
|
||||
def zipped(items: list[IssueFile]) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for item in items:
|
||||
# The images are compressed already: deflating them again costs time and wins nothing.
|
||||
method = zipfile.ZIP_STORED if item.mime.startswith("image/") else zipfile.ZIP_DEFLATED
|
||||
z.writestr(item.name, item.data, compress_type=method)
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,275 @@
|
||||
"""The Gitea issue tracker a debug session's issue goes to (E16 S4) — measured against Gitea 1.27.1
|
||||
before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
values, at Teljes what they typed into search — and a public repository would publish it. That is
|
||||
checked on every send, not only in the admin's "test connection", because a repository can be made
|
||||
public after the fact.
|
||||
|
||||
What an issue is made of comes from `bundle.files` (the same files the zip export carries). Sending
|
||||
is three kinds of call, measured:
|
||||
* `POST /repos/{repo}/issues` — the title and the body the user wrote;
|
||||
* `POST /repos/{repo}/issues/{n}/assets?name=…` — ONE file per request (field `attachment`);
|
||||
* `PATCH /repos/{repo}/issues/{n}` — the body again, with the screenshots shown inline once their
|
||||
links exist.
|
||||
A private repository's attachment answers 404 to a browser not signed in to Gitea, which is why the
|
||||
composer's own preview shows our copies, never these links.
|
||||
|
||||
Delivery keeps its progress in the session's draft (`delivery`: the issue number, the files still to
|
||||
upload, the links of those uploaded, whether the body shows the screenshots yet), so a failure
|
||||
half-way keeps the issue that exists and a retry does only what is missing — never a second issue.
|
||||
Every step is checkpointed (the caller's `db.commit`), so what did happen is kept whatever fails
|
||||
next — and the first checkpoint comes before the first call, with the tracker's settings read ABOVE
|
||||
it, so neither a row lock nor a read snapshot nor a pooled connection is held across the calls. The
|
||||
caller claims the session for the send first (`debugsession.claim_send`) and passes the claim's
|
||||
token in, which is what keeps a second send, an autosave or a screenshot out of the way; the claim
|
||||
is renewed by every progress report, and a report whose claim is gone raises rather than writing
|
||||
over the send that took it. `debugsession.not_uploaded` / `images_pending` read that progress.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logs, sysconfig
|
||||
from app.issues import bundle
|
||||
from app.models import DebugSession
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
TIMEOUT_SECONDS = 20.0
|
||||
|
||||
Reason = Literal[
|
||||
"not_configured", # url, repository or token missing
|
||||
"unreachable", # no answer, or a 5xx
|
||||
"bad_token", # 401/403: the token is wrong or lacks write:issue / read:repository
|
||||
"repo_not_found", # 404 on the repository
|
||||
"public_repo", # the repository is public — an issue would publish the reporter's log
|
||||
"issues_disabled", # the repository has its issue tracker switched off
|
||||
"rejected", # any other 4xx (a file type the instance does not allow, …), or an answer that is
|
||||
# not the API's (a redirect — an http:// address behind https —, a page that is not JSON)
|
||||
]
|
||||
|
||||
|
||||
class GiteaError(Exception):
|
||||
def __init__(self, reason: Reason) -> None:
|
||||
super().__init__(reason)
|
||||
self.reason: Reason = reason
|
||||
|
||||
|
||||
def _settings(db: Session) -> tuple[str, str, str]:
|
||||
return (
|
||||
sysconfig.get_str(db, "issues_gitea_url").strip().rstrip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_repo").strip().strip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_token").strip(),
|
||||
)
|
||||
|
||||
|
||||
def target(db: Session) -> dict | None:
|
||||
"""Where a report goes, for the composer to say so — the repository — or None when no tracker is
|
||||
configured (the report can then only be downloaded)."""
|
||||
url, repo, token = _settings(db)
|
||||
return {"repo": repo} if url and repo and token else None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _client(settings: tuple[str, str, str]) -> Iterator[tuple[httpx.Client, str]]:
|
||||
# Takes the settings already read, not the session: reading them HERE opened a transaction one
|
||||
# line after `deliver` had ended one, and held it across the calls below (deep-review round 4).
|
||||
url, repo, token = settings
|
||||
if not (url and repo and token):
|
||||
raise GiteaError("not_configured")
|
||||
with httpx.Client(
|
||||
base_url=f"{url}/api/v1",
|
||||
headers={"Authorization": f"token {token}", "Accept": "application/json"},
|
||||
timeout=TIMEOUT_SECONDS,
|
||||
) as client:
|
||||
yield client, repo
|
||||
|
||||
|
||||
def _call(client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
try:
|
||||
r = client.request(method, path, **kwargs)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning(
|
||||
"the issue tracker did not answer",
|
||||
extra=logs.event("issues.gitea.unreachable", path=path, error=type(exc).__name__),
|
||||
)
|
||||
raise GiteaError("unreachable") from None
|
||||
if r.status_code < 300:
|
||||
return r
|
||||
# The status only: Gitea's message can quote back what was sent — the user's own title.
|
||||
log.warning(
|
||||
"the issue tracker refused a call",
|
||||
extra=logs.event("issues.gitea.refused", path=path, status=r.status_code),
|
||||
)
|
||||
if r.status_code in (401, 403):
|
||||
raise GiteaError("bad_token")
|
||||
if r.status_code == 404:
|
||||
raise GiteaError("repo_not_found")
|
||||
if r.status_code >= 500:
|
||||
raise GiteaError("unreachable")
|
||||
raise GiteaError("rejected")
|
||||
|
||||
|
||||
def _json(r: httpx.Response, *keys: str) -> dict[str, Any]:
|
||||
"""The answer as the API's JSON, holding `keys` — anything else is refused, not a crash, so a
|
||||
step after the issue was created cannot take down the request that holds its record.
|
||||
|
||||
The CREATE call is the one place this cannot keep that record: its answer IS the record. An
|
||||
answer it cannot read there leaves the session unsent, and a retry opens another issue — the
|
||||
price of not being able to tell a refused create from an unreadable one."""
|
||||
try:
|
||||
data = r.json()
|
||||
# silent-ok: not JSON is one of the answers refused (and logged) just below.
|
||||
except ValueError:
|
||||
data = None
|
||||
if not isinstance(data, dict) or any(k not in data for k in keys):
|
||||
log.warning(
|
||||
"the issue tracker answered something that is not its API",
|
||||
extra=logs.event("issues.gitea.unexpected", status=r.status_code),
|
||||
)
|
||||
raise GiteaError("rejected")
|
||||
return data
|
||||
|
||||
|
||||
def _private_repo(client: httpx.Client, repo: str) -> dict:
|
||||
data = _json(_call(client, "GET", f"/repos/{repo}"))
|
||||
if not data.get("private"):
|
||||
raise GiteaError("public_repo")
|
||||
if not data.get("has_issues"):
|
||||
raise GiteaError("issues_disabled")
|
||||
return data
|
||||
|
||||
|
||||
def check(db: Session) -> dict:
|
||||
"""The admin's "test connection": the repository answers, is private and takes issues."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
data = _private_repo(client, repo)
|
||||
return {"repo": data.get("full_name") or repo, "url": data.get("html_url")}
|
||||
|
||||
|
||||
def preview(db: Session, text: str) -> str:
|
||||
"""The body as Gitea itself will render it (its markdown flavour, its sanitiser)."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
r = _call(client, "POST", "/markdown", json={"Text": text, "Mode": "gfm", "Context": repo})
|
||||
return r.text
|
||||
|
||||
|
||||
def deliver(
|
||||
db: Session,
|
||||
session: DebugSession,
|
||||
*,
|
||||
token: str,
|
||||
checkpoint: Callable[[], None],
|
||||
before_create: Callable[[], None] | None = None,
|
||||
on_created: Callable[[], None] | None = None,
|
||||
) -> str:
|
||||
"""Send the session's issue, or finish sending it; returns its URL.
|
||||
|
||||
`token` is the caller's claim on this session (`debugsession.claim_send`). Every progress report
|
||||
renews it and checks the row still holds it, so a send that outlived its lease stops here rather
|
||||
than writing over the one that took over (`debugsession.SendClaimLost`).
|
||||
|
||||
`checkpoint` persists what has happened so far and is called after every step (the caller's
|
||||
commit — this module does not decide when a transaction ends). `before_create` runs right before
|
||||
the tracker is asked to open a NEW issue, after the checks that can refuse it without asking,
|
||||
and may raise to stop it; `on_created` runs right after one was opened, in time to be committed
|
||||
by the SAME checkpoint that first writes `issue_url` — anything the caller must not be able to
|
||||
lose separately from the link belongs there.
|
||||
"""
|
||||
state = debugsession.delivery(session)
|
||||
files = bundle.files(db, session)
|
||||
attachments = [f for f in files if f.name != "issue.md"]
|
||||
settings = _settings(db)
|
||||
# Reading the bundle and the tracker's settings opened a transaction; end it before the first
|
||||
# call, so nothing holds a snapshot (and an idle connection) across minutes of tracker I/O.
|
||||
# The settings are read HERE and handed to `_client`, not read inside it: reading them there
|
||||
# was a SELECT one line after this commit, which reopened the transaction and held it — idle,
|
||||
# with its snapshot pinned — across the repository check and the create (deep-review round 4
|
||||
# measured it in pg_stat_activity). A comment saying otherwise sat right here.
|
||||
checkpoint()
|
||||
with _client(settings) as (client, repo):
|
||||
_private_repo(client, repo)
|
||||
if "number" not in state:
|
||||
if before_create is not None:
|
||||
before_create()
|
||||
created = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
json={"title": bundle.title_of(session), "body": bundle.body_of(session)},
|
||||
),
|
||||
"number",
|
||||
"html_url",
|
||||
)
|
||||
# Built ON the state read at the start so nothing already in it is dropped. The claim
|
||||
# is NOT among those things, whatever this comment used to say: `set_delivery` strips
|
||||
# any `sending` passed to it and stamps the row's own from `token` (round 4 — removing
|
||||
# the spread left the whole suite green, because the claim never travelled here).
|
||||
state = {
|
||||
**state,
|
||||
"number": created["number"],
|
||||
"pending": [f.name for f in attachments],
|
||||
"links": {},
|
||||
}
|
||||
session.issue_url = created["html_url"]
|
||||
try:
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
except debugsession.SendClaimLost:
|
||||
# The one step where losing the claim costs something irreversible: the issue is
|
||||
# already open on the tracker and this is what would have recorded its link. We
|
||||
# still must not write — another send owns the row and is opening its own issue —
|
||||
# so say which issue was left behind, because nothing else will.
|
||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||
# nothing else; the issue number below is the whole news.
|
||||
log.error( # noqa: TRY400
|
||||
"an issue was created by a send that had already lost its claim",
|
||||
extra=logs.event(
|
||||
"issues.gitea.orphaned",
|
||||
number=created["number"],
|
||||
session_id=session.id,
|
||||
),
|
||||
)
|
||||
raise
|
||||
if on_created is not None:
|
||||
on_created()
|
||||
checkpoint()
|
||||
log.info(
|
||||
"issue created",
|
||||
extra=logs.event("issues.gitea.created", number=created["number"]),
|
||||
)
|
||||
number = state["number"]
|
||||
for item in attachments:
|
||||
if item.name not in state["pending"]:
|
||||
continue
|
||||
asset = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues/{number}/assets",
|
||||
params={"name": item.name},
|
||||
files={"attachment": (item.name, item.data, item.mime)},
|
||||
),
|
||||
"browser_download_url",
|
||||
)
|
||||
state = {
|
||||
**state,
|
||||
"pending": [n for n in state["pending"] if n != item.name],
|
||||
"links": {**state["links"], item.name: asset["browser_download_url"]},
|
||||
}
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
checkpoint()
|
||||
shots = [n for n in state["links"] if n.startswith(debugsession.SCREENSHOT_PREFIX)]
|
||||
if shots and not state.get("images_shown"):
|
||||
images = "\n".join(f"" for n in sorted(shots))
|
||||
body = f"{bundle.body_of(session)}\n\n---\n\n{images}"
|
||||
_call(client, "PATCH", f"/repos/{repo}/issues/{number}", json={"body": body})
|
||||
debugsession.set_delivery(db, session, {**state, "images_shown": True}, token)
|
||||
checkpoint()
|
||||
return session.issue_url or ""
|
||||
@@ -0,0 +1,839 @@
|
||||
"""Structured logging for every Siftlode process — the ONE place logging is configured.
|
||||
|
||||
Before this module a single deployment printed FOUR different log formats: uvicorn's
|
||||
`log_config.json` for the API, a hand-rolled StreamHandler in `main.py` for everything that did not
|
||||
start under uvicorn (tests, scripts), `logging.basicConfig()` in the worker and in the download
|
||||
child (no timestamp at all), and alembic's `fileConfig`. None of them carried a correlation id, and
|
||||
uvicorn's access log printed the query string of every request (`?account=1`, `?source=feed`) — so
|
||||
the logs were both unjoinable and leaky. `configure_logging()` replaces all four.
|
||||
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what E22.6's admin console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
|
||||
log = logs.get_logger(__name__) # app/downloads/pipeline.py -> siftlode.downloads.pipeline
|
||||
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
||||
|
||||
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
||||
what makes a line queryable later, so add one wherever the line matters.
|
||||
|
||||
REDACTION IS APPLIED AT THE HANDLER, not at the call site: every message, every attr value and
|
||||
every traceback passes `redact()` on the way out. That is the whole point — a secret cannot be
|
||||
leaked by forgetting to redact it at one of ~125 call sites, and a new call site is safe by
|
||||
default. Never logged, at any level: passwords, tokens (session, reset/verify, setup, share,
|
||||
OAuth), cookie blobs, secret config values, and email addresses.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import logging.config
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from app.config import settings
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
|
||||
# One level below DEBUG for the "Teljes / TRACE" diagnostics level (search strings, redacted request
|
||||
# bodies). Registered on import so `logging.getLogger(...).log(logs.TRACE, …)` and a `LOG_LEVEL=TRACE`
|
||||
# both work in every process, including one that never calls configure_logging (a test).
|
||||
TRACE = 5
|
||||
logging.addLevelName(TRACE, "TRACE")
|
||||
|
||||
# OpenTelemetry severity numbers for our level names. Kept beside the level because the number is
|
||||
# what a log backend sorts and filters on; the name is for humans reading a terminal.
|
||||
SEVERITY: dict[str, int] = {
|
||||
"TRACE": 1,
|
||||
"DEBUG": 5,
|
||||
"INFO": 9,
|
||||
"WARNING": 13,
|
||||
"ERROR": 17,
|
||||
"CRITICAL": 21,
|
||||
}
|
||||
|
||||
# Which process this is. Set by configure_logging; every line carries it so one Loki/DB query can
|
||||
# separate the API's view of a job from the worker's.
|
||||
_source = "api"
|
||||
|
||||
|
||||
# --- correlation context -------------------------------------------------------------------
|
||||
# contextvars, like quota's actor attribution (app/quota.py): set at an entry point, read by the
|
||||
# formatter, never threaded through call signatures. A child task copies the context at creation,
|
||||
# so anything set before `await call_next(...)` is visible to the whole request.
|
||||
_request_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_request_id", default=None
|
||||
)
|
||||
_user_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_user_id", default=None)
|
||||
_debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_debug_session", default=None
|
||||
)
|
||||
_job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None)
|
||||
|
||||
_CONTEXT_VARS = {
|
||||
"request_id": _request_id,
|
||||
"user_id": _user_id,
|
||||
"debug_session_id": _debug_session,
|
||||
"job_id": _job_id,
|
||||
}
|
||||
|
||||
|
||||
class _RequestScope:
|
||||
"""What the auth dependency learns about a request, for every line logged after it.
|
||||
|
||||
A MUTABLE object behind a contextvar, not plain contextvars, because of how a request actually
|
||||
runs: FastAPI resolves a sync dependency (`auth.current_user`) in a threadpool on a COPY of the
|
||||
context, so a `ContextVar.set()` there is gone by the time the handler runs — measured: that is
|
||||
how the account never reached a handler's own lines. `log_requests` installs one scope per
|
||||
request; every copied context shares the same object, so what the dependency writes is seen by
|
||||
the handler and by the request line itself."""
|
||||
|
||||
__slots__ = ("user_id", "level", "debug_session_id")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
# The user's own open debug session this request named (E16), else None.
|
||||
self.debug_session_id: str | None = None
|
||||
|
||||
|
||||
_scope: contextvars.ContextVar[_RequestScope | None] = contextvars.ContextVar(
|
||||
"log_request_scope", default=None
|
||||
)
|
||||
|
||||
|
||||
def set_user(
|
||||
user_id: int | None,
|
||||
level: str = "INFO",
|
||||
until: datetime | None = None,
|
||||
debug_session: str | None = None,
|
||||
) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware.
|
||||
|
||||
`debug_session` tags every line with the user's open debug session (E16). The caller has
|
||||
already checked it is theirs and open; a session's level arrives here as `level`/`until`.
|
||||
|
||||
The root logger's floor is tracked per elevation: a request without the session's header (an
|
||||
`<img>`, a `<video>` — the browser sends those itself) carries only the user's own level, and
|
||||
must not lower the floor a concurrent request of the session still needs."""
|
||||
_user_id.set(user_id)
|
||||
scope = _scope.get()
|
||||
if scope is None: # outside log_requests (a script, a unit test): this context only
|
||||
scope = _RequestScope()
|
||||
_scope.set(scope)
|
||||
elevated = level != "INFO" and until is not None and until > datetime.now(timezone.utc)
|
||||
scope.user_id = user_id
|
||||
scope.debug_session_id = debug_session
|
||||
scope.level = resolve_level(level) if elevated else None
|
||||
if user_id is not None:
|
||||
key = user_id if debug_session is None else (user_id, debug_session)
|
||||
_track_elevation(key, scope.level, until)
|
||||
|
||||
|
||||
def set_job(job_id: int | None) -> None:
|
||||
"""Attribute this process's log lines to a download job — for the download CHILD, whose whole
|
||||
life is one job (the worker uses `bind()` per job instead, since it juggles several)."""
|
||||
_job_id.set(job_id)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def bind(**fields: Any):
|
||||
"""Bind correlation fields for the duration of a block (a worker job, a scheduler run):
|
||||
|
||||
with logs.bind(job_id=job.id):
|
||||
...
|
||||
|
||||
Unknown field names are ignored rather than raising — a caller binding a field this module does
|
||||
not know about is a typo, not a reason to take down a download."""
|
||||
tokens = [(_CONTEXT_VARS[k].set(v)) for k, v in fields.items() if k in _CONTEXT_VARS]
|
||||
keys = [k for k in fields if k in _CONTEXT_VARS]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for key, token in zip(keys, tokens):
|
||||
_CONTEXT_VARS[key].reset(token)
|
||||
|
||||
|
||||
def event(code: str, /, **attrs: Any) -> dict[str, Any]:
|
||||
"""`extra=` payload for a log call: a stable dotted event code plus structured attributes.
|
||||
|
||||
Returns a plain dict so it composes with stdlib logging and costs nothing when the level is off
|
||||
(the dict is built either way; keep the attrs cheap — no DB reads inside the call).
|
||||
|
||||
`code` is positional-only so that no attribute name is reserved: `event("x", code=...)` means
|
||||
the attribute, not this parameter. Without the `/` it was a TypeError, and `code` is exactly
|
||||
the word a caller reaches for when the thing that failed has one of its own."""
|
||||
return {"event": code, "attrs": attrs}
|
||||
|
||||
|
||||
# --- redaction -----------------------------------------------------------------------------
|
||||
# Substring matching, not exact keys: the same secret travels as `password`, `current_password`,
|
||||
# `new_password`, `smtp_password`, `plex_server_token`, `access_token`, … Matching on the PART is
|
||||
# what makes a key we have never seen safe by default.
|
||||
_SECRET_KEY_PARTS = (
|
||||
"password",
|
||||
"passphrase",
|
||||
"token",
|
||||
"secret",
|
||||
"cookie",
|
||||
"authorization",
|
||||
"auth_header",
|
||||
"ciphertext",
|
||||
"private_key",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"credential",
|
||||
"session_key",
|
||||
)
|
||||
|
||||
# Query/fragment parameters that carry a credential in a URL we log (OAuth `code`/`state`, our own
|
||||
# `?token=` setup and share links, `#verify=` / `#reset=` fragments).
|
||||
_SECRET_PARAM_NAMES = (
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"id_token",
|
||||
"token",
|
||||
"code",
|
||||
"state",
|
||||
"key",
|
||||
"secret",
|
||||
"password",
|
||||
"pwd",
|
||||
"sig",
|
||||
"signature",
|
||||
"verify",
|
||||
"reset",
|
||||
"setup",
|
||||
)
|
||||
_SECRET_PARAM_RE = re.compile(
|
||||
r"\b((?:" + "|".join(_SECRET_PARAM_NAMES) + r")=)([^&\s\"']+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Deliberately loose: over-matching redacts a non-address that looks like one (harmless), while
|
||||
# under-matching puts a real address in a log we attach to an issue (not harmless).
|
||||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
REDACTED = "[redacted]"
|
||||
# `extra={PERSIST: False}` keeps a line out of the `log_events` table while stdout still gets it —
|
||||
# how `log_requests` caps anonymous traffic per client (app/logsink.py reads it).
|
||||
PERSIST = "persist"
|
||||
REDACTED_EMAIL = "[email]"
|
||||
|
||||
|
||||
def redact_text(text: str) -> str:
|
||||
"""Scrub a free-text string: credential-bearing URL params and email addresses."""
|
||||
scrubbed = _SECRET_PARAM_RE.sub(lambda m: f"{m.group(1)}{REDACTED}", text)
|
||||
return _EMAIL_RE.sub(REDACTED_EMAIL, scrubbed)
|
||||
|
||||
|
||||
def redact_query(query: str) -> str:
|
||||
"""Scrub a raw query string. DECODED first: the text rules match what was typed, and a browser
|
||||
sends it percent-encoded — `someone%40example.com` is an address `_EMAIL_RE` cannot see. Each
|
||||
parameter also goes through the key rules, so `?new_password=` goes as `redact` would drop it."""
|
||||
pairs = parse_qsl(query, keep_blank_values=True)
|
||||
return "&".join(f"{key}={_redact_param(key, value)}" for key, value in pairs)
|
||||
|
||||
|
||||
def _redact_param(key: str, value: str) -> str:
|
||||
"""One decoded `key=value` pair. The parameter-NAME rule is applied to the key here rather than
|
||||
left to `_SECRET_PARAM_RE` over the rebuilt pair: that pattern was written for the raw,
|
||||
percent-encoded query, where a value holds no literal space, `&` or quote, and its value class
|
||||
stops at the first one. On decoded text `?state=a%20b` would keep its tail and `?code=%20abc`
|
||||
would not match at all — the whole credential straight into the line."""
|
||||
if key.lower() in _SECRET_PARAM_NAMES:
|
||||
return REDACTED
|
||||
return str(redact(value, key=key))
|
||||
|
||||
|
||||
def redact(value: Any, *, key: str = "") -> Any:
|
||||
"""Recursively scrub a structure destined for the log.
|
||||
|
||||
A value whose KEY names a secret is dropped whole (we never log a prefix of a token — a prefix
|
||||
is still a leak and still useless). Everything else is scrubbed as text. Depth is bounded by
|
||||
the structures we actually log (attrs dicts), and non-JSON types fall through to `str`."""
|
||||
lowered = key.lower()
|
||||
if key and any(part in lowered for part in _SECRET_KEY_PARTS):
|
||||
return REDACTED
|
||||
if isinstance(value, dict):
|
||||
return {k: redact(v, key=str(k)) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [redact(v, key=key) for v in value]
|
||||
if isinstance(value, str):
|
||||
return redact_text(value)
|
||||
return value
|
||||
|
||||
|
||||
# --- formatters ----------------------------------------------------------------------------
|
||||
_SCRUBBED_STACK = "scrubbed_stack"
|
||||
|
||||
|
||||
def scrubbed_stack(exc: BaseException, scrub: Callable[[BaseException], str]) -> dict[str, str]:
|
||||
"""`extra=` fragment: `exc`'s traceback, rendered where an `exc_info` stack goes, with the
|
||||
message of every exception in it put through `scrub`.
|
||||
|
||||
For a failure whose message can name someone's data (a download's URL, a file path, a video
|
||||
id). The frames name code and are kept whole; the text at the foot of each link is the data.
|
||||
A plain `exc_info` renders the chain too — the `__cause__`/`__context__` a DB error raised
|
||||
inside `except DownloadError` carries — so scrubbing only the line's own message is not
|
||||
enough. Merge it into the call's `extra`: `{**event(...), **scrubbed_stack(...)}`."""
|
||||
links: list[BaseException] = []
|
||||
link: BaseException | None = exc
|
||||
while link is not None and all(link is not seen for seen in links):
|
||||
links.append(link)
|
||||
link = link.__cause__ or (None if link.__suppress_context__ else link.__context__)
|
||||
parts: list[str] = []
|
||||
for i in range(len(links) - 1, -1, -1): # the oldest first, as Python prints a chain
|
||||
link = links[i]
|
||||
kind = type(link)
|
||||
name = kind.__qualname__
|
||||
if kind.__module__ not in ("builtins", "__main__"):
|
||||
name = f"{kind.__module__}.{name}"
|
||||
try:
|
||||
message = scrub(link)
|
||||
# silent-ok: the message is what could not be rendered, and the placeholder says so — the
|
||||
# caller is in the middle of reporting one failure and must not get a second from here.
|
||||
except Exception: # noqa: BLE001 — a `__str__` that raises, of any shape
|
||||
message = "<unprintable>"
|
||||
frames = "".join(traceback.format_tb(link.__traceback__))
|
||||
head = f"Traceback (most recent call last):\n{frames}" if frames else ""
|
||||
parts.append(f"{head}{name}: {message}" if message else f"{head}{name}")
|
||||
if i:
|
||||
parts.append(
|
||||
"\nThe above exception was the direct cause of the following exception:\n"
|
||||
if links[i - 1].__cause__ is link
|
||||
else "\nDuring handling of the above exception, another exception occurred:\n"
|
||||
)
|
||||
return {_SCRUBBED_STACK: "\n".join(parts)}
|
||||
|
||||
|
||||
# Attributes every LogRecord carries; anything else a caller passed via `extra=` is a structured
|
||||
# attribute and is merged into `attrs` (so `extra={"job_id": 7}` works as well as `event(...)`).
|
||||
_RECORD_STANDARD = frozenset(
|
||||
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
|
||||
| {"message", "asctime", "taskName", "event", "attrs", PERSIST, "elevated", _SCRUBBED_STACK}
|
||||
)
|
||||
|
||||
|
||||
class ContextFilter(logging.Filter):
|
||||
"""Copies the correlation contextvars onto the record. A Filter (not the formatter) so the
|
||||
values are captured at emit time, before a QueueHandler can move the record to another
|
||||
thread — the DB sink in S3 depends on that."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
for name, var in _CONTEXT_VARS.items():
|
||||
value = var.get()
|
||||
if value is not None and not hasattr(record, name):
|
||||
setattr(record, name, value)
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.user_id is not None and not hasattr(record, "user_id"):
|
||||
record.user_id = scope.user_id
|
||||
if (
|
||||
scope is not None
|
||||
and scope.debug_session_id is not None
|
||||
and not hasattr(record, "debug_session_id")
|
||||
):
|
||||
record.debug_session_id = scope.debug_session_id
|
||||
return True
|
||||
|
||||
|
||||
class LevelGate(logging.Filter):
|
||||
"""The configured level, enforced at the HANDLER, with one exception: the lines of a request
|
||||
whose user raised their diagnostics level (E22 S5).
|
||||
|
||||
The root logger's level cannot do this alone: it is one number for the whole process, and the
|
||||
point is to log ONE user's requests in detail while everyone else's stay at the base level. So
|
||||
the root is lowered to the lowest level any active elevation needs (`_track_elevation`), and
|
||||
this gate drops what only an elevation let through unless the line belongs to that elevated
|
||||
request. It also marks such a line `elevated`, which is what makes the `log_events` sink keep
|
||||
every line of that request, not just its warnings."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.level is not None:
|
||||
record.elevated = True
|
||||
return record.levelno >= min(scope.level, _base_level)
|
||||
return record.levelno >= _base_level
|
||||
|
||||
|
||||
def _record_attrs(record: logging.LogRecord) -> dict[str, Any]:
|
||||
extra = {k: v for k, v in vars(record).items() if k not in _RECORD_STANDARD}
|
||||
attrs = dict(getattr(record, "attrs", None) or {})
|
||||
attrs.update({k: v for k, v in extra.items() if k not in _CONTEXT_VARS})
|
||||
return attrs
|
||||
|
||||
|
||||
def _safe_message(record: logging.LogRecord) -> str:
|
||||
"""`record.getMessage()` raises if the args don't match the format string. A broken log call
|
||||
must not take down the caller, so the failure is reported IN the line instead."""
|
||||
try:
|
||||
return record.getMessage()
|
||||
# silent-ok: the failure is reported IN the returned line. Logging it would re-enter this same
|
||||
# formatter with a record built out of the one that just failed.
|
||||
except Exception as exc: # noqa: BLE001 — any formatting error, reported not raised
|
||||
return f"{record.msg!r} (log formatting failed: {exc})"
|
||||
|
||||
|
||||
def record_fields(record: logging.LogRecord) -> dict[str, Any]:
|
||||
"""The event as structured fields, REDACTED — the one definition both the stdout line and the
|
||||
`log_events` row are built from, so the two sinks can never disagree about what is safe to keep.
|
||||
Absent optional fields are left out; `ts` is an aware datetime."""
|
||||
fields: dict[str, Any] = {
|
||||
"ts": datetime.fromtimestamp(record.created, timezone.utc),
|
||||
"level": record.levelname,
|
||||
"sev": SEVERITY.get(record.levelname, 9),
|
||||
"source": _source,
|
||||
"logger": record.name,
|
||||
"msg": redact_text(_safe_message(record)),
|
||||
}
|
||||
code = getattr(record, "event", None)
|
||||
if code:
|
||||
fields["event"] = code
|
||||
for name in _CONTEXT_VARS:
|
||||
value = getattr(record, name, None)
|
||||
if value is not None:
|
||||
fields[name] = value
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
fields["attrs"] = redact(attrs)
|
||||
if record.exc_info:
|
||||
fields["exc"] = redact_text(logging.Formatter().formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
fields["exc"] = redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
elif record.exc_text:
|
||||
# A QueueHandler-style hand-off formats the traceback into exc_text and drops exc_info.
|
||||
fields["exc"] = redact_text(record.exc_text)
|
||||
fields["version"] = settings.app_version
|
||||
return fields
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line — the shape Loki and the `log_events` table both ingest."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = record_fields(record)
|
||||
payload["ts"] = payload["ts"].isoformat(timespec="milliseconds")
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
class TextFormatter(logging.Formatter):
|
||||
"""Human-readable single line for a terminal (`LOG_FORMAT=text`): the same content, ordered for
|
||||
reading. Redaction applies here too — a dev log is still a log someone pastes into an issue."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
stamp = datetime.fromtimestamp(record.created, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
|
||||
head = f"{stamp} {record.levelname:<5} [{record.name}]"
|
||||
rid = getattr(record, "request_id", None)
|
||||
if rid:
|
||||
head += f" {rid}"
|
||||
code = getattr(record, "event", None)
|
||||
line = f"{head} {code + ' — ' if code else ''}{redact_text(_safe_message(record))}"
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
line += f" {json.dumps(redact(attrs), ensure_ascii=False, default=str)}"
|
||||
if record.exc_info:
|
||||
line += "\n" + redact_text(self.formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
line += "\n" + redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
return line
|
||||
|
||||
|
||||
# --- configuration -------------------------------------------------------------------------
|
||||
# Third-party loggers that are noisy at our own level. Pinned explicitly so the diagnostics knob can
|
||||
# lower the ROOT level to DEBUG/TRACE without drowning the output in SQL statements and HTTP chatter
|
||||
# — raising THESE is a deliberate, separate act (E22.6's console).
|
||||
_THIRD_PARTY_LEVELS = {
|
||||
"sqlalchemy": "WARNING",
|
||||
"sqlalchemy.engine": "WARNING",
|
||||
"apscheduler": "WARNING",
|
||||
"urllib3": "WARNING",
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"multipart": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
# it is one line per request with the query string in it, replaced by `log_requests` below.
|
||||
"uvicorn": "INFO",
|
||||
"uvicorn.error": "INFO",
|
||||
"uvicorn.access": "CRITICAL",
|
||||
}
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""The logger for one module: `logs.get_logger(__name__)` in `app/plex/sync.py` is
|
||||
`siftlode.plex.sync`.
|
||||
|
||||
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||
module hand-rolls one again.
|
||||
|
||||
`__main__` is resolved to the module it really is. The worker runs as `python -m app.worker`
|
||||
and spawns `python -m app.downloads.runner`, so in the two long-running processes `__name__`
|
||||
is `__main__` — taken at its word, both would log as the bare root. `-m` records the real name
|
||||
on the module's spec; a plain script has none and stays the root."""
|
||||
if name == "__main__":
|
||||
spec = getattr(sys.modules.get("__main__"), "__spec__", None)
|
||||
name = getattr(spec, "name", None) or name
|
||||
stem = name.removeprefix("app.") if name.startswith("app.") else ""
|
||||
# logger-name-ok: this call IS the derivation
|
||||
return logging.getLogger(f"siftlode.{stem}" if stem else "siftlode")
|
||||
|
||||
|
||||
def resolve_level(name: str) -> int:
|
||||
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
||||
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
||||
value = logging.getLevelName(name)
|
||||
if not isinstance(value, int):
|
||||
raise ValueError(f"unknown log level {name!r}")
|
||||
return value
|
||||
|
||||
|
||||
def configure_logging(source: str) -> None:
|
||||
"""Configure logging for THIS process. Call once, first thing, in every entry point:
|
||||
`main.py` (api), `worker.main`, `downloads/runner.main`, `alembic/env.py`.
|
||||
|
||||
`source` names the process in every line. The level starts at `LOG_LEVEL` (the bootstrap default);
|
||||
the runtime admin knob goes through `set_level()` instead, so changing it needs no restart.
|
||||
|
||||
Our loggers deliberately keep `propagate=True` and own no handler: everything reaches the single
|
||||
root StreamHandler. That is what keeps `caplog` working in tests, and it means a new logger name
|
||||
is configured the moment it is created."""
|
||||
global _source
|
||||
_source = source
|
||||
formatter = settings.log_format
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
# Never disable loggers created at import time (module-level `getLogger` calls run
|
||||
# before this function does).
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"json": {"()": f"{__name__}.JsonFormatter"},
|
||||
"text": {"()": f"{__name__}.TextFormatter"},
|
||||
},
|
||||
"filters": {
|
||||
"context": {"()": f"{__name__}.ContextFilter"},
|
||||
"gate": {"()": f"{__name__}.LevelGate"},
|
||||
},
|
||||
"handlers": {
|
||||
"stdout": {
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": sys.stdout,
|
||||
"formatter": formatter,
|
||||
"filters": ["context", "gate"],
|
||||
}
|
||||
},
|
||||
"root": {"level": resolve_level(settings.log_level), "handlers": ["stdout"]},
|
||||
"loggers": {name: {"level": lvl} for name, lvl in _THIRD_PARTY_LEVELS.items()},
|
||||
}
|
||||
)
|
||||
set_level(settings.log_level)
|
||||
|
||||
|
||||
def set_level(name: str) -> None:
|
||||
"""Change the running process's level (the `log_level` admin knob). Third-party loggers keep
|
||||
their pinned levels, so turning on DEBUG shows OUR detail, not SQLAlchemy's."""
|
||||
global _base_level
|
||||
_base_level = resolve_level(name)
|
||||
_apply_root_level(force=True)
|
||||
|
||||
|
||||
def current_level() -> str:
|
||||
"""The configured level — not the root logger's, which an active elevation may have lowered."""
|
||||
return logging.getLevelName(_base_level or logging.getLogger().getEffectiveLevel())
|
||||
|
||||
|
||||
# --- per-user elevation (E22 S5) -----------------------------------------------------------
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# → (level number, expiry) for every elevation this process has seen and not yet expired: a user's
|
||||
# own, by user id, and a debug session's (E16), by (user id, session id) — its requests alone carry
|
||||
# its level, so it is tracked apart from the user's own requests.
|
||||
_ElevationKey = int | tuple[int, str]
|
||||
_elevations: dict[_ElevationKey, tuple[int, datetime]] = {}
|
||||
_elevations_lock = threading.Lock()
|
||||
_root_applied: int | None = None
|
||||
|
||||
|
||||
def _track_elevation(key: _ElevationKey, level: int | None, until: datetime | None) -> None:
|
||||
with _elevations_lock:
|
||||
if level is None or until is None:
|
||||
_elevations.pop(key, None)
|
||||
else:
|
||||
_elevations[key] = (level, until)
|
||||
active = bool(_elevations)
|
||||
if active or _root_applied != _base_level:
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def _apply_root_level(force: bool = False) -> None:
|
||||
"""Root level = the lowest of the base and every unexpired elevation (expired ones are pruned
|
||||
here). `setLevel` clears every logger's level cache, so it runs only when the number changes."""
|
||||
global _root_applied
|
||||
if _base_level == logging.NOTSET:
|
||||
return # not configured: the root keeps whatever the process set up
|
||||
now = datetime.now(timezone.utc)
|
||||
# The level is SET under the lock too, not just computed: two threads that each computed a
|
||||
# different answer would otherwise apply them in either order, and the older one could win —
|
||||
# the root back at the base while an elevation is still active.
|
||||
with _elevations_lock:
|
||||
for key in [k for k, (_, until) in _elevations.items() if until <= now]:
|
||||
del _elevations[key]
|
||||
lowest = min([_base_level, *(lvl for lvl, _ in _elevations.values())])
|
||||
if force or lowest != _root_applied:
|
||||
logging.getLogger().setLevel(lowest)
|
||||
_root_applied = lowest
|
||||
|
||||
|
||||
# --- erased accounts ------------------------------------------------------------------------
|
||||
# `auth.purge_user` deletes an account's `log_events` rows, but a line about that account can still
|
||||
# be on its way: the erasing request's own access line is logged AFTER the handler committed, a queued
|
||||
# row waits for the sink's writer, a background task logs later. Those rows are stored without the
|
||||
# account (`logsink` asks here). Remembered for an hour in this process — a line later than that has
|
||||
# no request left to come from, and the retention job is the backstop for any other process.
|
||||
ERASED_MEMORY_SECONDS = 3600.0
|
||||
_erased: dict[int, float] = {}
|
||||
_erased_lock = threading.Lock()
|
||||
# Held by whoever checks rows against `is_erased` AND writes them (`erasure_guard`). `forget_user`
|
||||
# takes it too, so an erasure waits for a write that already passed the check to commit — and the
|
||||
# purge's DELETE, which runs after `forget_user`, then removes those rows as well. Without it a row
|
||||
# checked a moment before the erasure could be inserted a moment after the DELETE.
|
||||
_erasure_guard = threading.Lock()
|
||||
|
||||
|
||||
def erasure_guard() -> threading.Lock:
|
||||
"""Hold this around "drop erased accounts from these rows, insert them, commit"."""
|
||||
return _erasure_guard
|
||||
|
||||
|
||||
def forget_user(user_id: int) -> None:
|
||||
"""Mark `user_id` erased: no line logged about it from now on is stored under its id. Call BEFORE
|
||||
deleting its rows (see `_erasure_guard`)."""
|
||||
now = time.monotonic()
|
||||
with _erasure_guard, _erased_lock:
|
||||
for stale in [u for u, at in _erased.items() if now - at > ERASED_MEMORY_SECONDS]:
|
||||
del _erased[stale]
|
||||
_erased[user_id] = now
|
||||
with _elevations_lock:
|
||||
for key in [
|
||||
k for k in _elevations if k == user_id or (isinstance(k, tuple) and k[0] == user_id)
|
||||
]:
|
||||
del _elevations[key]
|
||||
# That pop can have removed the lowest elevation, and the root level is the lowest of the base
|
||||
# and every unexpired one. Recomputed here rather than left to whichever later request happens
|
||||
# to call `_track_elevation` — the invariant `_apply_root_level` documents holds at every pop.
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def end_session_elevation(session_id: str) -> None:
|
||||
"""A debug session was stopped: its elevation ends now, not at its cap."""
|
||||
with _elevations_lock:
|
||||
for key in [k for k in _elevations if isinstance(k, tuple) and k[1] == session_id]:
|
||||
del _elevations[key]
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def is_erased(user_id: int | None) -> bool:
|
||||
if user_id is None:
|
||||
return False
|
||||
with _erased_lock:
|
||||
at = _erased.get(user_id)
|
||||
return at is not None and time.monotonic() - at <= ERASED_MEMORY_SECONDS
|
||||
|
||||
|
||||
def current_source() -> str:
|
||||
"""The process name `configure_logging` was given (api, worker, runner, alembic)."""
|
||||
return _source
|
||||
|
||||
|
||||
# --- request logging -----------------------------------------------------------------------
|
||||
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
|
||||
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
|
||||
# write arbitrary text into the log.
|
||||
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
|
||||
# Paths whose success is logged at DEBUG rather than INFO: the container healthcheck runs every 15s
|
||||
# (5.7k lines/day saying nothing), hashed static assets are served by the hundred per page load, and
|
||||
# the admin log console reads the stored log — its live tail every few seconds, so at INFO the log
|
||||
# would fill with the console watching itself. A FAILURE on any of them still logs at WARNING/ERROR
|
||||
# — this lowers the floor, it does not hide.
|
||||
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/", "/api/admin/logs")
|
||||
|
||||
# The reads the frontend POLLS (a `useLiveQuery`, the `me` heartbeat, the sync status, the subtitle
|
||||
# progress), by route template, GET only — logged at DEBUG like the prefixes above. One open tab asked
|
||||
# for the two unread counts, `/api/me` and the download index every 30 s, and at INFO those were ~95%
|
||||
# of the stored log (measured on prod 2026-09-19): 11k lines a day per tab burying every line someone
|
||||
# caused. The same route answered with a failure still logs at WARNING/ERROR.
|
||||
_POLLED_ROUTES = frozenset(
|
||||
{
|
||||
"/api/me",
|
||||
"/api/me/notifications/unread_count",
|
||||
"/api/messages/unread_count",
|
||||
"/api/downloads",
|
||||
"/api/downloads/index",
|
||||
"/api/downloads/usage",
|
||||
"/api/sync/my-status",
|
||||
"/api/admin/scheduler",
|
||||
"/api/admin/downloads",
|
||||
"/api/admin/downloads/storage",
|
||||
"/api/plex/subtitle/{rating_key}/{sub_ord}/status",
|
||||
}
|
||||
)
|
||||
|
||||
# Not `siftlode.logs`: this is the replacement for uvicorn's ACCESS log, and `siftlode.http` is
|
||||
# what an operator pins a level on when they want request lines and nothing else. The module it
|
||||
# happens to live in is not what the name is about.
|
||||
_request_log = logging.getLogger("siftlode.http") # logger-name-ok: named for the traffic
|
||||
|
||||
|
||||
async def log_requests(request, call_next):
|
||||
"""The single access-log line, replacing uvicorn's.
|
||||
|
||||
What changes versus `uvicorn.access`: the ROUTE TEMPLATE instead of the concrete path (so
|
||||
`/api/videos/{video_id}` groups), never the query string (it carried `?account=`, and at TRACE
|
||||
the redacted query belongs in attrs, not in the URL), a request id echoed back in
|
||||
`X-Request-Id` (with the template in `X-Route`), the duration, and the account — read from `request.state`, which the auth
|
||||
dependency fills, because this middleware sits outside both the session middleware and the
|
||||
downstream task's context."""
|
||||
incoming = request.headers.get("x-request-id", "")
|
||||
rid = incoming if _REQUEST_ID_RE.match(incoming) else uuid.uuid4().hex[:16]
|
||||
token = _request_id.set(rid)
|
||||
scope = _RequestScope()
|
||||
scope_token = _scope.set(scope)
|
||||
request.state.request_id = rid
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# The unhandled-exception path: FastAPI's ServerErrorMiddleware re-raises after its own
|
||||
# 500, so without this the ONE request that most needs a log line is the one with no
|
||||
# access-log entry at all.
|
||||
_request_log.exception(
|
||||
"request failed",
|
||||
extra={
|
||||
**event(
|
||||
"http.request",
|
||||
method=request.method,
|
||||
path=_route_template(request),
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 1),
|
||||
),
|
||||
**_request_owner(request, logging.ERROR),
|
||||
},
|
||||
)
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
raise
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
status = response.status_code
|
||||
quiet = request.url.path.startswith(_QUIET_PREFIXES) or (
|
||||
request.method == "GET" and _route_template(request) in _POLLED_ROUTES
|
||||
)
|
||||
if status >= 500:
|
||||
level = logging.ERROR
|
||||
elif status >= 400:
|
||||
level = logging.WARNING
|
||||
else:
|
||||
level = logging.DEBUG if quiet else logging.INFO
|
||||
attrs: dict[str, Any] = {
|
||||
"method": request.method,
|
||||
"path": _route_template(request),
|
||||
"status": status,
|
||||
"duration_ms": duration_ms,
|
||||
}
|
||||
# Teljes (TRACE) the user consented to: the query string too — redacted, and in attrs only, never
|
||||
# in the message.
|
||||
if request.url.query and _traced(scope):
|
||||
attrs["query"] = redact_query(request.url.query)
|
||||
_request_log.log(
|
||||
level,
|
||||
"%s %s %s",
|
||||
request.method,
|
||||
_route_template(request),
|
||||
status,
|
||||
extra={**event("http.request", **attrs), **_request_owner(request, level)},
|
||||
)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
# The browser logs its own copy of this line (frontend lib/http.ts) and must pick the same level:
|
||||
# it learns "this one is quiet" here instead of keeping a second copy of the polled-route list.
|
||||
if level == logging.DEBUG:
|
||||
response.headers["X-Log-Quiet"] = "1"
|
||||
# The template, so the browser's copy of this line groups the same way (frontend lib/http.ts).
|
||||
# Only a MATCHED route's: a template never carries a path parameter's value (a share token stays
|
||||
# `{token}`), while the raw-path fallback of an unmatched request could — the browser then
|
||||
# templates the path itself.
|
||||
matched = _matched_route(request)
|
||||
if matched:
|
||||
response.headers["X-Route"] = matched
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
return response
|
||||
|
||||
|
||||
def consented_trace() -> bool:
|
||||
"""Whether the current request's OWN user turned Teljes (TRACE) on — the gate for a line that
|
||||
carries what they typed or what was quoted back to them. The level alone is not enough: an
|
||||
admin running the whole process at TRACE would pass it for every user, and a scheduler thread,
|
||||
which has no request, for none of them in particular. Guard such a line with this, then log it
|
||||
at TRACE."""
|
||||
scope = _scope.get()
|
||||
return scope is not None and _traced(scope)
|
||||
|
||||
|
||||
def _traced(scope: _RequestScope) -> bool:
|
||||
"""Whether this request's query string is logged: only under its OWN user's Teljes consent. An
|
||||
admin setting the whole process to TRACE gets more detail, not every user's search terms — the
|
||||
privacy policy promises those are kept only for a user who agreed to it."""
|
||||
return scope.level is not None and scope.level <= TRACE
|
||||
|
||||
|
||||
# Anonymous request lines kept per client per minute. Signed-in traffic is not capped: it is a known
|
||||
# account and exactly what an admin needs to see. Anonymous traffic is where the volume can come from
|
||||
# nobody in particular — a scanner walking `/wp-login.php`, a bot on a share link — and each of its
|
||||
# 404s is a WARNING that would otherwise land in `log_events` without limit. The client key is an
|
||||
# in-memory counter key only; no address is ever written to a log line or a row.
|
||||
ANONYMOUS_LINES_PER_MINUTE = 60
|
||||
_anonymous_lines = RateLimiter(ANONYMOUS_LINES_PER_MINUTE, 60.0)
|
||||
|
||||
|
||||
def _request_owner(request, level: int) -> dict[str, Any]:
|
||||
"""`extra=` fields naming who the request belongs to, and whether its line may be stored. The
|
||||
account comes from `request.state` (filled by the auth dependency), so it lands in the line's
|
||||
top-level `user_id` — the correlation field — rather than in `attrs`."""
|
||||
user_id = getattr(request.state, "user_id", None)
|
||||
if user_id is not None:
|
||||
return {"user_id": user_id}
|
||||
if level < logging.INFO:
|
||||
return {} # a quiet line is never stored anyway — don't spend the client's allowance on it
|
||||
return {} if _anonymous_lines.allow(client_key(request)) else {PERSIST: False}
|
||||
|
||||
|
||||
def _matched_route(request) -> str | None:
|
||||
"""The matched route's template (`/api/videos/{video_id}`), or None when nothing matched."""
|
||||
return getattr(request.scope.get("route"), "path", None)
|
||||
|
||||
|
||||
def _route_template(request) -> str:
|
||||
"""The matched route's template, else the raw path for a 404 or a static file. Never the query
|
||||
string."""
|
||||
return _matched_route(request) or request.url.path
|
||||
@@ -0,0 +1,367 @@
|
||||
"""The `log_events` sink (E22 S3): the stored copy of the log the diagnostics console reads.
|
||||
|
||||
stdout stays the complete log (docker keeps 3×50 MB of it); this table keeps the part worth querying
|
||||
for `log_retention_days`:
|
||||
|
||||
* every WARNING and above,
|
||||
* the `http.request` line of every request at INFO or above (not the DEBUG-level health checks and
|
||||
static files),
|
||||
* every line of a request whose user raised their diagnostics level (E22 S5 — `logs.LevelGate`
|
||||
marks them), which is the detail they consented to have kept,
|
||||
* nothing a caller marked `extra={logs.PERSIST: False}` — how the request log caps anonymous
|
||||
traffic per client.
|
||||
|
||||
What the table must never do is slow down or break the code that logs. So a line is redacted and
|
||||
turned into a row on the CALLER's thread (the same `logs.record_fields` the stdout line is built
|
||||
from), put on a bounded queue without blocking, and written in batches by one background thread
|
||||
on its own connection — never inside a request's transaction. A full queue drops the row and
|
||||
counts it; a failed write drops the batch and says so on stderr. Both are reported rather than
|
||||
retried: a log that retries into a sick database is how a small outage becomes a big one.
|
||||
|
||||
Flood control is the second half. On 2026-09-16 a YouTube RSS outage made prod log 11k near-identical
|
||||
warnings in four hours. stdout keeps them all; the table keeps `REPEATS_PER_MINUTE` of each message
|
||||
template per minute and one `log.repeats_suppressed` row saying how many more there were.
|
||||
|
||||
Core layer: imports `logs`, `db`, `models`, `config` — nothing above them.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, insert
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.models import LogEvent
|
||||
from app.utils import iso
|
||||
|
||||
QUEUE_MAX = 10_000
|
||||
BATCH_MAX = 500
|
||||
FLUSH_SECONDS = 1.0
|
||||
REPEATS_PER_MINUTE = 30
|
||||
REPEAT_WINDOW_SECONDS = 60.0
|
||||
# Distinct templates tracked per window. Past this many, a new template is stored without counting
|
||||
# — the limiter must not become the unbounded thing it exists to prevent.
|
||||
_REPEAT_KEYS_MAX = 2_000
|
||||
|
||||
# Column widths from models.LogEvent. A value past its column would fail the WHOLE batch it rides in.
|
||||
_LIMITS = {
|
||||
"level": 10,
|
||||
"source": 16,
|
||||
"logger": 128,
|
||||
"event": 64,
|
||||
"request_id": 64,
|
||||
"debug_session_id": 64,
|
||||
"version": 64,
|
||||
"msg": 8_000,
|
||||
"exc": 32_000,
|
||||
}
|
||||
|
||||
_log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def should_store(record: logging.LogRecord) -> bool:
|
||||
"""Whether this record belongs in `log_events` (see the module docstring)."""
|
||||
if getattr(record, logs.PERSIST, True) is False:
|
||||
return False
|
||||
if record.levelno >= logging.WARNING or getattr(record, "elevated", False):
|
||||
return True
|
||||
return getattr(record, "event", None) == "http.request" and record.levelno >= logging.INFO
|
||||
|
||||
|
||||
def _without_nul(value: Any) -> Any:
|
||||
"""`value` with every NUL character replaced, in keys and nested values alike. Postgres refuses
|
||||
NUL in text and `\\u0000` in JSONB, and a refused row fails the WHOLE batch it rides in — every
|
||||
user's rows with it. Anything logged can carry one (`GET /assets/%00` logs the decoded path)."""
|
||||
if isinstance(value, str):
|
||||
return value.replace("\x00", "�")
|
||||
if isinstance(value, dict):
|
||||
return {_without_nul(k): _without_nul(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_without_nul(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def to_row(fields: dict[str, Any]) -> dict[str, Any]:
|
||||
"""`logs.record_fields` output → a `log_events` row: known columns only, text clipped to its
|
||||
column and free of NUL, attrs made JSON-safe (anything exotic becomes its `str`, as on stdout)."""
|
||||
row = {
|
||||
column: fields.get(column) for column in LogEvent.__table__.columns.keys() if column != "id"
|
||||
}
|
||||
for column, limit in _LIMITS.items():
|
||||
value = row.get(column)
|
||||
if isinstance(value, str):
|
||||
value = _without_nul(value)
|
||||
row[column] = value[: limit - 1] + "…" if len(value) > limit else value
|
||||
if row["attrs"] is not None:
|
||||
row["attrs"] = _without_nul(
|
||||
json.loads(json.dumps(row["attrs"], ensure_ascii=False, default=str))
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def without_erased(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""The rows, with the account taken off any whose account was erased meanwhile
|
||||
(`logs.forget_user`). Checked right before the INSERT, under `logs.erasure_guard()` held until
|
||||
the commit: a row built before the erasure can still be waiting for its batch."""
|
||||
for row in rows:
|
||||
if logs.is_erased(row.get("user_id")):
|
||||
row["user_id"] = None
|
||||
return rows
|
||||
|
||||
|
||||
def _notice(code: str, msg: str, **attrs: Any) -> dict[str, Any]:
|
||||
"""A row the sink writes about ITSELF. Built directly, not logged: logging it would route it back
|
||||
through this sink (and the writer thread's own records are deliberately ignored)."""
|
||||
return to_row(
|
||||
{
|
||||
"ts": datetime.now(timezone.utc),
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": logs.current_source(),
|
||||
"logger": _log.name,
|
||||
"event": code,
|
||||
"msg": msg,
|
||||
"attrs": attrs,
|
||||
"version": settings.app_version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_DIGITS = re.compile(r"\d+")
|
||||
|
||||
|
||||
def _template(record: logging.LogRecord) -> str:
|
||||
"""The flood limiter's grouping key: the message template. Except when the template is a bare
|
||||
`%s` — a pass-through for someone else's text (yt-dlp's lines, `formats._YdlLog`), where the
|
||||
template says nothing and every distinct line would share one bucket. Then the text itself
|
||||
groups, with its numbers folded, so `Retrying fragment 5 (1/10)` and `(2/10)` are still one
|
||||
repeat and a different warning is not crowded out by them."""
|
||||
msg = str(record.msg)
|
||||
if msg != "%s" or not record.args:
|
||||
return msg
|
||||
return _DIGITS.sub("#", record.getMessage())[:200]
|
||||
|
||||
|
||||
class _Repeats:
|
||||
"""Per-template admission for one minute at a time. Used by the writer thread only (unlocked)."""
|
||||
|
||||
def __init__(self, per_window: int, window_seconds: float):
|
||||
self.per_window = per_window
|
||||
self.window = window_seconds
|
||||
self.started = time.monotonic()
|
||||
self.seen: dict[tuple[str, str, int], int] = {}
|
||||
|
||||
def admit(self, key: tuple[str, str, int] | None) -> bool:
|
||||
if key is None:
|
||||
return True
|
||||
count = self.seen.get(key)
|
||||
if count is None and len(self.seen) >= _REPEAT_KEYS_MAX:
|
||||
return True
|
||||
self.seen[key] = (count or 0) + 1
|
||||
return self.seen[key] <= self.per_window
|
||||
|
||||
def roll(self, now: float, *, final: bool = False) -> list[dict[str, Any]]:
|
||||
"""Close the window if it is over — or, `final`, because the process is: one notice per
|
||||
template that went past the limit. Without `final` a process that lives under a minute
|
||||
(every download child) would drop its repeats and never say so."""
|
||||
if not final and now - self.started < self.window:
|
||||
return []
|
||||
notices = [
|
||||
_notice(
|
||||
"log.repeats_suppressed",
|
||||
f"{count - self.per_window} more like this were not stored",
|
||||
logger=logger,
|
||||
template=logs.redact_text(template),
|
||||
suppressed=count - self.per_window,
|
||||
window_seconds=int(self.window),
|
||||
)
|
||||
for (logger, template, _level), count in self.seen.items()
|
||||
if count > self.per_window
|
||||
]
|
||||
self.seen.clear()
|
||||
self.started = now
|
||||
return notices
|
||||
|
||||
|
||||
class LogSink(logging.Handler):
|
||||
"""The handler plus its writer. `start()` runs the writer thread; a test drives `write_pending()`
|
||||
directly instead, on its own thread, with no timing involved."""
|
||||
|
||||
def __init__(self, bind: Engine = engine):
|
||||
super().__init__(level=logging.NOTSET)
|
||||
# Correlation fields are contextvars: capture them now, on the thread that logged.
|
||||
self.addFilter(logs.ContextFilter())
|
||||
self._bind = bind
|
||||
self._queue: queue.Queue[tuple[tuple[str, str, int] | None, dict[str, Any]]] = queue.Queue(
|
||||
QUEUE_MAX
|
||||
)
|
||||
self._dropped = 0
|
||||
self._dropped_lock = threading.Lock()
|
||||
self._repeats = _Repeats(REPEATS_PER_MINUTE, REPEAT_WINDOW_SECONDS)
|
||||
self._stopping = threading.Event()
|
||||
self._writer: threading.Thread | None = None
|
||||
|
||||
# --- caller side ---------------------------------------------------------------------------
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if threading.current_thread() is self._writer:
|
||||
return # the writer's own trouble (a driver warning mid-insert) must not feed itself
|
||||
if not should_store(record):
|
||||
return
|
||||
try:
|
||||
row = to_row(logs.record_fields(record))
|
||||
# silent-ok: `handleError` is the report — logging's own out-of-band channel to stderr.
|
||||
# A log call here would be handed to this handler, which is the one that just failed.
|
||||
except Exception: # noqa: BLE001 — a record we cannot turn into a row is reported, not raised
|
||||
self.handleError(record)
|
||||
return
|
||||
# Request lines are one per request by nature; only the repeated SAME message is a flood.
|
||||
key = (
|
||||
None
|
||||
if row["event"] == "http.request"
|
||||
else (record.name, _template(record), record.levelno)
|
||||
)
|
||||
try:
|
||||
self._queue.put_nowait((key, row))
|
||||
# silent-ok: counted, not logged — the count goes out as the `log.sink.dropped` notice row
|
||||
# the writer emits, which is the one report that does not feed this queue again.
|
||||
except queue.Full:
|
||||
with self._dropped_lock:
|
||||
self._dropped += 1
|
||||
|
||||
# --- writer side ---------------------------------------------------------------------------
|
||||
def write_pending(self, wait: float = 0.0, *, final: bool = False) -> int:
|
||||
"""Write one batch: whatever is queued (up to BATCH_MAX), waiting up to `wait` seconds for the
|
||||
first item. Returns the number of rows written, notices included. `final` closes the
|
||||
repeat window early — the last batch before the process exits."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
item = self._queue.get(timeout=wait) if wait > 0 else self._queue.get_nowait()
|
||||
# silent-ok: an idle sink. Nothing failed; there was simply nothing waiting.
|
||||
except queue.Empty:
|
||||
item = None
|
||||
while item is not None:
|
||||
key, row = item
|
||||
if self._repeats.admit(key):
|
||||
rows.append(row)
|
||||
if len(rows) >= BATCH_MAX:
|
||||
break
|
||||
try:
|
||||
item = self._queue.get_nowait()
|
||||
# silent-ok: the batch ends here because the queue ran dry, which is the normal end.
|
||||
except queue.Empty:
|
||||
item = None
|
||||
rows.extend(self._repeats.roll(time.monotonic(), final=final))
|
||||
with self._dropped_lock:
|
||||
dropped, self._dropped = self._dropped, 0
|
||||
if dropped:
|
||||
rows.append(
|
||||
_notice("log.sink.dropped", f"{dropped} log events were dropped", dropped=dropped)
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
try:
|
||||
# The connection is checked out BEFORE the guard, and every request path takes them in
|
||||
# that order too (its session already holds a connection when it reaches the guard). The
|
||||
# other way round, this thread would hold a process-global lock while waiting up to
|
||||
# `pool_timeout` for a connection, and the requests that could return one are the ones
|
||||
# blocked on the lock. The guard still spans the check, the insert and the commit.
|
||||
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
||||
conn.execute(insert(LogEvent), without_erased(rows))
|
||||
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
||||
# silent-ok / noqa T201: the sink's OWN failure. A log call here would be handed to
|
||||
# this sink, which is the thing that just failed — straight into a loop. stderr is
|
||||
# the one way out that does not re-enter.
|
||||
print( # noqa: T201
|
||||
f"log sink: could not store {len(rows)} log events: "
|
||||
f"{logs.redact_text(str(exc))[:300]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
return len(rows)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
self.write_pending(wait=FLUSH_SECONDS)
|
||||
while self.write_pending(): # drain what arrived before the stop
|
||||
pass
|
||||
self.write_pending(final=True) # and the repeat notices of a window cut short
|
||||
|
||||
def start(self) -> None:
|
||||
self._writer = threading.Thread(target=self._run, name="log-sink", daemon=True)
|
||||
self._writer.start()
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> None:
|
||||
self._stopping.set()
|
||||
if self._writer is not None:
|
||||
self._writer.join(timeout)
|
||||
|
||||
|
||||
_sink: LogSink | None = None
|
||||
|
||||
|
||||
def start() -> None:
|
||||
"""Attach the sink to the root logger and start its writer. Call after `logs.configure_logging`
|
||||
(which replaces the root handlers) and once the schema exists. A no-op when `LOG_DB_SINK` is off
|
||||
or it is already running."""
|
||||
global _sink
|
||||
if _sink is not None or not settings.log_db_sink:
|
||||
return
|
||||
_sink = LogSink()
|
||||
# The same level gate as stdout: the root may be lowered for one user's elevated requests, and
|
||||
# everyone else's detail must not reach the table through that gap either.
|
||||
_sink.addFilter(logs.LevelGate())
|
||||
logging.getLogger().addHandler(_sink)
|
||||
_sink.start()
|
||||
# A short-lived process (the download child) exits without a lifespan: flush on the way out.
|
||||
atexit.register(stop)
|
||||
|
||||
|
||||
def stop() -> None:
|
||||
"""Detach the sink and write what is still queued."""
|
||||
global _sink
|
||||
if _sink is None:
|
||||
return
|
||||
logging.getLogger().removeHandler(_sink)
|
||||
_sink.stop()
|
||||
_sink = None
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage the deletion of rows older than `retention_days` (at least one day — stored logs always
|
||||
expire). The caller commits. Returns the number of rows removed."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(LogEvent).where(LogEvent.ts < cutoff))
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def serialize(row: LogEvent) -> dict:
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console (E22.6) and
|
||||
a debug session's evidence list (E16)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
"level": row.level,
|
||||
"source": row.source,
|
||||
"logger": row.logger,
|
||||
"event": row.event,
|
||||
"msg": row.msg,
|
||||
"attrs": row.attrs,
|
||||
"exc": row.exc,
|
||||
"request_id": row.request_id,
|
||||
"user_id": row.user_id,
|
||||
"job_id": row.job_id,
|
||||
"debug_session_id": row.debug_session_id,
|
||||
"version": row.version,
|
||||
}
|
||||
@@ -175,7 +175,7 @@ MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Someone has opened up Siftlode for you — a quiet, self-hosted reader for your YouTube "
|
||||
"subscriptions. Every upload from the channels you follow lands in one feed you can "
|
||||
"subscriptions. Every upload from the channels you follow lands in one Feed you can "
|
||||
"actually filter and sort, with no algorithm deciding what you see, and no Shorts or "
|
||||
"livestreams unless you want them.\n\n"
|
||||
"{next_step}\n\n"
|
||||
@@ -186,7 +186,7 @@ MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
"Szia,\n\n"
|
||||
"Valaki megnyitotta előtted a Siftlode-ot — ez egy csendes, saját üzemeltetésű olvasó "
|
||||
"a YouTube-feliratkozásaidhoz. A követett csatornák minden feltöltése egyetlen "
|
||||
"hírfolyamba érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"Feedbe érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"eldönti, mit látsz, és nincs Shorts vagy élő adás, hacsak nem kéred.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"Nem sürget semmi — az ajtó nyitva marad.\n\n"
|
||||
|
||||
+39
-14
@@ -1,7 +1,5 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import sys
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
|
||||
@@ -9,35 +7,38 @@ from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
# When started via uvicorn --log-config the "siftlode" logger is already configured
|
||||
# (see log_config.json). This block is a timestamped fallback for other entrypoints
|
||||
# (tests, scripts) so our logs are never silently dropped.
|
||||
_siftlode_logger = logging.getLogger("siftlode")
|
||||
if not _siftlode_logger.handlers:
|
||||
_handler = logging.StreamHandler(sys.stdout)
|
||||
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s"))
|
||||
_siftlode_logger.addHandler(_handler)
|
||||
_siftlode_logger.setLevel(logging.INFO)
|
||||
_siftlode_logger.propagate = False
|
||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||
from app import logs
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
log = logging.getLogger("siftlode.app")
|
||||
configure_logging("api")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.plex import stream as plex_stream
|
||||
from app.plex import subtitles as plex_subtitles
|
||||
from app.routes import (
|
||||
admin,
|
||||
admin_logs as admin_logs_routes,
|
||||
audit as audit_routes,
|
||||
auth as auth_routes,
|
||||
channels,
|
||||
config as config_routes,
|
||||
debug_sessions as debug_sessions_routes,
|
||||
diagnostics as diagnostics_routes,
|
||||
discover as discover_routes,
|
||||
downloads,
|
||||
feed,
|
||||
@@ -63,16 +64,28 @@ from app.scheduler import shutdown_scheduler, start_scheduler
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log.info("Siftlode starting up")
|
||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||
logsink.start()
|
||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||
# the wizard URL so the operator can open it (the app runs in setup mode until they finish).
|
||||
with SessionLocal() as db:
|
||||
# The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap
|
||||
# default, so a level chosen in the UI survives a restart without touching the environment.
|
||||
# Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from
|
||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||
try:
|
||||
set_level(sysconfig.get_str(db, "log_level"))
|
||||
except Exception: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||
# and `%s` of the exception does not say which of them raised.
|
||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||
if not state.is_configured(db):
|
||||
token = state.rotate_setup_token(db)
|
||||
url = f"{settings.app_base}/setup?token={token}"
|
||||
log.warning(
|
||||
"FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n"
|
||||
" Open the install wizard (internal access only):\n %s",
|
||||
url,
|
||||
url, # log-pii-ok: this instance's OWN setup URL, and `?token=` is redacted
|
||||
)
|
||||
# No HLS session survives a restart (they live in this process), so anything still on disk is
|
||||
# a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not
|
||||
@@ -100,6 +113,7 @@ async def lifespan(app: FastAPI):
|
||||
# Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for
|
||||
# the rest of the read (see plex_subtitles.shutdown).
|
||||
plex_subtitles.shutdown()
|
||||
logsink.stop() # last: write what the shutdown itself just logged
|
||||
|
||||
|
||||
# How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`).
|
||||
@@ -243,6 +257,14 @@ async def security_headers(request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what the three above do. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth_routes.router)
|
||||
app.include_router(sync.router)
|
||||
@@ -251,6 +273,8 @@ app.include_router(feed.router)
|
||||
app.include_router(discover_routes.router)
|
||||
app.include_router(search_routes.router)
|
||||
app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(debug_sessions_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
@@ -264,6 +288,7 @@ app.include_router(admin.router)
|
||||
app.include_router(config_routes.router)
|
||||
app.include_router(scheduler_routes.router)
|
||||
app.include_router(audit_routes.router)
|
||||
app.include_router(admin_logs_routes.router)
|
||||
app.include_router(quota.router)
|
||||
app.include_router(version.router)
|
||||
app.include_router(setup_routes.router)
|
||||
|
||||
@@ -12,6 +12,8 @@ from sqlalchemy import (
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
LargeBinary,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
@@ -102,6 +104,16 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
|
||||
# events (client-side signed cookies otherwise stay valid until they expire).
|
||||
session_epoch: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# When this account last signed in (Google or password), stamped by the one sign-in path,
|
||||
# routes/auth._establish_session. NULL = not since the column existed (0085; no backfill is
|
||||
# possible). The shared demo account is never stamped — its "last login" would mean nobody.
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The diagnostics level this user consented to (E22 S5) and when it lapses back to the base
|
||||
# level. Both NULL = base. Written only by `PUT /api/me/diagnostics`, which computes the expiry
|
||||
# itself; read through `app.diagnostics.effective_level`, never directly (an expired level is
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
@@ -230,6 +242,9 @@ class Channel(Base, TimestampMixin):
|
||||
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# When the newest uploads were last read through the Data API because the RSS feed would not
|
||||
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
||||
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this channel only entered the catalog because a live YouTube search
|
||||
@@ -539,6 +554,91 @@ class AuditLog(Base):
|
||||
)
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
"""One stored log event (E22 S3) — the `log_events` table the diagnostics console reads. Rows are
|
||||
written by app/logsink.py's background writer, never inside a request's transaction, already
|
||||
redacted (the same `logs.record_fields` the stdout line is built from).
|
||||
|
||||
`user_id` has NO foreign key on purpose: the writer runs behind the event, so a line about an
|
||||
account can arrive after that account was purged, and a constraint would make it fail the whole
|
||||
batch it travels in. `auth.purge_user` deletes an account's rows explicitly instead, after
|
||||
`logs.forget_user` makes every later write in this process store such a line without the account;
|
||||
the retention job (`log_retention_days`) bounds whatever another process writes late."""
|
||||
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (
|
||||
Index("ix_log_events_user_ts", "user_id", "ts"),
|
||||
# E16: an issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
Index(
|
||||
"ix_log_events_debug_session",
|
||||
"debug_session_id",
|
||||
postgresql_where="debug_session_id IS NOT NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
level: Mapped[str] = mapped_column(String(10))
|
||||
sev: Mapped[int] = mapped_column(SmallInteger)
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
logger: Mapped[str] = mapped_column(String(128))
|
||||
event: Mapped[str | None] = mapped_column(String(64))
|
||||
msg: Mapped[str] = mapped_column(Text)
|
||||
attrs: Mapped[dict | None] = mapped_column(JSONB)
|
||||
exc: Mapped[str | None] = mapped_column(Text)
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[int | None] = mapped_column(Integer)
|
||||
debug_session_id: Mapped[str | None] = mapped_column(String(64))
|
||||
job_id: Mapped[int | None] = mapped_column(Integer)
|
||||
version: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class DebugSession(Base):
|
||||
"""One recording: Start → reproduce → Stop (E16, app/debugsession.py). While it is open, every
|
||||
request of its user that names it (`X-Debug-Session`) is logged at `level` and tagged with `id`,
|
||||
and the issue it ends in is built from the rows carrying that tag. `until` is the hard cap the
|
||||
server set at Start; an open session past it is over whether or not the sweep has closed it yet.
|
||||
One OPEN session per user (`uq_debug_sessions_open_per_user`, partial on `ended_at IS NULL`)."""
|
||||
|
||||
__tablename__ = "debug_sessions"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"user_id",
|
||||
unique=True,
|
||||
postgresql_where="ended_at IS NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
level: Mapped[str] = mapped_column(String(8))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
until: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# app/debugsession.py `EndReason`: stopped by the user, or past `until`.
|
||||
ended_reason: Mapped[str | None] = mapped_column(String(16))
|
||||
issue_url: Mapped[str | None] = mapped_column(String(512))
|
||||
# The issue being written from it (E16 S3; app/debugsession.py `save_draft` holds its shape).
|
||||
draft: Mapped[dict | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class DebugScreenshot(Base):
|
||||
"""A screenshot attached to a debug session (E16) — pasted, dropped, or captured from the pill.
|
||||
Stored in the row: bounded per session (app/debugsession.py), and gone with it by cascade."""
|
||||
|
||||
__tablename__ = "debug_screenshots"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("debug_sessions.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
mime: Mapped[str] = mapped_column(String(16))
|
||||
size: Mapped[int] = mapped_column(Integer)
|
||||
data: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
|
||||
class AppState(Base):
|
||||
"""Single-row global app state (admin-controlled)."""
|
||||
|
||||
@@ -871,6 +971,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
|
||||
@@ -6,16 +6,15 @@ metadata, image proxying, and (P5) optional watch-state write-back. Requests JSO
|
||||
``Accept: application/json`` header (Plex serves XML by default).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class PlexError(Exception):
|
||||
@@ -106,7 +105,18 @@ class PlexClient:
|
||||
Best-effort (needs the library's related data) — returns [] if Plex has none."""
|
||||
try:
|
||||
mc = self._get(f"/library/metadata/{rating_key}/related")
|
||||
except PlexError:
|
||||
except PlexError as exc:
|
||||
# An empty list is also the answer for an item that genuinely has no related titles,
|
||||
# so the caller cannot tell the two apart — and it deliberately declines to CACHE an
|
||||
# empty one for exactly that reason. This line is what says which it was.
|
||||
log.debug(
|
||||
"the related items for a Plex item could not be read",
|
||||
extra=logs.event(
|
||||
"plex.client.related_unavailable",
|
||||
rating_key=rating_key,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for hub in mc.get("Hub", []) or []:
|
||||
@@ -139,6 +149,8 @@ class PlexClient:
|
||||
raise PlexError(str(e)) from e
|
||||
try:
|
||||
return (r.json() or {}).get("MediaContainer", {}) or {} if r.content else {}
|
||||
# silent-ok: the HTTP call already succeeded — this is a write endpoint, and Plex answers
|
||||
# those with an empty or non-JSON body as a matter of course.
|
||||
except ValueError:
|
||||
return {}
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ def rich_detail(meta: dict) -> dict:
|
||||
if str(r.get("image") or "").startswith("imdb://"):
|
||||
try:
|
||||
imdb_rating = round(float(r.get("value")), 1)
|
||||
# silent-ok: no IMDb score, which is the common and legitimate case — the UI has a
|
||||
# separate audience score to fall back on and shows nothing if neither exists.
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
break
|
||||
|
||||
@@ -11,10 +11,13 @@ The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see
|
||||
from fastapi import Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexCollection, PlexItem, PlexSeason, PlexShow
|
||||
from app.plex.cache import CACHE_DIR
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
||||
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||
# Target box per variant — Plex resizes to these server-side. Posters fill ≤176px cells (grid/info/
|
||||
@@ -46,6 +49,8 @@ def read(key: str) -> tuple[bytes, str] | None:
|
||||
try:
|
||||
if p.exists() and p.stat().st_size > 0:
|
||||
return p.read_bytes(), ct
|
||||
# silent-ok: a miss. The caller fetches from Plex and writes the file again, and the write
|
||||
# path below is the one that reports a cache that cannot be used.
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
@@ -58,8 +63,14 @@ def write(key: str, data: bytes, ct: str) -> None:
|
||||
tmp = CACHE_DIR / f"{key}.{ext}.tmp"
|
||||
tmp.write_bytes(data)
|
||||
tmp.replace(CACHE_DIR / f"{key}.{ext}") # atomic publish
|
||||
except OSError:
|
||||
pass # cache is best-effort
|
||||
except OSError as exc:
|
||||
# Best-effort, and silently so: with an unwritable cache every poster on every grid goes
|
||||
# to Plex live, for as long as nobody notices. Debug, because a grid is ~100 images and
|
||||
# the point is the condition, not the count.
|
||||
log.debug(
|
||||
"the Plex image cache could not be written",
|
||||
extra=logs.event("plex.imgcache.write_failed", reason=type(exc).__name__),
|
||||
)
|
||||
|
||||
|
||||
def response(data: bytes, ct: str) -> Response:
|
||||
|
||||
@@ -12,7 +12,9 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def parse_map(raw: str) -> list[tuple[str, str]]:
|
||||
@@ -47,6 +49,13 @@ def local_media_path(db: Session, plex_path: str | None) -> Path | None:
|
||||
return None
|
||||
try:
|
||||
p = Path(map_path(db, plex_path)).resolve()
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# None is answered as a 404 "no local file", which is also the answer for a title the
|
||||
# server does not hold — so a dead mount or a wrong path map reads as missing media for
|
||||
# the whole library at once. The path itself stays out: it carries the title.
|
||||
log.warning(
|
||||
"a Plex media path could not be resolved locally",
|
||||
extra=logs.event("plex.paths.resolve_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
return p if p.is_file() else None
|
||||
|
||||
@@ -298,10 +298,6 @@ class LibraryFilters:
|
||||
}
|
||||
|
||||
|
||||
def like_escape(s: str) -> str:
|
||||
return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
# --- Watch-state scope (shared by the /library grid AND the /facets aggregation) ---------------
|
||||
#
|
||||
# The `show` scope (all | watched | in_progress | unwatched) filters the browse result by per-user
|
||||
|
||||
@@ -43,7 +43,9 @@ def rollup(cards: list[dict]) -> dict:
|
||||
]
|
||||
resume = None
|
||||
if inprog:
|
||||
resume = inprog[-1] # continue the latest-started episode
|
||||
# The LAST in-progress episode in broadcast order — no timestamp is consulted. Home's
|
||||
# `discover.queries.next_episode` picks the same one, so both Resume buttons agree.
|
||||
resume = inprog[-1]
|
||||
else:
|
||||
resume = next((c for c in cards if c.get("status") != "watched"), None)
|
||||
return {
|
||||
|
||||
+182
-52
@@ -18,7 +18,6 @@ segments. The remux is CPU-light (video copy + a tiny audio transcode), so it ru
|
||||
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
@@ -31,15 +30,14 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session as DbSession
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads.edit import encode_thread_cap
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths
|
||||
from app.plex.sync import HW_DECODABLE_CODECS
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_HLS_ROOT = (
|
||||
Path(settings.plex_hls_dir)
|
||||
if settings.plex_hls_dir
|
||||
@@ -62,8 +60,10 @@ _WARMUP_SEG_TIMEOUT_S = 8.0
|
||||
# its index for that one. See `segment_name` for why this is a constant and not a comment.
|
||||
_VIDEO_VARIANT = "0"
|
||||
_SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# A session directory is named `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}` (see start_session);
|
||||
# before multi-audio it was just `{rating_key}_{int(start_s)}`. `sweep_orphans` deletes ONLY names
|
||||
# A session directory is named `{rating_key}_u{user_id}_{int(start_s)}_{tag}_{aoff:+.2f}` (see
|
||||
# start_session); before per-viewer sessions it was `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}`,
|
||||
# and before multi-audio just `{rating_key}_{int(start_s)}` — the older shapes stay sweepable, since a
|
||||
# restart leaves them behind like any other. `sweep_orphans` deletes ONLY names
|
||||
# matching one of those two shapes: `_HLS_ROOT` is admin-configurable (PLEX_HLS_DIR), so it may
|
||||
# legitimately point at a shared scratch path we do not own, and a blanket "delete every directory
|
||||
# under the root" would take the neighbours with it.
|
||||
@@ -74,7 +74,9 @@ _SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# non-numeric rating_key would merely leave its directory unswept, which is the safe direction to
|
||||
# fail. Covered by test_plex_stream.py — this regex is the only thing standing between the sweep
|
||||
# and someone else's data.
|
||||
_SESSION_DIR_RE = re.compile(r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$")
|
||||
_SESSION_DIR_RE = re.compile(
|
||||
r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$|^\d+_u\d+_\d+_[^_]+_[+-]\d+\.\d{2}$"
|
||||
)
|
||||
# When this process started. `sweep_orphans` only deletes directories older than this: everything
|
||||
# newer belongs to a session THIS run created, and since `start_session` reuses a directory path
|
||||
# for the same (rating_key, offset, tag), "stale name" is not the same as "stale directory".
|
||||
@@ -90,7 +92,16 @@ _PROCESS_START = time.time()
|
||||
_MTIME_GRACE_S = 2.0
|
||||
|
||||
_lock = threading.Lock()
|
||||
_sessions: dict[str, "HlsSession"] = {}
|
||||
# Keyed by VIEWER and item, (user_id, rating_key). Keyed by the item alone, a second account opening
|
||||
# the same film `_kill`ed the first one's session: the displaced viewer got a 404, reclaimed at its
|
||||
# playhead, and so evicted the other in turn — two people could not watch one film at once. (One
|
||||
# account on two devices still shares a session; that case ends in the player's bounded reclaim.)
|
||||
_sessions: dict[tuple[int, str], "HlsSession"] = {}
|
||||
# How many live sessions one account may hold at once (two devices, or a next item starting while the
|
||||
# last one winds down). A new one past it replaces that account's least recently used — whose waiting
|
||||
# session request then returns at once, since `wait_for` watches the killed ffmpeg — so one account
|
||||
# can neither pile up film-sized scratch copies nor park more than this many request workers.
|
||||
_MAX_SESSIONS_PER_USER = 2
|
||||
|
||||
|
||||
class HlsSession:
|
||||
@@ -104,16 +115,17 @@ class HlsSession:
|
||||
transcode: bool = False,
|
||||
hw: bool = False,
|
||||
tonemap: bool = False,
|
||||
user_id: int = 0,
|
||||
):
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. Sessions are keyed
|
||||
# by rating_key alone, so a seek-restart or a SECOND viewer of the same item replaces the
|
||||
# registered one — and the old viewer's next request for `seg_12.ts`, resolved by rating_key,
|
||||
# was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. A seek-restart
|
||||
# replaces the viewer's registered session for the item — and the old player's next request
|
||||
# for `seg_12.ts`, resolved by (viewer, rating_key) alone, was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it
|
||||
# (deep-review round 5). A request naming a session that is no longer registered is a 404,
|
||||
# which the player recovers from by restarting at the playhead.
|
||||
self.id = secrets.token_urlsafe(9)
|
||||
self.key = key
|
||||
self.key = key # the item's rating_key (what the logs name)
|
||||
self.user_id = user_id # whose session it is: the registry key is (user_id, key)
|
||||
self.dir = directory
|
||||
self.proc = proc
|
||||
self.transcode = transcode # video re-encoded (non-h264 source), not stream-copied
|
||||
@@ -135,6 +147,8 @@ class HlsSession:
|
||||
# single index. Derived, not passed, so it cannot disagree with the argv.
|
||||
self.entry = _MASTER_PL if self.variants else _INDEX_PL
|
||||
self.last_access = time.time()
|
||||
# `artifact_may_arrive` reports a refusing session directory once, not per segment.
|
||||
self.probe_failure_reported = False
|
||||
|
||||
|
||||
def _seek_opts(start_s: float) -> list[str]:
|
||||
@@ -232,7 +246,15 @@ def _probe_video_kind(src: Path) -> str:
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
||||
# ffprobe sends every HDR file down it, so playback quietly degrades to a software
|
||||
# transcode with nothing anywhere saying why. The path stays out of the line: it is the
|
||||
# title of whatever is playing.
|
||||
log.info(
|
||||
"video-kind probe failed; assuming the HW-safe pipeline",
|
||||
extra=logs.event("plex.stream.probe_kind_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return _VK_HWSAFE
|
||||
pix_fmt = bits = transfer = ""
|
||||
for line in out.stdout.splitlines():
|
||||
@@ -261,7 +283,14 @@ def _opencl_present() -> bool:
|
||||
software libx264. Its own function so it's monkeypatchable in tests, mirroring _render_node_present."""
|
||||
try:
|
||||
return any(_OPENCL_ICD_DIR.glob("*.icd"))
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# A MISSING directory does not come through here — `glob` simply yields nothing for one.
|
||||
# This is a directory we cannot read, and the answer it forces (no OpenCL) costs every HDR
|
||||
# source the hardware tonemap path.
|
||||
log.info(
|
||||
"the OpenCL ICD directory could not be read; treating OpenCL as absent",
|
||||
extra=logs.event("plex.stream.opencl_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -404,7 +433,7 @@ def _select_encode(db: DbSession, item: PlexItem, transcode: bool, src: Path) ->
|
||||
source that has an OpenCL runtime, while a 10-bit SDR source drops to software."""
|
||||
if not transcode:
|
||||
return (False, False)
|
||||
mode = (sysconfig.get_str(db, "plex_transcode_hw") or "auto").lower()
|
||||
mode = sysconfig.get_str(db, "plex_transcode_hw")
|
||||
if mode == "off" or not _render_node_present():
|
||||
return (False, False)
|
||||
if mode == "on":
|
||||
@@ -447,6 +476,8 @@ def _sw_child_preexec() -> None:
|
||||
playback."""
|
||||
try:
|
||||
os.nice(_SW_TRANSCODE_NICE)
|
||||
# silent-ok: this runs in the forked child, between fork and exec. Logging there can deadlock
|
||||
# on a lock some other thread held at fork time, and the child is about to become ffmpeg.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@@ -473,7 +504,14 @@ def _probe_audio_langs(src: Path) -> list[str]:
|
||||
timeout=10,
|
||||
)
|
||||
return [ln.strip() for ln in out.stdout.splitlines()]
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
# An empty list IS the single-audio path, so a failed probe silently collapses a file with
|
||||
# several audio tracks down to one — and the viewer gets a player with no language picker
|
||||
# rather than an error.
|
||||
log.warning(
|
||||
"audio-track probe failed; falling back to a single-audio session",
|
||||
extra=logs.event("plex.stream.probe_audio_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
@@ -582,8 +620,14 @@ def _probe_first_pts(seg: Path) -> float | None:
|
||||
tok = line.strip().strip(",")
|
||||
if tok:
|
||||
return float(tok)
|
||||
except (subprocess.SubprocessError, ValueError, OSError):
|
||||
pass
|
||||
except (subprocess.SubprocessError, ValueError, OSError) as exc:
|
||||
# None means "keep the offset the client asked for", and the whole point of this probe is
|
||||
# that the two differ: without it the subtitle track is shifted by whatever the container
|
||||
# started its timestamps at. Seconds of drift, and nothing on screen calls it a failure.
|
||||
log.warning(
|
||||
"first-PTS probe failed; the requested offset is used as-is",
|
||||
extra=logs.event("plex.stream.first_pts_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -592,29 +636,62 @@ def _kill(s: HlsSession) -> None:
|
||||
s.proc.terminate()
|
||||
try:
|
||||
s.proc.wait(timeout=3)
|
||||
# silent-ok: the escalation IS the handling. A transcode that ignores TERM for three
|
||||
# seconds gets KILL, which is exactly what this branch does.
|
||||
except subprocess.TimeoutExpired:
|
||||
s.proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — teardown finishes whatever the process does
|
||||
# Not nothing: what survives this is an ffmpeg nobody owns any more, holding a VAAPI
|
||||
# handle and a core, and the transcode cap that is supposed to bound them counts rows in
|
||||
# `_sessions` rather than processes. One line is the only way that ever surfaces.
|
||||
log.info(
|
||||
"could not stop the transcode process",
|
||||
extra=logs.event("plex.stream.kill_failed", key=s.key, reason=type(exc).__name__),
|
||||
)
|
||||
shutil.rmtree(s.dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _enforce_cap(cap: int) -> None:
|
||||
# Caller holds _lock. Only TRANSCODE sessions count toward plex_max_transcodes — remux
|
||||
# (stream-copy) and direct-serve are cheap and deliberately uncapped (see config.py:
|
||||
# plex_max_transcodes, and the user-facing config hint). Drop the least-recently-accessed
|
||||
# transcode sessions over the cap, leaving any concurrent remux streams untouched.
|
||||
tx = [(key, s) for key, s in _sessions.items() if s.transcode]
|
||||
if len(tx) <= cap:
|
||||
return
|
||||
for key, s in sorted(tx, key=lambda kv: kv[1].last_access)[: len(tx) - cap]:
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
def _enforce_caps(user_id: int, max_transcodes: int, max_sessions: int) -> None:
|
||||
"""Caller holds _lock. Three bounds, applied after a session is registered, each dropping the
|
||||
least recently accessed sessions over it:
|
||||
|
||||
* this account's own sessions (`_MAX_SESSIONS_PER_USER`) — first, so a single account filling the
|
||||
box evicts itself before anyone else;
|
||||
* TRANSCODE sessions (plex_max_transcodes): the CPU/GPU-heavy ones;
|
||||
* EVERY session (plex_max_sessions). A remux is cheap on the CPU but not on the disk: each one is a
|
||||
whole segmented copy of the film under `_HLS_ROOT`, written faster than realtime. Counted only
|
||||
by the transcode cap, N distinct items were N ffmpegs and N film-sized scratch directories.
|
||||
|
||||
Somebody's playback stops at each drop. The route turns the missing session into a retryable 404
|
||||
and the player restarts at the playhead, so from the server nothing failed — and "my film keeps
|
||||
rebuffering" has no other explanation anywhere, hence the line."""
|
||||
|
||||
def cull(pool: list[tuple[tuple[int, str], HlsSession]], cap: int, bound: str) -> None:
|
||||
if len(pool) <= cap:
|
||||
return
|
||||
for reg, sess in sorted(pool, key=lambda kv: kv[1].last_access)[: len(pool) - cap]:
|
||||
log.info(
|
||||
"a session was dropped to stay under a cap",
|
||||
extra=logs.event(
|
||||
"plex.stream.session_culled",
|
||||
key=sess.key,
|
||||
bound=bound,
|
||||
cap=cap,
|
||||
sessions=len(pool),
|
||||
),
|
||||
)
|
||||
_kill(sess)
|
||||
_sessions.pop(reg, None)
|
||||
|
||||
cull([kv for kv in _sessions.items() if kv[0][0] == user_id], _MAX_SESSIONS_PER_USER, "user")
|
||||
cull([kv for kv in _sessions.items() if kv[1].transcode], max_transcodes, "transcode")
|
||||
cull(list(_sessions.items()), max_sessions, "total")
|
||||
|
||||
|
||||
def start_session(
|
||||
db: DbSession,
|
||||
item: PlexItem,
|
||||
user_id: int,
|
||||
start_s: float,
|
||||
audio_ord: int | None = None,
|
||||
aoff: float = 0.0,
|
||||
@@ -634,6 +711,7 @@ def start_session(
|
||||
# session respawn (the HW->SW fallback) must not pick up a mid-playback config change.
|
||||
loudnorm = sysconfig.get_bool(db, "plex_audio_loudnorm")
|
||||
key = item.rating_key
|
||||
reg = (user_id, key) # the registry key: one session per viewer per item
|
||||
start_s = max(0.0, float(start_s))
|
||||
audio_langs = _probe_audio_langs(src) if multi else []
|
||||
is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks
|
||||
@@ -646,7 +724,7 @@ def start_session(
|
||||
# the tag with a DASH — `_SESSION_DIR_RE` treats the tag as one underscore-free field, and that
|
||||
# regex is the only thing keeping the orphan sweep off a shared scratch path (see its comment).
|
||||
tag = ("multi" if is_multi else str(audio_ord)) + ("-ln" if loudnorm else "")
|
||||
directory = _HLS_ROOT / f"{key}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
directory = _HLS_ROOT / f"{key}_u{user_id}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
|
||||
def _seg(i: int) -> Path:
|
||||
return directory / segment_name(i, _VIDEO_VARIANT if variants else None)
|
||||
@@ -664,7 +742,7 @@ def start_session(
|
||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||
preset, crf, max_height = "veryfast", 20, 0
|
||||
if transcode and not hw:
|
||||
preset = sysconfig.get_str(db, "plex_transcode_preset") or "veryfast"
|
||||
preset = sysconfig.get_str(db, "plex_transcode_preset")
|
||||
crf = sysconfig.get_int(db, "plex_transcode_crf")
|
||||
max_height = sysconfig.get_int(db, "plex_transcode_max_height")
|
||||
# Nice the SOFTWARE transcode child (not the HW path, and not remux) so a CPU-bound re-encode
|
||||
@@ -672,9 +750,9 @@ def start_session(
|
||||
# platform without fork (the API always runs in a Linux container in prod) it's simply omitted.
|
||||
preexec = _sw_child_preexec if (transcode and not hw and os.name == "posix") else None
|
||||
with _lock:
|
||||
if replace_only is not None and _sessions.get(key) is not replace_only:
|
||||
if replace_only is not None and _sessions.get(reg) is not replace_only:
|
||||
return None # superseded/culled while we waited out of lock — don't clobber the newer one
|
||||
old = _sessions.pop(key, None)
|
||||
old = _sessions.pop(reg, None)
|
||||
if old is not None:
|
||||
_kill(old)
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
@@ -708,11 +786,31 @@ def start_session(
|
||||
transcode=transcode,
|
||||
hw=hw,
|
||||
tonemap=tonemap,
|
||||
user_id=user_id,
|
||||
)
|
||||
_sessions[reg] = s
|
||||
# The one line that says which pipeline a session actually got. Everything downstream
|
||||
# — the cap, the HW→SW fallback, the subtitle offset — is about this choice, and until
|
||||
# now it was only inferable from the ffmpeg command nobody keeps.
|
||||
log.info(
|
||||
"an HLS session started",
|
||||
extra=logs.event(
|
||||
"plex.stream.session_started",
|
||||
key=key,
|
||||
transcode=transcode,
|
||||
hw=hw,
|
||||
tonemap=tonemap,
|
||||
variants=len(variants or ()),
|
||||
start_s=round(start_s, 1),
|
||||
),
|
||||
)
|
||||
# Admin-configurable caps (Configuration → Plex); at least 1 each so playback can't be
|
||||
# capped to zero.
|
||||
_enforce_caps(
|
||||
user_id,
|
||||
max(1, sysconfig.get_int(db, "plex_max_transcodes")),
|
||||
max(1, sysconfig.get_int(db, "plex_max_sessions")),
|
||||
)
|
||||
_sessions[key] = s
|
||||
# Admin-configurable concurrency cap (Configuration → Plex → Max concurrent transcodes);
|
||||
# at least 1 so playback can't be capped to zero.
|
||||
_enforce_cap(max(1, sysconfig.get_int(db, "plex_max_transcodes")))
|
||||
return s
|
||||
|
||||
s = _spawn(*_select_encode(db, item, transcode, src))
|
||||
@@ -728,7 +826,8 @@ def start_session(
|
||||
# ONE lock in _spawn, so we can never tear down the newer session. s.hw is now False, so the route
|
||||
# honestly reports "CPU" and the player's Preparing-stream state stays correct.
|
||||
log.warning(
|
||||
"plex transcode: HW session produced no segments — falling back to software key=%s", key
|
||||
"the hardware transcode produced no segments; falling back to software",
|
||||
extra=logs.event("plex.stream.hw_fallback", key=key),
|
||||
)
|
||||
respawned = _spawn(False, False, replace_only=s)
|
||||
if respawned is None:
|
||||
@@ -737,7 +836,7 @@ def start_session(
|
||||
# turns a None return into a "Media file not found on disk" 404 — dishonest for a file that
|
||||
# exists. A (dead) session instead degrades to a retryable 404 ("No active playback session" /
|
||||
# "Not available yet") the client re-requests.
|
||||
return current_session(key) or s
|
||||
return current_session(user_id, key) or s
|
||||
s = respawned
|
||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||
if ready:
|
||||
@@ -768,9 +867,9 @@ def start_session(
|
||||
return s
|
||||
|
||||
|
||||
def current_session(key: str) -> HlsSession | None:
|
||||
def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
with _lock:
|
||||
s = _sessions.get(key)
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
return s
|
||||
@@ -867,7 +966,23 @@ def artifact_may_arrive(s: "HlsSession", filename: str) -> bool:
|
||||
return True
|
||||
try:
|
||||
return (s.dir / segment_name(seq - 1, variant)).exists()
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# `exists()` answers False for a missing file without raising, so this is the session
|
||||
# directory itself refusing — a full or unmounted disk. Every segment then 404s, which the
|
||||
# player shows as a stall. Once per session: this runs for every segment request and
|
||||
# every hls.js retry of one, and the sink's limiter still stores 30 a minute of a
|
||||
# template — stdout would get them all.
|
||||
if not s.probe_failure_reported:
|
||||
s.probe_failure_reported = True
|
||||
log.warning(
|
||||
"a session segment could not be checked",
|
||||
extra=logs.event(
|
||||
"plex.stream.segment_probe_failed",
|
||||
key=s.key,
|
||||
seq=seq,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -884,12 +999,16 @@ def wait_for(path: Path, timeout: float = 20.0, proc: subprocess.Popen | None =
|
||||
try:
|
||||
if path.exists() and path.stat().st_size > 0:
|
||||
return True
|
||||
# silent-ok: this is a 150 ms poll — a line here is a line every 150 ms for the whole
|
||||
# timeout. The final check below decides, and the 404 the caller then returns is honest.
|
||||
except OSError:
|
||||
pass
|
||||
if proc is not None and proc.poll() is not None:
|
||||
# the producer exited — no more segments are coming; one last check then give up.
|
||||
try:
|
||||
return path.exists() and path.stat().st_size > 0
|
||||
# silent-ok: one last look after the producer exited. False here becomes a 404 the
|
||||
# player recovers from, which is the honest answer either way.
|
||||
except OSError:
|
||||
return False
|
||||
time.sleep(0.15)
|
||||
@@ -922,7 +1041,16 @@ def sweep_orphans(older_than: float | None = None) -> int:
|
||||
dropped = 0
|
||||
try:
|
||||
entries = list(_HLS_ROOT.iterdir())
|
||||
except OSError: # the root doesn't exist yet — nothing to sweep
|
||||
# silent-ok: nothing has been transcoded yet, so there is nothing to sweep.
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
except OSError as exc:
|
||||
# Any OTHER reason the root will not list is not "nothing to sweep": the sweep reports 0
|
||||
# like a clean run while every crashed session's segments stay on disk forever.
|
||||
log.warning(
|
||||
"the HLS root could not be listed; no orphans were swept",
|
||||
extra=logs.event("plex.stream.sweep_root_unreadable", reason=type(exc).__name__),
|
||||
)
|
||||
return 0
|
||||
for p in entries:
|
||||
if not _SESSION_DIR_RE.match(p.name):
|
||||
@@ -932,7 +1060,9 @@ def sweep_orphans(older_than: float | None = None) -> int:
|
||||
try:
|
||||
if p.stat().st_mtime >= cutoff:
|
||||
continue # this run made (or reused) it — never ours to delete
|
||||
except OSError: # vanished under us (a concurrent restart of that session)
|
||||
# silent-ok: vanished under us — a concurrent restart of that session removed it, which
|
||||
# is the outcome this sweep wanted anyway.
|
||||
except OSError:
|
||||
continue
|
||||
with _lock:
|
||||
if any(s.dir == p for s in _sessions.values()):
|
||||
@@ -968,21 +1098,21 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
now = time.time()
|
||||
dropped = 0
|
||||
with _lock:
|
||||
for key, s in list(_sessions.items()):
|
||||
for reg, s in list(_sessions.items()):
|
||||
done = s.proc.poll() is not None
|
||||
idle = now - s.last_access
|
||||
if idle > _SESSION_IDLE_S or (done and idle > finished_idle_s):
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
_sessions.pop(reg, None)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
|
||||
def touch_session(key: str) -> None:
|
||||
"""Say a viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
def touch_session(user_id: int, key: str) -> None:
|
||||
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||
viewer's session alive was to keep EVERY abandoned one just as long."""
|
||||
with _lock:
|
||||
s = _sessions.get(key)
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
|
||||
+179
-20
@@ -33,17 +33,18 @@ zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths as plex_paths
|
||||
@@ -51,8 +52,7 @@ from app.plex.cache import CACHE_DIR
|
||||
from app.plex.client import PlexClient, PlexError
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Image-based subtitle codecs (Blu-ray/DVD) — can't be turned into text WebVTT (would need burn-in).
|
||||
BITMAP_SUBS = {
|
||||
"pgs",
|
||||
@@ -104,6 +104,7 @@ def shift_vtt(vtt: str, offset: float) -> str:
|
||||
straddling 0 is clamped to start at 0."""
|
||||
if not offset:
|
||||
return vtt
|
||||
unshifted = 0
|
||||
out_blocks: list[str] = []
|
||||
for block in re.split(r"\n[ \t]*\n", vtt):
|
||||
if "-->" not in block:
|
||||
@@ -121,7 +122,12 @@ def shift_vtt(vtt: str, offset: float) -> str:
|
||||
st = _vtt_ts_to_s(left.strip()) - offset
|
||||
en = _vtt_ts_to_s(rparts[0]) - offset
|
||||
except (ValueError, IndexError):
|
||||
break # unparseable timing → leave the block as-is
|
||||
# silent-ok: left as-is means left UNSHIFTED — the viewer sees that one cue at
|
||||
# the wrong moment. Counted rather than logged here: this is per cue, and the file
|
||||
# being parsed is one ffmpeg wrote for us a moment ago, so a broken timestamp is
|
||||
# a fact about the extraction, reported once below.
|
||||
unshifted += 1
|
||||
break
|
||||
if en <= 0:
|
||||
keep = False # whole cue is before the session start → drop the block
|
||||
else:
|
||||
@@ -131,6 +137,11 @@ def shift_vtt(vtt: str, offset: float) -> str:
|
||||
break
|
||||
if keep:
|
||||
out_blocks.append("\n".join(lines))
|
||||
if unshifted:
|
||||
log.info(
|
||||
"some cues could not be shifted and stay at their original time",
|
||||
extra=logs.event("plex.subtitles.cues_unshifted", cues=unshifted, offset_s=offset),
|
||||
)
|
||||
return "\n\n".join(out_blocks)
|
||||
|
||||
|
||||
@@ -162,7 +173,15 @@ def cache_read(key: str) -> str | None:
|
||||
try:
|
||||
p = CACHE_DIR / f"{key}.vtt"
|
||||
return p.read_text(encoding="utf-8") if p.exists() else None
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# None is the same answer as "never cached", so a cache we cannot read looks exactly like
|
||||
# a cold one — and every request re-extracts instead of saying the disk is the problem.
|
||||
log.warning(
|
||||
"the cached subtitle could not be read",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.cache_read_failed", key=key, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -175,7 +194,16 @@ def cache_write(key: str, vtt: str) -> bool:
|
||||
tmp.write_text(vtt, encoding="utf-8")
|
||||
tmp.replace(CACHE_DIR / f"{key}.vtt") # atomic publish
|
||||
return True
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# `publish` turns False into "the region was not claimed" and moves on, so the track just
|
||||
# never appears. Which errno it was — full disk, read-only mount, permissions — is the
|
||||
# whole diagnosis, and it exists only here.
|
||||
log.warning(
|
||||
"the subtitle cache could not be written",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.cache_write_failed", key=key, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -206,7 +234,17 @@ def read_meta(key: str) -> tuple[list[list[float]], bool]:
|
||||
try:
|
||||
raw = json.loads(_meta_path(key).read_text(encoding="utf-8"))
|
||||
return [[float(a), float(b)] for a, b in raw.get("ranges", [])], bool(raw.get("complete"))
|
||||
except (OSError, ValueError, TypeError):
|
||||
# silent-ok: no sidecar at all is the documented older-build case the fallback exists for.
|
||||
except FileNotFoundError:
|
||||
return ([[0.0, _FULL_END_S]], True) if _exists(_vtt_path(key)) else ([], False)
|
||||
except (OSError, ValueError, TypeError) as exc:
|
||||
# A sidecar that EXISTS and will not parse is different, and the fallback is the dangerous
|
||||
# answer to it: beside a cached VTT it reports the track as complete, so a partial track is
|
||||
# never extended again and the viewer loses every cue past the window that was cached.
|
||||
log.warning(
|
||||
"the subtitle sidecar could not be read; the cached track is taken as complete",
|
||||
extra=logs.event("plex.subtitles.meta_unreadable", key=key, reason=type(exc).__name__),
|
||||
)
|
||||
return ([[0.0, _FULL_END_S]], True) if _exists(_vtt_path(key)) else ([], False)
|
||||
|
||||
|
||||
@@ -216,6 +254,8 @@ def _exists(p: Path) -> bool:
|
||||
escaped as a 500 from the status route."""
|
||||
try:
|
||||
return p.exists()
|
||||
# silent-ok: this function's whole job is to answer False for a path the OS refuses. The
|
||||
# refusal IS the return value; a caller that needs to report one reports it itself.
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
@@ -226,8 +266,18 @@ def _write_meta(key: str, ranges: list[list[float]], complete: bool) -> None:
|
||||
tmp = CACHE_DIR / f"{key}.json.tmp"
|
||||
tmp.write_text(json.dumps({"ranges": ranges, "complete": complete}), encoding="utf-8")
|
||||
tmp.replace(_meta_path(key))
|
||||
except OSError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
# Without the sidecar beside it, a cached window reads back as a COMPLETE track (see
|
||||
# `read_meta`) — so one failed write here permanently costs the rest of that subtitle.
|
||||
log.warning(
|
||||
"the subtitle sidecar could not be written",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.meta_write_failed",
|
||||
key=key,
|
||||
complete=complete,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def generation(key: str) -> int:
|
||||
@@ -235,7 +285,9 @@ def generation(key: str) -> int:
|
||||
window is superseded by a wider one the browser refetches instead of keeping the short VTT."""
|
||||
try:
|
||||
return _vtt_path(key).stat().st_mtime_ns
|
||||
except (OSError, ValueError): # ValueError: a NUL in the key — `stat` refuses the path outright
|
||||
# silent-ok: 0 for a track that is not cached yet, which is the ordinary case on every first
|
||||
# request. (ValueError: a NUL in the key — `stat` refuses the path outright.)
|
||||
except (OSError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
@@ -267,16 +319,27 @@ def merge_vtt(*sources: str) -> str:
|
||||
happen to share a time (two speakers, a sign over dialogue) are both kept."""
|
||||
seen: set[str] = set()
|
||||
cues: list[tuple[float, str]] = []
|
||||
dropped = 0
|
||||
for src in sources:
|
||||
for block in re.split(r"\n[ \t]*\n", src or ""):
|
||||
canon = _canonical_cue(block.strip())
|
||||
stripped = block.strip()
|
||||
canon = _canonical_cue(stripped)
|
||||
if canon is None:
|
||||
# A block WITH a timing line that still would not parse is a cue we are throwing
|
||||
# away, not a header — and the merged track would simply be missing those lines.
|
||||
if "-->" in stripped:
|
||||
dropped += 1
|
||||
continue # header / NOTE / STYLE — the merged file gets its own single header
|
||||
start, key, text = canon
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
cues.append((start, text))
|
||||
if dropped:
|
||||
log.info(
|
||||
"some cues were dropped from the merged subtitle: their timing would not parse",
|
||||
extra=logs.event("plex.subtitles.cues_dropped", cues=dropped, kept=len(cues)),
|
||||
)
|
||||
cues.sort(key=lambda c: c[0])
|
||||
return "WEBVTT\n\n" + "\n\n".join(b for _, b in cues) + "\n"
|
||||
|
||||
@@ -295,6 +358,8 @@ def _canonical_cue(block: str) -> tuple[float, str, str] | None:
|
||||
try:
|
||||
st = _vtt_ts_to_s(left.strip())
|
||||
en = _vtt_ts_to_s(rparts[0])
|
||||
# silent-ok: None here means "not a cue", and the caller tells a header apart from a
|
||||
# broken timing line by whether the block had a `-->` at all — see `merge_vtt`.
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
lines[i] = f"{_vtt_s_to_ts(st)} --> {_vtt_s_to_ts(en)}{settings}"
|
||||
@@ -310,6 +375,8 @@ def _last_cue_end(vtt: str) -> float:
|
||||
continue
|
||||
try:
|
||||
end = max(end, _vtt_ts_to_s(ln.partition("-->")[2].strip().split(None, 1)[0]))
|
||||
# silent-ok: this is a floor — "how far the pass provably got". Skipping a line it cannot
|
||||
# read makes the answer smaller, never larger, so the error is always on the safe side.
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
return end
|
||||
@@ -394,11 +461,18 @@ _FULL_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="plex-subs-ful
|
||||
# which time something has finished.
|
||||
_MAX_WINDOWS_INFLIGHT = 8
|
||||
_MAX_FULL_INFLIGHT = 6
|
||||
# …of which one account may hold this many. The full-pass slots are minutes each, and a hostile (or
|
||||
# merely curious) account opening subtitles on film after film could otherwise hold all of them while
|
||||
# everyone else's tracks never completed. A viewer over their share still gets windows — the part
|
||||
# that makes a track watchable now — just not another whole-file read until one of theirs ends.
|
||||
_MAX_FULL_PER_USER = 3
|
||||
_MAX_TRACK_WINDOWS = 2
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
# Queued or running jobs → the absolute span a window job reads (None for a full pass).
|
||||
_INFLIGHT: dict[tuple, tuple[float, float] | None] = {}
|
||||
# rating_key → the account whose poll queued that item's full pass (see _MAX_FULL_PER_USER).
|
||||
_FULL_OWNERS: dict[str, int] = {}
|
||||
# Track-level failures: a track that cannot be served at all (no item, a bitmap track, Plex down…).
|
||||
_FAILURES: dict[str, tuple[str, float]] = {}
|
||||
# rating_key → when its full pass last failed (see FULL_RETRY_S).
|
||||
@@ -448,6 +522,7 @@ def reset_state() -> None:
|
||||
global _STOPPING
|
||||
with _LOCK:
|
||||
_INFLIGHT.clear()
|
||||
_FULL_OWNERS.clear()
|
||||
_FAILURES.clear()
|
||||
_FULL_FAILED.clear()
|
||||
_WIN_FAILED.clear()
|
||||
@@ -531,7 +606,7 @@ def _window_failed(key: str, at: float) -> bool:
|
||||
return any(a <= at <= b for a, b, _ in _live_window_failures(key))
|
||||
|
||||
|
||||
def ensure(rating_key: str, sub_ord: int, at: float = 0.0) -> dict:
|
||||
def ensure(rating_key: str, sub_ord: int, at: float = 0.0, viewer: int = 0) -> dict:
|
||||
"""State of one subtitle track, scheduling whatever extraction is still missing. NEVER blocks.
|
||||
|
||||
``ready`` = fully extracted · ``partial`` = usable, covers ``at``, still growing · ``preparing``
|
||||
@@ -574,8 +649,23 @@ def ensure(rating_key: str, sub_ord: int, at: float = 0.0) -> dict:
|
||||
if edge - at <= LOOKAHEAD_S and edge < _FULL_END_S:
|
||||
probe = edge + 1.0
|
||||
want_window = not _window_failed(key, probe)
|
||||
_schedule(rating_key, sub_ord, probe, want_window=want_window, want_full=not full_failed)
|
||||
return {"state": "partial" if covered else "preparing", "code": None, "gen": gen}
|
||||
_schedule(
|
||||
rating_key,
|
||||
sub_ord,
|
||||
probe,
|
||||
want_window=want_window,
|
||||
want_full=not full_failed,
|
||||
viewer=viewer,
|
||||
)
|
||||
# `growing`: whether the track will fill in on its own. False once the full pass has failed and
|
||||
# is backed off — a partial track then stays partial until a window is asked for, so the player
|
||||
# may poll slowly (only to read ahead of an edge) instead of every two seconds for the whole film.
|
||||
return {
|
||||
"state": "partial" if covered else "preparing",
|
||||
"code": None,
|
||||
"gen": gen,
|
||||
"growing": not full_failed,
|
||||
}
|
||||
|
||||
|
||||
def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]:
|
||||
@@ -592,7 +682,12 @@ def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]
|
||||
|
||||
|
||||
def _schedule(
|
||||
rating_key: str, sub_ord: int, at: float, want_window: bool, want_full: bool = True
|
||||
rating_key: str,
|
||||
sub_ord: int,
|
||||
at: float,
|
||||
want_window: bool,
|
||||
want_full: bool = True,
|
||||
viewer: int = 0,
|
||||
) -> None:
|
||||
"""Kick off the jobs this track still needs: the window that makes it watchable now, and the one
|
||||
full pass that makes it complete. A 1 s status poll (or a second viewer) joins a job already
|
||||
@@ -625,8 +720,15 @@ def _schedule(
|
||||
ident = ("win", rating_key, sub_ord, w_start)
|
||||
_INFLIGHT[ident] = (w_start, w_start + WINDOW_PRE_S + WINDOW_LEN_S)
|
||||
jobs.append((ident, w_start))
|
||||
if want_full and n_full < _MAX_FULL_INFLIGHT and ("full", rating_key) not in _INFLIGHT:
|
||||
mine = sum(1 for owner in _FULL_OWNERS.values() if owner == viewer)
|
||||
if (
|
||||
want_full
|
||||
and n_full < _MAX_FULL_INFLIGHT
|
||||
and ("full", rating_key) not in _INFLIGHT
|
||||
and mine < _MAX_FULL_PER_USER
|
||||
):
|
||||
_INFLIGHT[("full", rating_key)] = None
|
||||
_FULL_OWNERS[rating_key] = viewer
|
||||
jobs.append((("full", rating_key), None))
|
||||
for ident, window_start in jobs:
|
||||
pool = _FULL_POOL if window_start is None else _WIN_POOL
|
||||
@@ -652,12 +754,35 @@ def _run(
|
||||
return
|
||||
_extract_and_publish(rating_key, sub_ord, window_start)
|
||||
except SubtitleUnavailable as e:
|
||||
# The player's own state was the only place this ever showed. `_pass_failed` backs the item
|
||||
# off for an hour and `_fail` records a code, but neither leaves a line, so "my subtitles
|
||||
# never appear" had nothing on the server to read.
|
||||
# A final code is a fact about the file (no such track, a bitmap subtitle); anything else
|
||||
# is this instance failing at something it could do.
|
||||
log.log(
|
||||
logging.INFO if e.code in FINAL_CODES else logging.WARNING,
|
||||
"subtitle extraction gave up",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.pass_failed",
|
||||
rating_key=rating_key,
|
||||
sub_ord=sub_ord,
|
||||
code=e.code,
|
||||
windowed=window_start is not None,
|
||||
),
|
||||
)
|
||||
if e.code == "extract_failed":
|
||||
_pass_failed(rating_key, [key], window_start)
|
||||
else:
|
||||
_fail(key, e.code)
|
||||
except PlexError as e:
|
||||
log.warning("subtitle extraction failed for %s/%s: %s", rating_key, sub_ord, e)
|
||||
# No traceback on purpose: a PlexError carries httpx's own message, which names the method,
|
||||
# the URL and the status — the stack under it is this `except` and the client call, and
|
||||
# says nothing the message does not. (The URL holds no token; it travels as a header.)
|
||||
log.warning(
|
||||
"subtitle extraction failed talking to Plex: %s",
|
||||
e,
|
||||
extra=logs.event("plex.subtitles.plex_error", rating_key=rating_key, sub_ord=sub_ord),
|
||||
)
|
||||
_fail(key, "plex_error")
|
||||
except (
|
||||
Exception
|
||||
@@ -667,6 +792,8 @@ def _run(
|
||||
finally:
|
||||
with _LOCK:
|
||||
_INFLIGHT.pop(ident, None)
|
||||
if ident[0] == "full":
|
||||
_FULL_OWNERS.pop(rating_key, None)
|
||||
|
||||
|
||||
def _pass_failed(
|
||||
@@ -732,10 +859,13 @@ def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | No
|
||||
for i, s in enumerate(subs)
|
||||
if s.get("index") is not None and not _is_bitmap(s)
|
||||
}
|
||||
start = window_start or 0.0
|
||||
# A full pass starts where every track's leading coverage ends: a pass that ran out of time
|
||||
# (`EXTRACT_TIMEOUT_S`) kept what it reached, and restarting from 0 each `FULL_RETRY_S` meant a file
|
||||
# too big to read in one go never completed. Windows handle an offset start already; so does this.
|
||||
start = window_start if window_start is not None else _full_pass_start(rating_key, targets)
|
||||
length = (WINDOW_LEN_S + WINDOW_PRE_S) if window_start is not None else None
|
||||
end = _FULL_END_S if length is None else start + length
|
||||
texts, clean = _ffmpeg_extract(str(src), sorted(targets), window_start, length)
|
||||
texts, clean = _ffmpeg_extract(str(src), sorted(targets), start or None, length)
|
||||
for emb_ord, vtt in texts.items():
|
||||
key = cache_key(rating_key, targets[emb_ord])
|
||||
# A pass that exited non-zero holds only what ffmpeg reached before it died. Claim no more
|
||||
@@ -756,6 +886,24 @@ def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | No
|
||||
_pass_failed(rating_key, [key], window_start, (covered_end, end))
|
||||
|
||||
|
||||
def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
||||
"""Where the full pass may begin: the end of the span from 0 that EVERY target track already has
|
||||
(one read feeds them all, so the least-covered track decides), backed off by `WINDOW_PRE_S` so a
|
||||
cue running across the join is read whole — `merge_vtt` folds the overlap. 0 when any track has
|
||||
nothing from the start."""
|
||||
ends = []
|
||||
for ord_ in targets.values():
|
||||
key = cache_key(rating_key, ord_)
|
||||
ranges, _ = read_meta(key) if _exists(_vtt_path(key)) else ([], False)
|
||||
lead = [b for a, b in ranges if a <= 0.0]
|
||||
if not lead:
|
||||
return 0.0
|
||||
ends.append(max(lead))
|
||||
if not ends:
|
||||
return 0.0
|
||||
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
||||
|
||||
|
||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
||||
with PlexClient(db) as plex:
|
||||
@@ -794,9 +942,10 @@ def _ffmpeg_extract(
|
||||
cmd += ["-map", f"0:s:{emb_ord}"]
|
||||
if length is not None:
|
||||
cmd += ["-t", f"{length:.3f}"]
|
||||
if start:
|
||||
# The input seek lands on the keyframe at or BEFORE `start`, so a cue already running
|
||||
# at `start` comes out at a negative time — and by default the muxer then shifts the
|
||||
# WHOLE output so that cue sits at 0: every cue of the window late by the distance to
|
||||
# WHOLE output so that cue sits at 0: every cue of the read late by the distance to
|
||||
# that keyframe (1–5 s measured on a real library file, a whole GOP on a synthetic
|
||||
# one), and doubled for good once the full pass merged in beside it. Kept negative, the
|
||||
# times are exact on the zero-based clock (`_vtt_ts_to_s` reads the sign).
|
||||
@@ -829,6 +978,16 @@ def _ffmpeg_extract(
|
||||
try:
|
||||
_, err = proc.communicate(timeout=_TERM_GRACE_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
# TERM is what makes a timed-out pass worth keeping: ffmpeg closes its outputs
|
||||
# on it. An ffmpeg that ignores TERM leaves only whole blocks already flushed,
|
||||
# so the partial track is short for a reason the later line cannot tell apart.
|
||||
log.info(
|
||||
"the subtitle extraction ignored TERM and was killed",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.extract_killed_after_term",
|
||||
windowed=start is not None,
|
||||
),
|
||||
)
|
||||
proc.kill()
|
||||
_, err = proc.communicate()
|
||||
finally:
|
||||
|
||||
+24
-12
@@ -15,20 +15,18 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
||||
item-detail/player path (P2), not here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import detail, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||
_BROWSER_VIDEO = {"h264"}
|
||||
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
||||
@@ -102,6 +100,9 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
||||
bd = s.get("bitDepth")
|
||||
try:
|
||||
depth = int(bd) if bd else None
|
||||
# silent-ok: unknown bit depth, which is also what most listings report. The
|
||||
# playback class then falls back to the codec alone, and per-stream noise over a
|
||||
# whole library would drown the sync's own lines.
|
||||
except (TypeError, ValueError):
|
||||
depth = None
|
||||
hdr = str(s.get("colorTrc") or "").lower() in _HDR_TRANSFERS
|
||||
@@ -112,6 +113,8 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
||||
def _epoch(v) -> datetime | None:
|
||||
try:
|
||||
return datetime.fromtimestamp(int(v), tz=timezone.utc) if v else None
|
||||
# silent-ok: a field Plex leaves out of most listing rows. None is a legitimate stored value,
|
||||
# and every caller treats it as "not known" rather than as an error.
|
||||
except (TypeError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
@@ -120,6 +123,7 @@ def _date(v):
|
||||
"""Parse a Plex 'YYYY-MM-DD' string (originallyAvailableAt) → date, or None."""
|
||||
try:
|
||||
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
||||
# silent-ok: as `_epoch` above — an absent or unparsable air date is stored as no air date.
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -135,6 +139,7 @@ def _tags(meta: dict, key: str, limit: int | None = None) -> list | None:
|
||||
def _rating(meta: dict) -> float | None:
|
||||
try:
|
||||
return float(meta.get("audienceRating") or meta.get("rating"))
|
||||
# silent-ok: TypeError on `float(None)` is the ordinary path — most items carry no rating.
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -266,15 +271,19 @@ def sync(db: Session) -> dict:
|
||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||
try:
|
||||
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
||||
except Exception as e: # best-effort — never fail the sync over the backfill
|
||||
log.warning("Plex people enrich batch failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
||||
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
||||
log.exception("the Plex people-enrich batch failed")
|
||||
# Probe a bounded batch of HW-decodable transcode items for bit-depth/HDR, so the
|
||||
# playback-class filter's HW/SW split is precise (a 10-bit/HDR HEVC → software) over
|
||||
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
||||
try:
|
||||
stats["media_probed"] = _probe_media_batch(db, plex)
|
||||
except Exception as e: # best-effort — never fail the sync over the backfill
|
||||
log.warning("Plex media probe batch failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
log.exception("the Plex media-probe batch failed")
|
||||
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
||||
# logs the job result, and to the admin route, which answers it to the caller.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -285,8 +294,8 @@ def sync(db: Session) -> dict:
|
||||
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
||||
try:
|
||||
stats["people_pruned"] = people.prune_orphan_people(db)
|
||||
except Exception as e: # best-effort housekeeping — never fail the sync over it
|
||||
log.warning("Plex people prune failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort housekeeping; never fail the sync over it
|
||||
log.exception("the Plex people prune failed")
|
||||
log.info("Plex sync done: %s", stats)
|
||||
return stats
|
||||
|
||||
@@ -325,7 +334,9 @@ def _probe_media_batch(db: Session, plex: PlexClient, limit: int = _PROBE_BATCH)
|
||||
for row in rows:
|
||||
try:
|
||||
meta = plex.metadata(row.rating_key) or {}
|
||||
except Exception as e: # transient — leave media_probed_at NULL so it retries next run
|
||||
# No traceback and only debug: this is per item over a whole library, and the stack is
|
||||
# the same `plex.metadata` call every time. The retry next run is the handling.
|
||||
except Exception as e: # noqa: BLE001 — transient; leave media_probed_at NULL to retry
|
||||
log.debug("Plex media probe fetch failed for %s: %s", row.rating_key, e)
|
||||
continue
|
||||
bit_depth, hdr = _video_stream_props(meta)
|
||||
@@ -372,7 +383,8 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
|
||||
for row in [*movies, *shows]:
|
||||
try:
|
||||
meta = plex.metadata(row.rating_key) or {}
|
||||
except Exception as e:
|
||||
# As in `_probe_media_batch` above: per item, so the message without the stack.
|
||||
except Exception as e: # noqa: BLE001 — transient; leave people_enriched_at NULL
|
||||
log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e)
|
||||
continue # leave people_enriched_at NULL so it's retried on a later run
|
||||
people.enrich_row_people(db, row, people.cast_from_meta(meta))
|
||||
|
||||
@@ -10,7 +10,6 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -19,12 +18,11 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
|
||||
log = logging.getLogger("siftlode.tmdb")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
_MIN_INTERVAL = 0.03
|
||||
|
||||
@@ -142,6 +140,12 @@ def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int
|
||||
try:
|
||||
return int(ext)
|
||||
except ValueError:
|
||||
# The caller stamps the watermark on a None, so this title is never looked at again:
|
||||
# one unreadable guid costs it its artwork and its cast permanently.
|
||||
log.info(
|
||||
"a Plex guid named TMDB but did not carry a numeric id",
|
||||
extra=logs.event("plex.tmdb.guid_unparsable", provider=provider),
|
||||
)
|
||||
return None
|
||||
source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider)
|
||||
if not source:
|
||||
@@ -499,7 +503,17 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
# doesn't tie up a threadpool worker for the default 30s.
|
||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
||||
except TmdbError:
|
||||
except TmdbError as exc:
|
||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||
# front of the user right now shows an empty biography with nothing to say why.
|
||||
log.warning(
|
||||
"the TMDB person lookup failed",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.person_enrich_failed",
|
||||
person_id=person.id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return False
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
|
||||
@@ -17,14 +17,17 @@ and the incremental pull resolves a genuine divergence to Plex's state (see `_pu
|
||||
running a last-write-wins comparison.
|
||||
"""
|
||||
|
||||
import logging
|
||||
# On the PlexError lines in this module: none of them carries a traceback, deliberately. A
|
||||
# PlexError is httpx's own message — the method, the URL and the status of one call — and the stack
|
||||
# beneath it is always this `except` plus `PlexClient._get`. The attrs beside the line say which
|
||||
# user and which phase, which is the part the message cannot know.
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem, PlexLink, PlexState
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
@@ -36,8 +39,7 @@ from app.plex.sync import (
|
||||
_paginate,
|
||||
)
|
||||
|
||||
log = logging.getLogger("siftlode.plex.watch")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# A resume position below this many seconds isn't worth mirroring — matches the player's own
|
||||
# _PROGRESS_MIN_S (a stray few-second seek is not "in progress").
|
||||
_PROGRESS_MIN_S = 5
|
||||
@@ -148,6 +150,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||
link.initial_import_done = True
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report — the admin route answers it and the
|
||||
# scheduler logs it on the job line.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -326,8 +330,18 @@ def _resolve_account_id(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
if int(a.get("id") or -1) == 1:
|
||||
username = a.get("name")
|
||||
break
|
||||
except PlexError:
|
||||
pass
|
||||
except PlexError as exc:
|
||||
# The fallback account id 1 is right for the admin-token setup, so sync keeps working —
|
||||
# but the username is never stored, and the UI then has no name to show for the link,
|
||||
# exactly as if the server had not offered one.
|
||||
log.info(
|
||||
"the Plex account list could not be read; the link keeps the default account",
|
||||
extra=logs.event(
|
||||
"plex.watch.accounts_probe_failed",
|
||||
user_id=link.user_id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
link.plex_account_id = acct_id
|
||||
if username:
|
||||
link.plex_username = username
|
||||
@@ -566,6 +580,7 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = now
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report, as above.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -617,6 +632,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report, as above.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
|
||||
@@ -37,6 +37,7 @@ class QuotaAction:
|
||||
PLAYLISTS_DELETE = "playlists_delete"
|
||||
VIDEOS_BACKFILL_RECENT = "videos_backfill_recent"
|
||||
VIDEOS_BACKFILL_FULL = "videos_backfill_full"
|
||||
VIDEOS_FEED_FALLBACK = "videos_feed_fallback"
|
||||
VIDEOS_ENRICH = "videos_enrich"
|
||||
VIDEOS_LOOKUP = "videos_lookup"
|
||||
VIDEOS_SEARCH = "videos_search"
|
||||
|
||||
+67
-11
@@ -1,21 +1,30 @@
|
||||
"""A tiny in-process sliding-window rate limiter.
|
||||
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client IP), so it can throttle any
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client — `client_key`), so it can throttle any
|
||||
endpoint. Single-worker uvicorn → in-memory state is sufficient; it resets on restart, which
|
||||
is fine for abuse throttling (not for anything that must survive a deploy). Not shared across
|
||||
processes — if we ever run multiple workers, swap the backing store for Redis behind the same
|
||||
``allow()`` interface.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import threading
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, max_events: int, window_seconds: float):
|
||||
"""At most `max_events` attempts per `window_seconds` per key — or, when each attempt passes a
|
||||
`cost`, at most that much total cost (a byte budget counts bytes, not calls)."""
|
||||
|
||||
def __init__(self, max_events: float, window_seconds: float):
|
||||
self.max_events = max_events
|
||||
self.window = window_seconds
|
||||
self._hits: dict[str, list[float]] = {}
|
||||
# key → (monotonic time, cost) of every counted attempt still inside the window
|
||||
self._hits: dict[str, list[tuple[float, float]]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def reset(self) -> None:
|
||||
@@ -25,27 +34,74 @@ class RateLimiter:
|
||||
with self._lock:
|
||||
self._hits.clear()
|
||||
|
||||
def allow(self, key: str) -> bool:
|
||||
"""Record an attempt for ``key`` and return whether it is within the limit.
|
||||
def exhausted(self, key: str) -> bool:
|
||||
"""Whether ``key``'s window is already full (``allow`` would refuse it), WITHOUT counting an
|
||||
attempt — for a limit that counts only some outcomes: ask it first, ``allow`` the ones that
|
||||
count."""
|
||||
cutoff = time.monotonic() - self.window
|
||||
with self._lock:
|
||||
return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events
|
||||
|
||||
True -> under the cap (the attempt is counted).
|
||||
False -> the cap for the current window is already reached (attempt NOT counted, so a
|
||||
def allow(self, key: str, cost: float = 1) -> bool:
|
||||
"""Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit.
|
||||
|
||||
True -> within the cap (the attempt is counted).
|
||||
False -> the attempt would pass the cap for the current window (attempt NOT counted, so a
|
||||
blocked caller can't keep pushing the window forward)."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.window
|
||||
with self._lock:
|
||||
hits = [t for t in self._hits.get(key, []) if t > cutoff]
|
||||
if len(hits) >= self.max_events:
|
||||
hits = [(t, c) for t, c in self._hits.get(key, []) if t > cutoff]
|
||||
if sum(c for _, c in hits) + cost > self.max_events:
|
||||
self._hits[key] = hits
|
||||
return False
|
||||
hits.append(now)
|
||||
hits.append((now, cost))
|
||||
self._hits[key] = hits
|
||||
# Opportunistic cleanup so abandoned keys don't accumulate unboundedly.
|
||||
if len(self._hits) > 4096:
|
||||
for k in list(self._hits):
|
||||
fresh = [t for t in self._hits[k] if t > cutoff]
|
||||
fresh = [(t, c) for t, c in self._hits[k] if t > cutoff]
|
||||
if fresh:
|
||||
self._hits[k] = fresh
|
||||
else:
|
||||
del self._hits[k]
|
||||
return True
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
# silent-ok: parsing an untrusted header value, where "not an IP" is a documented answer this
|
||||
# function returns — and a line per request is what an attacker would aim for.
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def client_key(request: Request) -> str:
|
||||
"""Best-effort client identity for rate limiting: its IP, or for IPv6 the /64 the address sits in — a
|
||||
subscriber is handed a whole /64, so keyed on the full address one client would have endless identities
|
||||
to spread its attempts over. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
client = peer
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
client = parts[-1]
|
||||
ip = _norm_ip(client)
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
return str(ipaddress.IPv6Network((ip, 64), strict=False))
|
||||
return client
|
||||
|
||||
+13
-6
@@ -13,12 +13,12 @@ mutated from a single thread.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.websockets import WebSocket
|
||||
from app import logs
|
||||
|
||||
log = logging.getLogger("siftlode.realtime")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
@@ -42,9 +42,16 @@ class ConnectionManager:
|
||||
for ws in list(self._conns.get(user_id, ())):
|
||||
try:
|
||||
await ws.send_json(payload)
|
||||
except Exception:
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler.
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — one dead socket must not stop the others
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler. But this is
|
||||
# also where a live update goes missing: the UI simply never changes, and there
|
||||
# is no request behind it whose failure anyone could look up.
|
||||
log.debug(
|
||||
"a live update could not be pushed to a socket",
|
||||
extra=logs.event(
|
||||
"realtime.push_failed", user_id=user_id, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
|
||||
def push(self, user_id: int, payload: dict) -> None:
|
||||
"""Schedule a push onto the WebSocket event loop. Safe to call from a sync route
|
||||
@@ -54,7 +61,7 @@ class ConnectionManager:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._push(user_id, payload), loop)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a closed loop or socket must not fail the caller
|
||||
log.debug("ws push failed", exc_info=True)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""A request the app refuses, said in a way the UI can put in the reader's language.
|
||||
|
||||
Every refusal the API answers carries a structured `detail` — `{code, reason, …params}` — and the client
|
||||
translates it (`lib/api/core.ts` `localizeDetail`). A bare English string reached the Hungarian UI
|
||||
verbatim: the blocking error dialog, and every inline surface that printed `HttpError.detail`, showed
|
||||
whatever sentence the route happened to hand-write. `refused` is the generic family
|
||||
(`errors.refused.<reason>`, the params interpolated into the sentence); the older specific codes
|
||||
(`not_found`, `youtube`, `playlist`, the admin refusals, …) stay, since each is already translated.
|
||||
|
||||
A test (`test_http_details.py`) reads every `HTTPException` in the app and fails on a string detail,
|
||||
and the frontend's contract test requires every `refused(…, "<reason>")` to be named in both
|
||||
languages — so the rule is enforced, not described.
|
||||
"""
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
def refused(status: int, reason: str, **params: object) -> HTTPException:
|
||||
"""The refusal to raise: `raise refused(409, "view_name_taken", name=name)`. `params` travel in
|
||||
the detail for the sentence to interpolate (`{{name}}`); keep them to values the reader may see."""
|
||||
return HTTPException(status_code=status, detail={"code": "refused", "reason": reason, **params})
|
||||
|
||||
|
||||
# `sysconfig.InvalidSetting.reason` → the refusal it answers as. A dict of literals, so the frontend's
|
||||
# contract test can read every reason this can produce.
|
||||
_SETTING_REASONS = {
|
||||
"not_a_number": "setting_not_a_number",
|
||||
"too_low": "setting_too_low",
|
||||
"too_high": "setting_too_high",
|
||||
"not_a_choice": "setting_not_a_choice",
|
||||
"no_encryption_key": "no_encryption_key",
|
||||
}
|
||||
|
||||
|
||||
def setting_refused(exc) -> HTTPException:
|
||||
"""A value the admin entered that `sysconfig.set_value` refused (`exc`: its `InvalidSetting`)."""
|
||||
return refused(400, _SETTING_REASONS[exc.reason], key=exc.key, **exc.params)
|
||||
@@ -8,11 +8,14 @@ one existence-leak policy.
|
||||
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi import HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
@@ -45,9 +48,53 @@ def owned_or_404(
|
||||
) -> M:
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
Generalizes the near-identical per-route ownership helpers (e.g. plex's
|
||||
`_own_playlist_or_404`, the saved-views owner check)."""
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
def youtube_refusal(exc: Exception) -> HTTPException:
|
||||
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
||||
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
||||
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
||||
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
||||
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
||||
YouTube being unavailable to us — the retryable answer."""
|
||||
reason = refusal_reason(exc) if isinstance(exc, YouTubeError) else "unavailable"
|
||||
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
||||
|
||||
|
||||
async def capped_body(request: Request, limit: int) -> bytes:
|
||||
"""The request body, refused (413 `too_large`) the moment it passes `limit` bytes while still
|
||||
arriving. Counted, not taken from Content-Length: a declared length can be absent (chunked) or
|
||||
wrong, and counting stops reading at the same point a declared-length check would have."""
|
||||
body = bytearray()
|
||||
async for chunk in request.stream():
|
||||
body += chunk
|
||||
if len(body) > limit:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
return bytes(body)
|
||||
|
||||
|
||||
def issue_refused(exc: gitea.GiteaError) -> HTTPException:
|
||||
"""The refusal an issue-tracker failure (E16) answers as — the send, the preview and the
|
||||
admin's connection test alike. Literal reasons, one per case: the frontend's contract test reads
|
||||
them to require a sentence for each in both languages."""
|
||||
match exc.reason:
|
||||
case "not_configured":
|
||||
return refused(400, "issues_not_configured")
|
||||
case "unreachable":
|
||||
return refused(422, "issues_unreachable")
|
||||
case "bad_token":
|
||||
return refused(422, "issues_bad_token")
|
||||
case "repo_not_found":
|
||||
return refused(422, "issues_repo_not_found")
|
||||
case "public_repo":
|
||||
return refused(422, "issues_public_repo")
|
||||
case "issues_disabled":
|
||||
return refused(422, "issues_disabled")
|
||||
case _:
|
||||
return refused(422, "issues_rejected")
|
||||
|
||||
+29
-25
@@ -39,6 +39,7 @@ from app.routes.auth import (
|
||||
send_verification_link,
|
||||
)
|
||||
from app.utils import canonical_email, iso, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
@@ -169,7 +170,7 @@ def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
user = next((u for u in candidates if u.email == email), None) or next(iter(candidates), None)
|
||||
if user is not None and not user.is_active:
|
||||
user.is_active = True
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
|
||||
|
||||
def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
@@ -179,7 +180,7 @@ def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
# sending a duplicate decision mail, even flipping an already-approved invite to "denied" and
|
||||
# mailing a rejection to an already-active account.
|
||||
if inv.status != "pending":
|
||||
raise HTTPException(status_code=409, detail="Invite already decided")
|
||||
raise refused(409, "invite_decided")
|
||||
inv.status = "approved" if approved else "denied"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
@@ -191,7 +192,7 @@ def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
target_id=inv.id,
|
||||
summary=inv.email,
|
||||
)
|
||||
db.commit()
|
||||
db.flush() # the handler commits, with the activation that goes with it
|
||||
return inv
|
||||
|
||||
|
||||
@@ -210,6 +211,9 @@ def approve_invite(
|
||||
link, has_account, hours = _welcome_mail_parts(db, inv.email)
|
||||
lang = mailtext.lang_for_email(db, inv.email)
|
||||
background.add_task(email_mod.send_access_approved, inv.email, lang, link, has_account, hours)
|
||||
# Before the response, so the mail (a background task, after it) never describes a decision
|
||||
# the database does not hold yet.
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@@ -226,6 +230,7 @@ def deny_invite(
|
||||
background.add_task(
|
||||
email_mod.send_access_denied, inv.email, mailtext.lang_for_email(db, inv.email)
|
||||
)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@@ -242,7 +247,7 @@ def add_invite(
|
||||
"""Manually whitelist an email (approved straight away). Convenience for the admin."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
# Reuse a canonically-equal row instead of adding a second one for the same mailbox. Prod carries
|
||||
# exactly that pair for one person from 2026-06-16 (hartmann.peter1986@ / hartmannpeter1986@),
|
||||
# which is how an admin ends up approving the same human twice and still seeing them bounce.
|
||||
@@ -259,11 +264,13 @@ def add_invite(
|
||||
inv.status = "approved"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
db.flush() # a new invite's id, for the audit row — before it, the row named no target
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.INVITE_ADD, target_type="invite", target_id=inv.id, summary=email
|
||||
)
|
||||
db.commit()
|
||||
# One transaction: the whitelisting and the activation it implies land together or not at all.
|
||||
_activate_user_by_email(db, email)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [email])))
|
||||
|
||||
|
||||
@@ -366,10 +373,7 @@ def delete_invite(
|
||||
promoted to admin on its next Google sign-in). The admin has to hear that, not guess it."""
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if _accounts_for(db, [inv.email]):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="That address already has an account — delete the account instead.",
|
||||
)
|
||||
raise refused(409, "address_has_account")
|
||||
still_allowed_by_env = _env_allowed(inv.email)
|
||||
audit.record(
|
||||
db,
|
||||
@@ -408,6 +412,8 @@ def _serialize_user(u: User) -> dict:
|
||||
# confusing sign-in.
|
||||
"google_email": u.google_email if u.google_email != u.email else None,
|
||||
"created_at": iso(u.created_at),
|
||||
# NULL = no sign-in since the column existed (0085), or the shared demo account.
|
||||
"last_login_at": iso(u.last_login_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -433,19 +439,19 @@ def set_user_role(
|
||||
account, and can't remove the last remaining admin. Emails the user when it actually changes."""
|
||||
role = body.role
|
||||
if role not in ("user", "admin"):
|
||||
raise HTTPException(status_code=400, detail="role must be 'user' or 'admin'")
|
||||
raise refused(400, "invalid_role")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account's role can't be changed.")
|
||||
raise refused(400, "demo_role")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't change your own role.")
|
||||
raise refused(400, "own_role")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and role == "user"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't remove the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
changed = target.role != role
|
||||
if changed:
|
||||
audit.record(
|
||||
@@ -491,16 +497,16 @@ def set_user_suspended(
|
||||
suspended = body.suspended
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be suspended.")
|
||||
raise refused(400, "demo_suspend")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't suspend your own account.")
|
||||
raise refused(400, "own_suspend")
|
||||
if (
|
||||
suspended
|
||||
and target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't suspend the last active admin.")
|
||||
raise refused(400, "last_active_admin")
|
||||
was_suspended = target.is_suspended
|
||||
if was_suspended != suspended:
|
||||
audit.record(
|
||||
@@ -535,17 +541,15 @@ def admin_delete_user(
|
||||
not the demo account, not yourself (use Settings → Account), and not the last admin."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be deleted.")
|
||||
raise refused(400, "demo_delete")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Delete your own account from Settings → Account."
|
||||
)
|
||||
raise refused(400, "own_delete")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't delete the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
# Record BEFORE the purge — the target row (and its email) is gone afterwards.
|
||||
audit.record(
|
||||
db,
|
||||
@@ -689,7 +693,7 @@ def activate_user(
|
||||
`_activate_user_by_email`). What was missing was any way to know they had to."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be activated.")
|
||||
raise refused(400, "demo_activate")
|
||||
if target.is_active:
|
||||
return _serialize_user(target) # idempotent: a double click is not an error
|
||||
if not is_allowed(db, target.email):
|
||||
@@ -786,7 +790,7 @@ def add_demo_whitelist(
|
||||
sign-in). Idempotent — re-adding an existing email just returns it."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
row = db.execute(select(DemoWhitelist).where(DemoWhitelist.email == email)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DemoWhitelist(
|
||||
@@ -833,7 +837,7 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
||||
)
|
||||
db.commit()
|
||||
return {"reset": True, "playlists_seeded": seeded}
|
||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||
|
||||
|
||||
@router.post("/purge-discovery")
|
||||
@@ -852,4 +856,4 @@ def purge_discovery(admin: User = Depends(admin_user), db: Session = Depends(get
|
||||
after=result,
|
||||
)
|
||||
db.commit()
|
||||
return result
|
||||
return {"ok": True, **result}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""The admin diagnostics console's read side (E22.6): query `log_events`, the server's stored log.
|
||||
|
||||
Admin-only and read-only — rows leave the table through the retention job (`log_retention_days`),
|
||||
never through here. Reading is not audited: it changes nothing, and the console's live tail polls,
|
||||
so a row per read would bury the audit log's real entries.
|
||||
|
||||
What a row holds was redacted when it was written (`logs.record_fields`, and the ingest route for a
|
||||
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
|
||||
`[redacted]` / `[email]`, which is the point — the console shows that something was there.
|
||||
|
||||
Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent
|
||||
window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds
|
||||
new ones at the top. The live tail re-reads the first page rather than asking for "ids after N":
|
||||
several processes write this table, and a batch's ids are taken when it inserts, not when it
|
||||
commits, so a slower batch can land below an id the tail already passed — and would never show.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Text, and_, cast, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.utils import like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/logs", tags=["admin"])
|
||||
|
||||
PAGE_MAX = 200
|
||||
# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
|
||||
# through the free-text search, and the response stays small whatever a self-host logs.
|
||||
FACET_EVENTS_MAX = 100
|
||||
|
||||
Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
|
||||
Facet = Literal["levels", "sources", "user", "event"]
|
||||
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
|
||||
# request) — a real value to filter on, not the absence of a filter.
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _encode_cursor(row: LogEvent) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
return _utc(datetime.fromisoformat(ts)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the
|
||||
line was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogFilters:
|
||||
"""The console's filters, shared by the page and its facet counts."""
|
||||
|
||||
levels: list[str]
|
||||
sources: list[str]
|
||||
user: str | None
|
||||
event: str | None
|
||||
request_id: str | None
|
||||
job_id: int | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Facet | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed by the caller. `skip` leaves one filter out: a facet counts
|
||||
its own values under every OTHER filter, so picking a level does not zero the other
|
||||
levels' counts (the way back), while a request pivot does narrow them (honest counts)."""
|
||||
out = []
|
||||
if self.levels and skip != "levels":
|
||||
out.append(LogEvent.level.in_(self.levels))
|
||||
if self.sources and skip != "sources":
|
||||
out.append(LogEvent.source.in_(self.sources))
|
||||
if self.user is not None and skip != "user":
|
||||
out.append(
|
||||
LogEvent.user_id.is_(None)
|
||||
if self.user == NONE
|
||||
else LogEvent.user_id == int(self.user)
|
||||
)
|
||||
if self.event is not None and skip != "event":
|
||||
out.append(
|
||||
LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
|
||||
)
|
||||
if self.request_id is not None:
|
||||
out.append(LogEvent.request_id == self.request_id)
|
||||
if self.job_id is not None:
|
||||
out.append(LogEvent.job_id == self.job_id)
|
||||
if self.since is not None:
|
||||
out.append(LogEvent.ts >= _utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(LogEvent.ts < _utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
LogEvent.msg.ilike(like),
|
||||
LogEvent.event.ilike(like),
|
||||
LogEvent.request_id.ilike(like),
|
||||
LogEvent.exc.ilike(like),
|
||||
cast(LogEvent.attrs, Text).ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def log_filters(
|
||||
levels: list[Level] = Query(default=[]),
|
||||
sources: list[str] = Query(default=[], max_length=8),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
event: str | None = Query(
|
||||
default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
|
||||
),
|
||||
request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"),
|
||||
job_id: int | None = Query(default=None, ge=1),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> LogFilters:
|
||||
return LogFilters(list(levels), list(sources), user, event, request_id, job_id, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_logs(
|
||||
filters: LogFilters = Depends(log_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""One page of rows, newest first, matching every filter given (they AND together); `before`
|
||||
continues from a previous page's `next_cursor`."""
|
||||
query = select(LogEvent).where(*filters.conditions())
|
||||
if before is not None:
|
||||
ts, row_id = _decode_cursor(before)
|
||||
# The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the
|
||||
# `ts` index scan instead of filtering every newer row first.
|
||||
query = query.where(
|
||||
and_(
|
||||
LogEvent.ts <= ts,
|
||||
or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)),
|
||||
)
|
||||
)
|
||||
|
||||
# One more than asked, to know whether there is a next page without a second count query.
|
||||
rows = (
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"items": [logsink.serialize(r) for r in rows],
|
||||
"accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": _encode_cursor(rows[-1]) if more else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def log_facets(
|
||||
filters: LogFilters = Depends(log_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""What the filter chips offer, with counts: the levels, sources and accounts present, and the
|
||||
most frequent events — each counted under every filter but its own (`LogFilters.conditions`).
|
||||
`total` is the "all levels" count: every filter but the level one."""
|
||||
|
||||
def counts(col, skip: Facet, cap: int | None = None) -> list[dict]:
|
||||
n = func.count().label("n")
|
||||
stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
|
||||
if cap is not None:
|
||||
stmt = stmt.limit(cap)
|
||||
return [
|
||||
{"value": NONE if value is None else value, "count": count}
|
||||
for value, count in db.execute(stmt).all()
|
||||
]
|
||||
|
||||
users = counts(LogEvent.user_id, "user")
|
||||
total = select(func.count()).select_from(LogEvent).where(*filters.conditions("levels"))
|
||||
return {
|
||||
"levels": counts(LogEvent.level, "levels"),
|
||||
"sources": counts(LogEvent.source, "sources"),
|
||||
"events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
|
||||
"users": [{**u, "value": str(u["value"])} for u in users],
|
||||
"accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
@@ -68,4 +68,4 @@ def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
||||
db.commit()
|
||||
return {"cleared": n}
|
||||
return {"ok": True, "cleared": n}
|
||||
|
||||
+77
-85
@@ -3,7 +3,6 @@
|
||||
Split out of app/auth.py (R12 S2); the core auth surface (current_user, scopes, token
|
||||
helpers) stays in app/auth.py, imported below."""
|
||||
|
||||
import ipaddress
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import NamedTuple
|
||||
@@ -18,6 +17,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.auth import (
|
||||
@@ -40,9 +40,10 @@ from app.auth import (
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import encrypt, hash_password, hash_token, verify_password
|
||||
from app.utils import canonical_email, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
VERIFY_TTL = timedelta(hours=24)
|
||||
# A confirmation link that rides an APPROVAL or an invitation lives longer than one the person is
|
||||
@@ -172,37 +173,6 @@ def is_demo_allowed(db: Session, email: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
"""Best-effort client IP for rate limiting. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
return parts[-1]
|
||||
return peer
|
||||
|
||||
|
||||
class InviteRequest(NamedTuple):
|
||||
"""What an access request settled on.
|
||||
|
||||
@@ -409,7 +379,7 @@ async def callback(
|
||||
|
||||
userinfo = token.get("userinfo")
|
||||
if not userinfo or not userinfo.get("sub"):
|
||||
raise HTTPException(status_code=400, detail="No user info returned by Google")
|
||||
raise refused(400, "google_no_userinfo")
|
||||
|
||||
sub = userinfo["sub"]
|
||||
email = (userinfo.get("email") or "").lower()
|
||||
@@ -418,7 +388,15 @@ async def callback(
|
||||
return _complete_link(request, db, link_uid, link_explicit, sub, userinfo, token)
|
||||
|
||||
if not is_allowed(db, email):
|
||||
log.warning("Login denied (not approved): %s", email or "<no email>")
|
||||
# No uid to name — having no account here is the branch. And the address is left out on
|
||||
# purpose: `redact_text` turns it into `[email]` before either sink sees it, so a line
|
||||
# whose only variable is always `[email]` says nothing. WHO asked is in the pending
|
||||
# invite recorded just below, which is where an admin looks anyway.
|
||||
log.warning(
|
||||
"Google sign-in denied: address not approved",
|
||||
# log-pii-ok: whether an address came back, not what it was
|
||||
extra=logs.event("auth.login.denied", reason="not_approved", has_email=bool(email)),
|
||||
)
|
||||
# A denied Google login doubles as an access request: record it for the admin and
|
||||
# land the user on a friendly "request received" screen instead of a raw 403.
|
||||
if email:
|
||||
@@ -451,15 +429,20 @@ async def callback(
|
||||
if existing.google_sub is not None and existing.google_sub != sub:
|
||||
# A different Google identity is already bound to this email — refuse rather than
|
||||
# hijack it. (Near-impossible with real Google accounts: email↔sub is stable.)
|
||||
log.warning("Google login email collision (different sub): %s", email)
|
||||
log.warning(
|
||||
"Google sign-in refused: a different Google identity holds this address",
|
||||
extra=logs.event("auth.login.denied", reason="sub_collision", uid=existing.id),
|
||||
)
|
||||
return RedirectResponse(url=_access_notice_url("denied"))
|
||||
if not userinfo.get("email_verified", False):
|
||||
# Adopting (and activating) a pre-existing password account requires Google to
|
||||
# actually attest the address — otherwise an unverified Google email that merely
|
||||
# collides with a pending registration could seize that account. Defense-in-depth.
|
||||
log.warning(
|
||||
"Google login: unverified email, refusing to adopt account: %s",
|
||||
email,
|
||||
"Google sign-in refused: unverified address, not adopting the account",
|
||||
extra=logs.event(
|
||||
"auth.login.denied", reason="email_unverified", uid=existing.id
|
||||
),
|
||||
)
|
||||
return RedirectResponse(url=_access_notice_url("denied"))
|
||||
user = existing
|
||||
@@ -477,7 +460,10 @@ async def callback(
|
||||
# A suspended account can't sign in by any method. Block before establishing the session and
|
||||
# tell the (verified) owner why — a successful Google auth proves they control this account.
|
||||
if user.is_suspended:
|
||||
log.info("Suspended account blocked at Google login: %s", email)
|
||||
log.info(
|
||||
"Google sign-in blocked: the account is suspended",
|
||||
extra=logs.event("auth.login.denied", reason="suspended", uid=user.id),
|
||||
)
|
||||
_notify_suspended(background, user)
|
||||
return RedirectResponse(url="/?login=suspended")
|
||||
# Record what Google actually answers with, beside the address the user registered with. The two
|
||||
@@ -511,17 +497,12 @@ async def callback(
|
||||
db.flush()
|
||||
|
||||
_store_token(db, user, token)
|
||||
_establish_session(request, user)
|
||||
db.commit()
|
||||
|
||||
# Multi-session: remember every account that has authenticated in this browser so the
|
||||
# user can switch between them without going through Google again. Only accounts that
|
||||
# actually completed OAuth here land in this list (it's the proof they may be switched to).
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
request.session["user_id"] = user.id
|
||||
_remember_epoch(request, user)
|
||||
log.info("Login: %s (id=%s, role=%s)", email, user.id, user.role)
|
||||
log.info(
|
||||
"signed in",
|
||||
extra=logs.event("auth.login.ok", method="google", uid=user.id, role=user.role),
|
||||
)
|
||||
return RedirectResponse(url="/")
|
||||
|
||||
|
||||
@@ -631,6 +612,8 @@ async def logout(request: Request):
|
||||
out in one tab doesn't disturb another tab running a different account. If the account being
|
||||
removed was the session default, promote the most recent remaining account (or clear the
|
||||
session when none remain)."""
|
||||
# A tab still pinned to an account already signed out in another tab resolves to nobody, so it
|
||||
# signs nobody out — it used to fall back to the default and sign THAT account out in its place.
|
||||
active, is_default = resolved_user_id(request)
|
||||
remaining = [a for a in (request.session.get("account_ids") or []) if a != active]
|
||||
request.session["account_ids"] = remaining
|
||||
@@ -671,15 +654,15 @@ async def request_access(
|
||||
endpoint can't be probed for throttling either."""
|
||||
email = (body.email or "").strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
if not _request_access_limiter.allow(_client_ip(request)):
|
||||
return {"status": "pending"}
|
||||
raise refused(400, "invalid_email")
|
||||
if not _request_access_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "pending"}
|
||||
if is_allowed(db, email):
|
||||
return {"status": "pending"} # already allowed — don't reveal it
|
||||
return {"ok": True, "status": "pending"} # already allowed — don't reveal it
|
||||
req = upsert_pending_invite(db, email, body.lang)
|
||||
if req.is_new:
|
||||
notify_admins_of_request(db, background, email)
|
||||
return {"status": "pending"}
|
||||
return {"ok": True, "status": "pending"}
|
||||
|
||||
|
||||
@router.post("/demo")
|
||||
@@ -689,8 +672,8 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
so there is no visible 'demo login' button. Rate-limited per IP; when blocked it answers
|
||||
exactly like a non-match (authenticated=false) so the field can't be hammered as an
|
||||
enumeration oracle. On a match it sets the session and the client reloads into the app."""
|
||||
if not _demo_limiter.allow(_client_ip(request)):
|
||||
return {"authenticated": False}
|
||||
if not _demo_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "authenticated": False}
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email) and is_demo_allowed(db, email):
|
||||
demo = get_or_create_demo_user(db)
|
||||
@@ -699,19 +682,25 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
request.session.clear()
|
||||
request.session["user_id"] = demo.id
|
||||
_remember_epoch(request, demo)
|
||||
log.info("Demo login (key=%s, demo_id=%s)", email, demo.id)
|
||||
return {"authenticated": True}
|
||||
return {"authenticated": False}
|
||||
# The demo key IS an address, so it is not in the line — see the denied branch above.
|
||||
log.info("signed in", extra=logs.event("auth.login.ok", method="demo", uid=demo.id))
|
||||
return {"ok": True, "authenticated": True}
|
||||
return {"ok": True, "authenticated": False}
|
||||
|
||||
|
||||
def _establish_session(request: Request, user: User) -> None:
|
||||
"""Sign `user` in for this browser session, mirroring the Google callback's multi-account
|
||||
handling so password sign-in joins the same account switcher."""
|
||||
"""Sign `user` in for this browser session — the ONE sign-in path, shared by the Google callback
|
||||
and password login, so both join the same account switcher and both stamp `last_login_at` (the
|
||||
caller commits). Multi-session: every account that authenticated in this browser is remembered so
|
||||
the user can switch between them; only accounts that actually proved themselves HERE land in that
|
||||
wallet (it is the proof they may be switched to). The demo entry deliberately does not come
|
||||
through here: it clears the wallet instead, and its shared account has no meaningful last login."""
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
request.session["user_id"] = user.id
|
||||
_remember_epoch(request, user)
|
||||
user.last_login_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _app_base() -> str:
|
||||
@@ -827,23 +816,20 @@ def register(
|
||||
password = body.password or ""
|
||||
# Input validation is safe to reveal (independent of whether the email exists).
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if len(password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
if not sysconfig.get_bool(db, "allow_registration"):
|
||||
raise HTTPException(status_code=403, detail="Registration is currently closed.")
|
||||
if not _register_limiter.allow(_client_ip(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
raise refused(403, "registration_closed")
|
||||
if not _register_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "ok"} # silently throttle; uniform response
|
||||
|
||||
# Do the existence check + account creation OFF the response path (a background task with its own
|
||||
# session). The create path hashes the password + writes several rows + schedules emails; doing it
|
||||
# inline would make a NEW email respond measurably slower than an already-registered one (which
|
||||
# skips all that) — a timing oracle for enumeration. Off-path, any valid email responds the same.
|
||||
background.add_task(_register_account, email, password, body.lang)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _register_account(email: str, password: str, lang: str | None = None) -> None:
|
||||
@@ -913,7 +899,12 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non
|
||||
except IntegrityError:
|
||||
# A concurrent registration for the same new email won the race between the pre-check
|
||||
# above and this flush (the unique-email constraint fires). Same already-registered
|
||||
# no-op as the pre-check hit — roll back and stop.
|
||||
# no-op as the pre-check hit — roll back and stop. The line carries nothing: the
|
||||
# answer to the caller is deliberately identical either way, and so is this.
|
||||
log.info(
|
||||
"a registration lost the race for its address",
|
||||
extra=logs.event("auth.register.race_lost"),
|
||||
)
|
||||
db.rollback()
|
||||
return
|
||||
upsert_pending_invite(db, email) # admin-approval gate
|
||||
@@ -1003,12 +994,12 @@ def verify_email_resend(
|
||||
and re-registering an already-registered email is a silent no-op, so a user who lost the first
|
||||
mail had no way back. Uniform response + timing regardless of whether the email has a pending
|
||||
account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _verify_resend_limiter.allow(_client_ip(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
if not _verify_resend_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "ok"} # silently throttle; uniform response
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email):
|
||||
background.add_task(_resend_verify_if_eligible, email)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _resend_verify_if_eligible(email: str) -> None:
|
||||
@@ -1062,7 +1053,7 @@ def password_login(
|
||||
"""Email+password sign-in. Wrong email/password give a single uniform 401 (no enumeration).
|
||||
Once the password is proven correct, the owner gets a specific reason if their account is
|
||||
still pending or suspended — that's not an enumeration leak (they already hold the password)."""
|
||||
if not _login_limiter.allow(_client_ip(request)):
|
||||
if not _login_limiter.allow(client_key(request)):
|
||||
raise _login_refused(429, "rate_limited")
|
||||
email = (body.email or "").strip().lower()
|
||||
password = body.password or ""
|
||||
@@ -1081,7 +1072,11 @@ def password_login(
|
||||
if not user.is_active:
|
||||
raise _login_refused(403, "pending_approval", user)
|
||||
_establish_session(request, user)
|
||||
log.info("Password login: %s (id=%s, role=%s)", email, user.id, user.role)
|
||||
db.commit()
|
||||
log.info(
|
||||
"signed in",
|
||||
extra=logs.event("auth.login.ok", method="password", uid=user.id, role=user.role),
|
||||
)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@@ -1093,15 +1088,15 @@ def password_reset_request(
|
||||
) -> dict:
|
||||
"""Request a password-reset link. Uniform response + timing regardless of whether the email has a
|
||||
password account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _reset_limiter.allow(_client_ip(request)):
|
||||
return {"status": "ok"}
|
||||
if not _reset_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "ok"}
|
||||
email = (body.email or "").strip().lower()
|
||||
# Do the account lookup + token issue + email OFF the response path (a background task with its
|
||||
# own session), so the endpoint takes the same time for any valid email whether or not it has a
|
||||
# password account — no timing-based enumeration. Any valid email schedules the same task.
|
||||
if valid_email(email):
|
||||
background.add_task(_send_reset_if_eligible, email)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _send_reset_if_eligible(email: str) -> None:
|
||||
@@ -1142,7 +1137,7 @@ def password_reset_check(body: TokenIn, request: Request, db: Session = Depends(
|
||||
told `true`, never `false`: this check may only ever save a user a wasted form, so when it cannot
|
||||
answer it must degrade to the old behaviour instead of declaring a live link dead.
|
||||
"""
|
||||
if not _reset_check_limiter.allow(_client_ip(request)):
|
||||
if not _reset_check_limiter.allow(client_key(request)):
|
||||
return {"ok": True}
|
||||
return {"ok": any(_peek_token(db, body.token, k) is not None for k in _PASSWORD_TOKEN_KINDS)}
|
||||
|
||||
@@ -1296,13 +1291,10 @@ def set_password(
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
new_password = body.password or ""
|
||||
if len(new_password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
if user.password_hash:
|
||||
if not verify_password(body.current_password or "", user.password_hash):
|
||||
raise HTTPException(status_code=403, detail="Current password is incorrect.")
|
||||
raise refused(403, "wrong_password")
|
||||
user.password_hash = hash_password(new_password)
|
||||
user.password_revoked_at = None # same as the reset path: a password exists again
|
||||
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
|
||||
|
||||
@@ -2,15 +2,14 @@
|
||||
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
|
||||
(subscribe / unsubscribe) that require the optional write scope."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app import logs, quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -26,16 +25,17 @@ from app.models import (
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
||||
|
||||
log = logging.getLogger("siftlode.api")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -166,7 +166,15 @@ def discover_channels(
|
||||
user.yt_channel_id = own
|
||||
db.commit()
|
||||
except YouTubeError as exc:
|
||||
log.warning("discovery: own-channel resolve failed (user %s): %s", user.id, exc)
|
||||
# No traceback anywhere in this file's YouTube handlers: for an upstream answer
|
||||
# `_upstream_failure` already logged the structured `youtube.api.error`, and the
|
||||
# message names the call, its status and its reason (or, with no answer at all, the
|
||||
# transport failure). The stack under it is this `except` and the client call.
|
||||
log.warning(
|
||||
"discovery: own-channel resolve failed: %s",
|
||||
exc,
|
||||
extra=logs.event("channels.discovery.own_channel_failed", reason=exc.reason),
|
||||
)
|
||||
|
||||
rows = _discovery_rows(db, user)
|
||||
|
||||
@@ -190,7 +198,11 @@ def discover_channels(
|
||||
rows, key=lambda r: (-int(r[1]), r[0].title is None, (r[0].title or "").lower())
|
||||
)
|
||||
except YouTubeError as exc:
|
||||
log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc)
|
||||
log.warning(
|
||||
"discovery: channel enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event("channels.discovery.enrich_failed", reason=exc.reason),
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
return [
|
||||
@@ -298,7 +310,13 @@ def channel_detail(
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
except YouTubeError as exc:
|
||||
log.warning("channel detail enrich failed (%s): %s", channel_id, exc)
|
||||
log.warning(
|
||||
"channel detail enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event(
|
||||
"channels.detail.enrich_failed", channel_id=channel_id, reason=exc.reason
|
||||
),
|
||||
)
|
||||
db.rollback()
|
||||
sub = db.execute(
|
||||
select(Subscription).where(
|
||||
@@ -366,12 +384,9 @@ def explore_channel(
|
||||
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise HTTPException(status_code=403, detail="You've blocked this channel.")
|
||||
raise refused(403, "channel_blocked")
|
||||
if quota.backfill_reserve_reached():
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared daily quota reserve is reached — try exploring later.",
|
||||
)
|
||||
raise refused(429, "quota_reserve")
|
||||
|
||||
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
|
||||
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
|
||||
@@ -407,8 +422,9 @@ def explore_channel(
|
||||
channel = db.get(Channel, channel_id)
|
||||
result = explore_ingest_page(db, yt, channel, page_token)
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't load this channel: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
return {
|
||||
"ok": True,
|
||||
"channel_id": channel_id,
|
||||
"ingested": result["ingested"],
|
||||
"next_page_token": result["next_page_token"],
|
||||
@@ -490,6 +506,7 @@ def update_channel(
|
||||
background.add_task(backfill_channel_now, channel_id, user.id)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"id": channel_id,
|
||||
"priority": sub.priority,
|
||||
"hidden": sub.hidden,
|
||||
@@ -524,7 +541,7 @@ def reset_backfill(
|
||||
db.commit()
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
return {"id": channel_id, "reset": True}
|
||||
return {"ok": True, "id": channel_id, "reset": True}
|
||||
|
||||
|
||||
@router.post("/{channel_id}/subscribe")
|
||||
@@ -545,7 +562,7 @@ def subscribe(
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
|
||||
raise refused(409, "already_subscribed")
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
|
||||
YouTubeClient(db, user) as yt,
|
||||
@@ -559,13 +576,14 @@ def subscribe(
|
||||
# subscription resync fills in the resource id. Any other YouTube error is a real
|
||||
# problem — surface it with a clear message (422, not a 502 that the client would
|
||||
# mistake for a transient "connection lost").
|
||||
msg = str(exc).lower()
|
||||
if "already exists" in msg or "duplicate" in msg:
|
||||
# On the reason code, never on the prose: the error no longer carries the body (it is
|
||||
# the request quoted back), and a message YouTube rewords would silently turn "already
|
||||
# following" back into a 422 the user cannot act on. `reason` is comma-joined when the
|
||||
# body listed several.
|
||||
if "subscriptionDuplicate" in exc.reason.split(","):
|
||||
yt_sub_id = ""
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube couldn't subscribe you: {exc}"
|
||||
)
|
||||
raise youtube_refusal(exc)
|
||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||
@@ -574,7 +592,15 @@ def subscribe(
|
||||
try:
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
except YouTubeError as exc:
|
||||
log.warning("subscribe: stub enrich failed (%s): %s", channel_id, exc)
|
||||
log.warning(
|
||||
"subscribe: stub enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event(
|
||||
"channels.subscribe.enrich_failed",
|
||||
channel_id=channel_id,
|
||||
reason=exc.reason,
|
||||
),
|
||||
)
|
||||
db.add(
|
||||
Subscription(
|
||||
user_id=user.id,
|
||||
@@ -594,7 +620,7 @@ def subscribe(
|
||||
.values(via_explore=False)
|
||||
)
|
||||
db.commit()
|
||||
return {"subscribed": channel_id}
|
||||
return {"ok": True, "subscribed": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/subscription")
|
||||
@@ -608,10 +634,7 @@ def unsubscribe(
|
||||
should Hide instead."""
|
||||
sub = _user_subscription(db, user, channel_id)
|
||||
if not sub.yt_subscription_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No YouTube subscription id on record — sync subscriptions first.",
|
||||
)
|
||||
raise refused(400, "no_subscription_id")
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_UNSUBSCRIBE),
|
||||
@@ -621,10 +644,10 @@ def unsubscribe(
|
||||
except YouTubeError as exc:
|
||||
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
||||
# a transient "connection lost" gateway blip.
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't unsubscribe you: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
db.delete(sub)
|
||||
db.commit()
|
||||
return {"unsubscribed": channel_id}
|
||||
return {"ok": True, "unsubscribed": channel_id}
|
||||
|
||||
|
||||
@router.get("/blocked/list")
|
||||
@@ -665,7 +688,7 @@ def block_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"blocked": channel_id}
|
||||
return {"ok": True, "blocked": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/block")
|
||||
@@ -680,7 +703,7 @@ def unblock_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"unblocked": channel_id}
|
||||
return {"ok": True, "unblocked": channel_id}
|
||||
|
||||
|
||||
class AttachTagIn(BaseModel):
|
||||
@@ -715,7 +738,7 @@ def attach_tag(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": body.tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": body.tag_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/tags/{tag_id}")
|
||||
@@ -735,4 +758,4 @@ def detach_tag(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": tag_id}
|
||||
|
||||
@@ -4,18 +4,23 @@ app.sysconfig. Secret values (e.g. SMTP password) are write-only — never retur
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.refusal import refused
|
||||
from app.issues import gitea
|
||||
from app.routes._common import issue_refused, not_found
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin/config", tags=["admin"])
|
||||
|
||||
@@ -29,6 +34,14 @@ def _redact(v):
|
||||
return _URL_USERINFO.sub(r"\1***@", v) if isinstance(v, str) else v
|
||||
|
||||
|
||||
def _apply_runtime(db: Session, key: str) -> None:
|
||||
"""Some knobs change how THIS process behaves, not just what a later read returns — they have to
|
||||
take effect now, without a restart. One dispatch for both call sites (set and reset), so the next
|
||||
such key can't be wired into one of them and forgotten in the other."""
|
||||
if key == "log_level":
|
||||
logs.set_level(sysconfig.get_str(db, "log_level"))
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_config(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
return sysconfig.describe(db)
|
||||
@@ -50,19 +63,16 @@ def set_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
if "value" not in body.model_fields_set:
|
||||
raise HTTPException(status_code=400, detail="Missing 'value'")
|
||||
raise refused(400, "value_missing")
|
||||
if s.secret and not sysconfig.secrets_manageable():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Set TOKEN_ENCRYPTION_KEY to store secrets in the database.",
|
||||
)
|
||||
raise refused(400, "no_encryption_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
try:
|
||||
sysconfig.set_value(db, key, body.value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
# Never log a secret's value — only that it was changed, by whom. Non-secret values are logged
|
||||
# but with URL credentials redacted (a proxy/URL setting can embed user:pass). Skip the row when
|
||||
# a non-secret value didn't actually change (a no-op re-save shouldn't add trail noise); secrets
|
||||
@@ -80,6 +90,7 @@ def set_config(
|
||||
after=None if s.secret else {"value": _redact(new)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -91,7 +102,7 @@ def reset_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
sysconfig.reset(db, key)
|
||||
audit.record(
|
||||
@@ -104,6 +115,7 @@ def reset_config(
|
||||
before=None if s.secret else {"value": _redact(old)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -115,8 +127,21 @@ def send_test_email(
|
||||
"""Send a test email to the requesting admin using the current (DB or env) SMTP config —
|
||||
lets the admin verify the settings actually work."""
|
||||
if not email_mod.email_enabled():
|
||||
raise HTTPException(status_code=400, detail="SMTP is not configured.")
|
||||
raise refused(400, "smtp_off")
|
||||
sent = email_mod.send_test(user.email, mailtext.lang_for_user(user))
|
||||
if not sent:
|
||||
raise HTTPException(status_code=422, detail="SMTP send failed — check the settings.")
|
||||
return {"sent": True, "to": user.email}
|
||||
raise refused(422, "smtp_failed")
|
||||
return {"ok": True, "sent": True, "to": user.email}
|
||||
|
||||
|
||||
@router.post("/test-issues")
|
||||
def test_issue_tracker(
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Check the configured issue tracker (E16): the repository answers with this token, takes
|
||||
issues, and is PRIVATE — the one it must be, since an issue carries its reporter's log."""
|
||||
try:
|
||||
return {"ok": True, **gitea.check(db)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, the screenshots, and the evidence under review.
|
||||
|
||||
The session itself — what tags a request, its cap, its level — is `app/debugsession.py`. These routes
|
||||
are the user's handle on their OWN sessions only: another user's id 404s (`owned_or_404`), including
|
||||
for an admin, who reads everyone's lines in the diagnostics console instead. The shared demo account
|
||||
cannot record: its log would interleave every visitor under one account.
|
||||
|
||||
Screenshots arrive as a raw image body (`image/png`, `image/jpeg`, `image/webp`), not a multipart
|
||||
form — one image per request, capped while it arrives, and kept only if its bytes really are one of
|
||||
those types, whatever the Content-Type claimed.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, debugsession, logs, logsink
|
||||
from app.audit import AuditAction
|
||||
from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.issues import bundle, gitea
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.refusal import refused
|
||||
from app.models import DebugScreenshot, DebugSession, User
|
||||
from app.routes._common import capped_body, issue_refused, not_found, owned_or_404
|
||||
from app.utils import iso
|
||||
|
||||
router = APIRouter(prefix="/api/debug-sessions", tags=["diagnostics"])
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Sessions listed in Settings → Diagnosztika, newest first: enough to reach last week's unsent draft.
|
||||
LIST_MAX = 20
|
||||
|
||||
|
||||
class StartIn(BaseModel):
|
||||
level: debugsession.SessionLevel
|
||||
|
||||
|
||||
_view = debugsession.view
|
||||
|
||||
# New sessions a user may open per hour (joining the running one is not counted): a session is a
|
||||
# place to keep screenshots, and this — with the per-user storage cap — bounds how many there are.
|
||||
_new_sessions = RateLimiter(max_events=20, window_seconds=3600)
|
||||
|
||||
|
||||
@router.post("")
|
||||
def start_session(
|
||||
body: StartIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Start recording, or join the recording already running (another tab started it)."""
|
||||
joining = debugsession.open_session(db, user) is not None
|
||||
if not joining and not _new_sessions.allow(str(user.id)):
|
||||
raise refused(429, "debug_session_rate_limited", limit=_new_sessions.max_events)
|
||||
try:
|
||||
session = debugsession.start(db, user, body.level)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent Start won the one-open-per-user index: join that one.
|
||||
db.rollback()
|
||||
session = debugsession.open_session(db, user)
|
||||
if session is None:
|
||||
raise refused(409, "debug_session_conflict") from None
|
||||
# The rest of THIS request is part of the session too.
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id)
|
||||
log.info(
|
||||
"debug session started",
|
||||
extra=logs.event("debug.session.started", level=session.level, until=iso(session.until)),
|
||||
)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.post("/{sid}/stop")
|
||||
def stop_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if session.ended_at is None:
|
||||
# Logged BEFORE the session closes, while this request still carries its tag: the last line
|
||||
# of the evidence is the Stop itself.
|
||||
log.info("debug session stopped", extra=logs.event("debug.session.stopped"))
|
||||
debugsession.stop(session)
|
||||
db.commit()
|
||||
logs.end_session_elevation(session.id)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_sessions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
sessions = db.scalars(
|
||||
select(DebugSession)
|
||||
.where(DebugSession.user_id == user.id)
|
||||
.order_by(DebugSession.started_at.desc())
|
||||
.limit(LIST_MAX)
|
||||
).all()
|
||||
return {"items": [_view(db, s) for s in sessions]}
|
||||
|
||||
|
||||
@router.get("/{sid}")
|
||||
def get_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
return _view(db, owned_or_404(db, DebugSession, sid, user, what="debug_session"))
|
||||
|
||||
|
||||
@router.get("/{sid}/events")
|
||||
def session_events(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The rows an issue from this session would carry — browser and server, in time order."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
rows, total = debugsession.bundle(db, session)
|
||||
return {"items": [logsink.serialize(r) for r in rows], "total": total}
|
||||
|
||||
|
||||
# --- the draft and its export ---------------------------------------------------------------
|
||||
# The composer's own fields (steps, description…) are its business, kept whole so it reopens as it
|
||||
# was — bounded here so a draft cannot become a place to store anything else.
|
||||
FORM_MAX_BYTES = 64 * 1024
|
||||
# Zips a user may build per minute: each holds every screenshot in memory while it is built.
|
||||
_exports = RateLimiter(max_events=6, window_seconds=60)
|
||||
|
||||
|
||||
class DraftIn(BaseModel):
|
||||
title: str = Field(max_length=200)
|
||||
# The rendered markdown the export and the issue carry.
|
||||
body: str = Field(max_length=50_000)
|
||||
# Log rows taken out of the evidence in the review.
|
||||
excluded: list[int] = Field(default_factory=list, max_length=debugsession.BUNDLE_MAX)
|
||||
form: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@router.put("/{sid}/draft")
|
||||
def save_draft(
|
||||
sid: str,
|
||||
body: DraftIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if len(json.dumps(body.form, ensure_ascii=False).encode()) > FORM_MAX_BYTES:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
# Under the lock, and refused while a send is under way: the issue is being built from the draft
|
||||
# as it was saved, and this would change it under the send.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
if session.issue_url is not None:
|
||||
# Sent: the issue is on the tracker now, and a draft edited after it would say otherwise.
|
||||
raise refused(409, "issue_already_sent")
|
||||
debugsession.save_draft(session, body.model_dump())
|
||||
db.commit()
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("/{sid}/export")
|
||||
def export_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
"""The issue as a zip — the text, the two logs and the screenshots — for an instance without an
|
||||
issue tracker, or for sending it by hand. Built from the SAVED draft: what the composer shows is
|
||||
what it saved."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _exports.allow(str(user.id)):
|
||||
raise refused(429, "export_rate_limited")
|
||||
data = bundle.zipped(bundle.files(db, session))
|
||||
name = f"siftlode-debug-{session.started_at:%Y%m%d-%H%M}-{session.id[:6]}.zip"
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/zip",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{name}"',
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- the issue tracker (S4) -----------------------------------------------------------------
|
||||
# Issues a user may open per hour — counted once the tracker is ASKED to open one (an answer that
|
||||
# goes missing may still have created it, so an attempt cannot be given back); finishing a half-sent
|
||||
# one, or an attempt refused before the tracker was asked, is not counted. The composer sends one at
|
||||
# a time; this is room for a bad afternoon, not a pace anything legitimate keeps up.
|
||||
_new_issues = RateLimiter(max_events=5, window_seconds=3600)
|
||||
# Attempts on the tracker per hour, creating or finishing. Deliberately NOT the budget above: a
|
||||
# half-sent issue must always be finishable, so the pace that guards creating one must never be what
|
||||
# stands between a user and their own report. This one only bounds a loop — every attempt re-reads
|
||||
# the whole bundle into memory and re-offers the pending file to the tracker, and a retry that is
|
||||
# refused for good (the tracker's own limits) never leaves `pending`. No hand reaches it.
|
||||
_sends = RateLimiter(max_events=60, window_seconds=3600)
|
||||
# The composer's live preview, rendered BY the tracker: one authenticated call each, with our token.
|
||||
# The composer's toggle fires it once per switch INTO preview — the editor is the other branch, so
|
||||
# no keystroke can change the query key while the preview is up — and react-query's cache makes an
|
||||
# unchanged draft free. (It does NOT debounce, whatever this comment used to say: a key that changes
|
||||
# per keystroke is one fetch per keystroke, measured. Showing the preview BESIDE the fields would
|
||||
# need a real debounce first.) This bounds what a loop can aim at the tracker.
|
||||
_previews = RateLimiter(max_events=60, window_seconds=60)
|
||||
|
||||
|
||||
def _issues_rate_limited() -> HTTPException:
|
||||
return refused(429, "issues_rate_limited", limit=_new_issues.max_events)
|
||||
|
||||
|
||||
@router.post("/{sid}/issue")
|
||||
def send_issue(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Send the session's issue to the tracker — from the SAVED draft, as the export is built — or
|
||||
finish a send that stopped half-way (the issue exists; only what is missing is sent)."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
# The session is CLAIMED for the send rather than locked through it: the tracker is slow (a
|
||||
# dozen calls, 20 s each), and a row lock held across them would queue this user's Stop, their
|
||||
# autosave and their screenshots behind it. They see the claim instead and are refused at once.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
new = session.issue_url is None
|
||||
if not new and not debugsession.unfinished(session):
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
key = str(user.id)
|
||||
if new:
|
||||
if not ((session.draft or {}).get("title") or "").strip():
|
||||
raise refused(422, "issue_title_required")
|
||||
if _new_issues.exhausted(key):
|
||||
raise _issues_rate_limited()
|
||||
if not _sends.allow(key):
|
||||
raise refused(429, "issue_rate_limited", limit=_sends.max_events)
|
||||
token = debugsession.claim_send(session)
|
||||
db.commit() # the claim is what everything else sees; the lock ends here
|
||||
|
||||
def count_new_issue() -> None:
|
||||
if not _new_issues.allow(key):
|
||||
raise _issues_rate_limited()
|
||||
|
||||
try:
|
||||
gitea.deliver(
|
||||
db,
|
||||
session,
|
||||
token=token,
|
||||
checkpoint=db.commit,
|
||||
before_create=count_new_issue,
|
||||
on_created=lambda: _record_created(db, user, session),
|
||||
)
|
||||
except debugsession.SendClaimLost:
|
||||
# This send outlived its lease and another one took the session over while it was still
|
||||
# talking to the tracker. It stopped rather than writing over that one's progress — and a
|
||||
# send IS under way, just not ours, which is what the caller needs to hear.
|
||||
raise refused(409, "issue_sending") from None
|
||||
except gitea.GiteaError as exc:
|
||||
# Keep what did happen: an issue created before an upload failed stays linked, and the
|
||||
# retry does only what is still missing.
|
||||
raise issue_refused(exc) from None
|
||||
except HTTPException:
|
||||
raise # the pace, refused before the tracker was asked
|
||||
finally:
|
||||
_end_send(db, session, token)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
def _end_send(db: Session, session: DebugSession, token: str) -> None:
|
||||
"""Give the claim back, whatever else happened. A failure here must not replace the failure that
|
||||
brought us: a dead transaction (the database went away mid-send) is rolled back, and the claim
|
||||
then simply ages out."""
|
||||
try:
|
||||
debugsession.lock(db, session)
|
||||
debugsession.release_send(session, token)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 — whatever went wrong here, the send's own failure is the news
|
||||
db.rollback()
|
||||
log.warning(
|
||||
"a send could not give its claim back",
|
||||
extra=logs.event("issues.claim.stuck", session_id=session.id),
|
||||
)
|
||||
|
||||
|
||||
def _record_created(db: Session, user: User, session: DebugSession) -> None:
|
||||
"""The trail for the one act that takes a user's log off this instance. Staged inside the send,
|
||||
so the checkpoint that first writes `issue_url` commits BOTH — recorded later it could be lost
|
||||
on its own (the process dies mid-send), and no retry could write it afterwards: the next send
|
||||
sees the link and knows the issue is not new."""
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
AuditAction.ISSUE_CREATE,
|
||||
target_type="debug_session",
|
||||
target_id=session.id,
|
||||
summary=session.issue_url or "",
|
||||
)
|
||||
|
||||
|
||||
class PreviewIn(BaseModel):
|
||||
text: str = Field(max_length=50_000)
|
||||
|
||||
|
||||
@router.post("/{sid}/preview")
|
||||
def preview_issue(
|
||||
sid: str,
|
||||
body: PreviewIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The body as the tracker will render it. Gitea sanitises its own output; the composer still
|
||||
shows it in a sandboxed frame, never in the app's own document."""
|
||||
owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _previews.allow(str(user.id)):
|
||||
raise refused(429, "preview_rate_limited")
|
||||
try:
|
||||
return {"ok": True, "html": gitea.preview(db, body.text)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
def _shot_view(shot: DebugScreenshot) -> dict:
|
||||
return {"id": shot.id, "mime": shot.mime, "size": shot.size, "created_at": iso(shot.created_at)}
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots")
|
||||
def list_screenshots(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
return {"items": [_shot_view(s) for s in debugsession.screenshots(db, session)]}
|
||||
|
||||
|
||||
def _upload_target(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> DebugSession:
|
||||
return owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
|
||||
|
||||
async def _image_body(request: Request) -> bytes:
|
||||
return await capped_body(request, debugsession.SCREENSHOT_MAX_BYTES)
|
||||
|
||||
|
||||
@router.post("/{sid}/screenshots")
|
||||
def add_screenshot(
|
||||
# The owner check FIRST: dependencies resolve in order, so a stranger's upload is refused
|
||||
# before a byte of it is read.
|
||||
session: DebugSession = Depends(_upload_target),
|
||||
data: bytes = Depends(_image_body),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
try:
|
||||
shot = debugsession.add_screenshot(db, session, data)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "screenshot": _shot_view(shot)}
|
||||
|
||||
|
||||
def _screenshot_refused(code: str) -> HTTPException:
|
||||
# Literal reasons, one per case: the frontend's contract test reads them to require a sentence
|
||||
# for each in both languages.
|
||||
if code == "already_sent":
|
||||
return refused(409, "screenshot_already_sent")
|
||||
if code == "too_many":
|
||||
return refused(422, "screenshot_too_many", limit=debugsession.SCREENSHOTS_MAX)
|
||||
if code == "sending":
|
||||
return refused(409, "issue_sending")
|
||||
if code == "storage_full":
|
||||
mb = debugsession.SCREENSHOT_USER_MAX_BYTES // (1024 * 1024)
|
||||
return refused(422, "screenshot_storage_full", limit=mb)
|
||||
if code == "empty":
|
||||
return refused(422, "screenshot_empty")
|
||||
return refused(422, "screenshot_not_an_image")
|
||||
|
||||
|
||||
def _session_screenshot(db: Session, sid: str, shot_id: int, user: User) -> DebugScreenshot:
|
||||
return _screenshot_of(
|
||||
db, owned_or_404(db, DebugSession, sid, user, what="debug_session"), shot_id
|
||||
)
|
||||
|
||||
|
||||
def _screenshot_of(db: Session, session: DebugSession, shot_id: int) -> DebugScreenshot:
|
||||
shot = db.get(DebugScreenshot, shot_id)
|
||||
if shot is None or shot.session_id != session.id:
|
||||
raise not_found("screenshot")
|
||||
return shot
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots/{shot_id}")
|
||||
def get_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
shot = _session_screenshot(db, sid, shot_id, user)
|
||||
# `nosniff`: the type was checked against the bytes at upload, and the browser must not guess.
|
||||
return Response(
|
||||
content=shot.data,
|
||||
media_type=shot.mime,
|
||||
headers={"Cache-Control": "private, max-age=3600", "X-Content-Type-Options": "nosniff"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{sid}/screenshots/{shot_id}")
|
||||
def delete_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
shot = _screenshot_of(db, session, shot_id)
|
||||
try:
|
||||
debugsession.remove_screenshot(db, session, shot)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "deleted": shot_id}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""The browser's log, sent while its user has a raised diagnostics level (E22 S5).
|
||||
|
||||
The client keeps its log in memory (frontend `lib/log.ts`). Only after the user consented to Részletes
|
||||
or Teljes in Settings → Diagnosztika does it stream that log here, in batches, and each batch lands in
|
||||
`log_events` beside the server's own rows: `source="web"`, the account from the SESSION (never from
|
||||
the body), and the request id each event carries, which is what joins a browser line to the server
|
||||
line of the same request.
|
||||
|
||||
Nothing from the body is trusted as-is:
|
||||
* only a JSON Content-Type is read, and the whole body is capped before it is parsed, and so are
|
||||
the number of events and each event's attributes;
|
||||
* the pace is capped per account, and so is what an account may store per hour;
|
||||
* everything is redacted AGAIN with the server's own rules — the client redacts at capture, but a
|
||||
modified client would not;
|
||||
* an event below the consented level is dropped (a Részletes consent does not store what only
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed — where an open debug
|
||||
session the request names counts as consent to ITS level (E16), and only its own events keep
|
||||
its tag;
|
||||
* the timestamp is clamped to the consent window, so a row cannot dodge the retention job by
|
||||
claiming to come from the future.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, diagnostics, logs, logsink
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import DebugSession, LogEvent, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import capped_body
|
||||
|
||||
router = APIRouter(prefix="/api/diagnostics", tags=["diagnostics"])
|
||||
|
||||
BODY_MAX_BYTES = 512 * 1024
|
||||
EVENTS_MAX = 200
|
||||
ATTRS_MAX_BYTES = 16 * 1024
|
||||
# The client flushes every few seconds and when its buffer fills; this is room for a burst, not a
|
||||
# pace anything legitimate keeps up for a minute.
|
||||
_batches = RateLimiter(max_events=30, window_seconds=60)
|
||||
# What one account may STORE per hour. The pace cap alone still lets a self-consented account write
|
||||
# ~15 MiB a minute into the one shared database, kept for the whole retention window; a real session
|
||||
# at Teljes stores a few hundred kB an hour. Past the budget a batch is accepted and not stored (a
|
||||
# refusal would only make the client resend it), and one notice per hour says so.
|
||||
STORED_BYTES_PER_HOUR = 5 * 1024 * 1024
|
||||
_stored_bytes = RateLimiter(max_events=STORED_BYTES_PER_HOUR, window_seconds=3600)
|
||||
_budget_notices = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
ClientLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
|
||||
|
||||
class ClientEvent(BaseModel):
|
||||
ts: datetime
|
||||
level: ClientLevel
|
||||
event: str = Field(max_length=64, pattern=r"^[a-z][a-z0-9_]*(\.[a-z0-9_]+)*$")
|
||||
msg: str = Field(max_length=8_000)
|
||||
attrs: dict[str, Any] | None = None
|
||||
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$")
|
||||
tab_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,32}$")
|
||||
# The debug session the event was captured in (E16); kept only when it is the one this request
|
||||
# names in `X-Debug-Session` and the auth dependency found it the caller's own and open.
|
||||
debug_session_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{16}$")
|
||||
version: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class ClientEventsIn(BaseModel):
|
||||
events: list[ClientEvent] = Field(max_length=EVENTS_MAX)
|
||||
|
||||
|
||||
async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
"""The body as the model — parsed only after the size cap, which is why this is a dependency and
|
||||
not a plain body parameter (FastAPI reads a declared body in full before anything else runs).
|
||||
|
||||
JSON only, by its Content-Type, as a declared body would be. Parsing any body as JSON would let a
|
||||
CORS-simple `text/plain` form post from a same-site page write events into a signed-in user's
|
||||
log; the app's own client always sends `application/json`, the unload beacon included."""
|
||||
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media_type != "application/json" and not media_type.endswith("+json"):
|
||||
raise HTTPException(status_code=415, detail={"code": "json_required"})
|
||||
raw = await capped_body(request, BODY_MAX_BYTES)
|
||||
try:
|
||||
return ClientEventsIn.model_validate_json(raw)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=json.loads(exc.json())) from exc
|
||||
|
||||
|
||||
def _row(
|
||||
item: ClientEvent,
|
||||
user: User,
|
||||
now: datetime,
|
||||
window_start: datetime,
|
||||
session: DebugSession | None,
|
||||
) -> dict[str, Any]:
|
||||
attrs = logs.redact(item.attrs) if item.attrs else None
|
||||
if attrs is not None:
|
||||
size = len(json.dumps(attrs, ensure_ascii=False, default=str).encode())
|
||||
if size > ATTRS_MAX_BYTES:
|
||||
attrs = {"attrs_dropped_bytes": size}
|
||||
if item.tab_id:
|
||||
attrs = {**(attrs or {}), "tab_id": item.tab_id}
|
||||
ts = item.ts if item.ts.tzinfo else item.ts.replace(tzinfo=timezone.utc)
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": min(max(ts, window_start), now),
|
||||
"level": item.level,
|
||||
"sev": logs.SEVERITY[item.level],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": item.event,
|
||||
"msg": logs.redact_text(item.msg),
|
||||
"attrs": attrs,
|
||||
"request_id": item.request_id,
|
||||
"user_id": user.id,
|
||||
"debug_session_id": (
|
||||
session.id if session and item.debug_session_id == session.id else None
|
||||
),
|
||||
"version": item.version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/events")
|
||||
def ingest_events(
|
||||
# The account first: dependencies resolve in order, so an anonymous caller is refused before a
|
||||
# byte of its body is read.
|
||||
request: Request,
|
||||
user: User = Depends(current_user),
|
||||
body: ClientEventsIn = Depends(_parsed_events),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# A debug session the request names raises the level for its own events (E16): starting one is
|
||||
# the consent to that level for its duration, whatever the user's own level is.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, _ = debugsession.level_for(user, session)
|
||||
if level == diagnostics.BASE:
|
||||
# Lapsed or withdrawn: the client stops streaming when it sees this.
|
||||
raise HTTPException(status_code=409, detail={"code": "diagnostics_off"})
|
||||
if not _batches.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail={"code": "rate_limited"})
|
||||
now = datetime.now(timezone.utc)
|
||||
window_start = now - timedelta(hours=diagnostics.ELEVATION_HOURS)
|
||||
floor = logs.SEVERITY[level]
|
||||
rows = [
|
||||
_row(item, user, now, window_start, session)
|
||||
for item in body.events
|
||||
if logs.SEVERITY[item.level] >= floor
|
||||
]
|
||||
if not rows:
|
||||
return {"ok": True, "stored": 0}
|
||||
size = sum(len(json.dumps(row, ensure_ascii=False, default=str).encode()) for row in rows)
|
||||
stored = len(rows)
|
||||
if not _stored_bytes.allow(str(user.id), size):
|
||||
stored = 0
|
||||
if not _budget_notices.allow(str(user.id)):
|
||||
return {"ok": True, "stored": 0}
|
||||
rows = [_budget_notice(user, now)]
|
||||
# Under the erasure guard until the commit: an account erased meanwhile gets no rows back.
|
||||
with logs.erasure_guard():
|
||||
db.execute(insert(LogEvent), logsink.without_erased(rows))
|
||||
db.commit()
|
||||
return {"ok": True, "stored": stored}
|
||||
|
||||
|
||||
def _budget_notice(user: User, now: datetime) -> dict[str, Any]:
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": now,
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": "diagnostics.budget_spent",
|
||||
# What the budget does, exactly: it is a SLIDING hour, and a refused batch is not
|
||||
# charged to it, so smaller batches keep landing while there is room for them. Saying
|
||||
# "the rest of the hour is not kept" would be a row contradicting the rows beneath it.
|
||||
"msg": "The browser log passed its hourly storage budget; a batch bigger than what is left of it is not kept",
|
||||
"attrs": {"budget_bytes": STORED_BYTES_PER_HOUR},
|
||||
"user_id": user.id,
|
||||
}
|
||||
)
|
||||
@@ -11,9 +11,9 @@ thing it exists to remove.
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -23,7 +23,7 @@ from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
|
||||
@@ -53,7 +53,9 @@ def get_discover(
|
||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||
are not queried twice for a single page load."""
|
||||
resume = rails.continue_watching(db, user)
|
||||
shelf = pins.shelf(db, user)
|
||||
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
||||
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
||||
@@ -125,7 +127,7 @@ def add_pin(
|
||||
constraint is the mechanism, not the message."""
|
||||
resolved = pins.resolve_ref(db, user, body.kind, body.ref)
|
||||
if resolved is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
column, value = resolved
|
||||
|
||||
existing = db.execute(
|
||||
@@ -157,7 +159,7 @@ def add_pin(
|
||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
return {"ok": True, "id": winner.id, "created": False}
|
||||
return {"ok": True, "id": pin.id, "created": True}
|
||||
|
||||
@@ -180,18 +182,11 @@ def reorder_pins(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rewrite the shelf order from a full list of pin ids.
|
||||
"""Rewrite the shelf order from the list of pin ids the client shows.
|
||||
|
||||
Whole-list rather than a move-one delta: the shelf is short, and a full rewrite cannot leave two
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. Ids that are not
|
||||
this user's are ignored rather than 404'd — a stale tab replaying an order that includes a pin
|
||||
deleted meanwhile should reorder what still exists, not fail wholesale."""
|
||||
owned = {p.id for p in pins.list_pins(db, user)}
|
||||
for position, pin_id in enumerate(i for i in body.ids if i in owned):
|
||||
db.execute(
|
||||
update(ComfortPin)
|
||||
.where(ComfortPin.id == pin_id, ComfortPin.user_id == user.id)
|
||||
.values(position=position)
|
||||
)
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. The list may be
|
||||
PARTIAL (hidden Plex pins, a stale tab) — `pins.reorder` keeps what it leaves out in place."""
|
||||
pins.reorder(db, user, body.ids)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
@@ -38,10 +38,11 @@ from app.models import (
|
||||
MediaAsset,
|
||||
User,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
|
||||
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
|
||||
@@ -63,7 +64,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
as a raw URL for yt-dlp's generic extractor."""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="A video link or id is required.")
|
||||
raise refused(400, "link_required")
|
||||
if _VIDEO_ID.match(raw):
|
||||
return "youtube", raw
|
||||
if raw.startswith(("http://", "https://")):
|
||||
@@ -93,7 +94,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
if _VIDEO_ID.match(vid):
|
||||
return "youtube", vid
|
||||
return "url", raw
|
||||
raise HTTPException(status_code=400, detail="That doesn't look like a valid video link.")
|
||||
raise refused(400, "invalid_link")
|
||||
|
||||
|
||||
def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
@@ -107,6 +108,7 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"phase": job.phase,
|
||||
"error": job.error,
|
||||
"error_code": job.error_code,
|
||||
"error_detail": job.error_detail,
|
||||
"queue_pos": job.queue_pos,
|
||||
"created_at": iso(job.created_at),
|
||||
"source_kind": job.source_kind,
|
||||
@@ -165,11 +167,30 @@ def _serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
return _serialize(job, asset)
|
||||
|
||||
|
||||
def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
return job
|
||||
def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
|
||||
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
@@ -213,7 +234,7 @@ def create_profile(
|
||||
name = body.name.strip()
|
||||
spec = body.spec
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
DownloadProfile.user_id == user.id
|
||||
@@ -241,16 +262,16 @@ def update_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
p.name = name[:80]
|
||||
if "spec" in sent:
|
||||
if body.spec is None:
|
||||
raise HTTPException(status_code=400, detail="spec must be an object")
|
||||
raise refused(400, "spec_not_object")
|
||||
p.spec = body.spec
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
@@ -262,7 +283,7 @@ def delete_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
db.delete(p)
|
||||
db.commit()
|
||||
return {"deleted": profile_id}
|
||||
@@ -292,7 +313,7 @@ def set_credential(
|
||||
"""Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is
|
||||
never echoed back, logged, or audited — only the fact that it changed."""
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
raise not_found("host")
|
||||
try:
|
||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||
except dlcreds.InvalidCookies as e:
|
||||
@@ -314,7 +335,7 @@ def delete_credential(
|
||||
host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
raise not_found("host")
|
||||
if dlcreds.delete_credential(db, user.id, host_key):
|
||||
audit.record(
|
||||
db,
|
||||
@@ -335,7 +356,7 @@ def _resolve_spec(
|
||||
if profile_id is not None:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
return p.spec, p.id
|
||||
if spec is not None:
|
||||
return spec, None
|
||||
@@ -346,7 +367,7 @@ def _resolve_spec(
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if fallback is None:
|
||||
raise HTTPException(status_code=400, detail="No download profile available.")
|
||||
raise refused(400, "no_profile")
|
||||
return fallback.spec, fallback.id
|
||||
|
||||
|
||||
@@ -457,7 +478,7 @@ def enqueue_edit(
|
||||
source file is only read, never modified, so editing a shared video is safe."""
|
||||
source_job = _accessible_job(db, user, body.source_job_id)
|
||||
if source_job.status != "done" or source_job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="You can only edit a finished download.")
|
||||
raise refused(409, "edit_unfinished")
|
||||
spec = body.edit_spec
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
try:
|
||||
@@ -560,6 +581,8 @@ def _clean_url(raw) -> str | None:
|
||||
# authority. This one already has a "not a plausible URL" answer, so the typo belongs in it.
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
# silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
|
||||
# answer, the caller already has one, and any line here would carry the URL itself.
|
||||
except ValueError:
|
||||
return None
|
||||
if not parsed.netloc:
|
||||
@@ -585,7 +608,7 @@ def update_download_meta(
|
||||
) -> dict:
|
||||
"""Update a download's display metadata: title, channel name + link, and extra reference URLs.
|
||||
All are display-only overrides — the on-disk file keeps its system-decided, id-bound name."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
sent = body.model_fields_set
|
||||
if "display_name" in sent:
|
||||
job.display_name = ((body.display_name or "").strip())[:255] or None
|
||||
@@ -605,7 +628,7 @@ def update_download_meta(
|
||||
def pause_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -614,7 +637,7 @@ def pause_download(
|
||||
def resume_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -629,18 +652,19 @@ def redownload_download(
|
||||
resolves to the newly-downloaded file — a re-download never breaks a link you already sent out
|
||||
(only a *failed* re-download leaves the job in `error`, which is the intended broken-link case).
|
||||
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the job
|
||||
id + current asset id) rather than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the new
|
||||
asset's own id, and the user's cookie salt when one applies — `credentials.rebuild_salt`) rather
|
||||
than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
under the current format/naming rules — never silently reuses a shared cached file — and (b)
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
raise HTTPException(status_code=409, detail="This item can't be re-downloaded.")
|
||||
raise refused(409, "not_redownloadable")
|
||||
if job.status in ("queued", "running", "paused"):
|
||||
raise HTTPException(status_code=409, detail="This download is still in progress.")
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -649,7 +673,7 @@ def redownload_download(
|
||||
def cancel_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -658,7 +682,7 @@ def cancel_download(
|
||||
def delete_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.delete(db, job)
|
||||
return {"deleted": job_id}
|
||||
|
||||
@@ -669,7 +693,7 @@ def delete_download(
|
||||
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
if job.user_id == user.id:
|
||||
return job
|
||||
shared = db.execute(
|
||||
@@ -678,7 +702,7 @@ def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
)
|
||||
).first()
|
||||
if shared is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
@@ -692,10 +716,10 @@ def download_file(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="File is no longer available.")
|
||||
raise not_found("file")
|
||||
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
@@ -734,7 +758,7 @@ def storyboard_meta(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.download_root)
|
||||
if meta is None:
|
||||
return {"available": False}
|
||||
@@ -755,10 +779,10 @@ def poster_image(job_id: int, user: User = Depends(require_human), db: Session =
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -769,10 +793,10 @@ def storyboard_image(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -790,12 +814,12 @@ def share_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
raise HTTPException(status_code=400, detail="A recipient email is required.")
|
||||
raise refused(400, "recipient_required")
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
@@ -804,7 +828,7 @@ def share_download(
|
||||
if not is_messageable_user(recipient):
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise HTTPException(status_code=400, detail="That's already your download.")
|
||||
raise refused(400, "share_own")
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
|
||||
@@ -823,7 +847,7 @@ def share_download(
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
return {"shared_with": recipient.email}
|
||||
return {"ok": True, "shared_with": recipient.email}
|
||||
|
||||
|
||||
@router.delete("/{job_id}/share/{email}")
|
||||
@@ -833,7 +857,9 @@ def unshare_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
_own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
|
||||
owned_or_404(
|
||||
db, DownloadJob, job_id, user, what="download"
|
||||
) # ownership guard (404s if not the caller's job)
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
@@ -865,7 +891,7 @@ def remove_shared_with_me(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"removed": job_id}
|
||||
return {"ok": True, "removed": job_id}
|
||||
|
||||
|
||||
# --- recipients (for the internal "share with a user" picker) ------------------------------
|
||||
@@ -895,10 +921,10 @@ def share_recipients(
|
||||
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
|
||||
link = db.get(DownloadLink, link_id)
|
||||
if link is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
return link
|
||||
|
||||
|
||||
@@ -914,7 +940,7 @@ def _link_expiry(days: int | None) -> datetime | None:
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
_own_job(db, user, job_id)
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadLink)
|
||||
@@ -945,9 +971,9 @@ def create_link(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
pw = body.password.strip()
|
||||
link = DownloadLink(
|
||||
job_id=job_id,
|
||||
|
||||
+17
-12
@@ -1,6 +1,6 @@
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
@@ -29,10 +29,11 @@ from app.models import (
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import get_or_404
|
||||
from app.routes._common import get_or_404, youtube_refusal, not_found
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["feed"])
|
||||
|
||||
@@ -120,7 +121,7 @@ def get_feed(
|
||||
vals, cursor_id = decode_cursor(keys, cursor)
|
||||
except ValueError:
|
||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||
raise HTTPException(status_code=400, detail="Invalid feed cursor") from None
|
||||
raise refused(400, "invalid_cursor") from None
|
||||
for k, v in zip(keys, vals):
|
||||
k["value"] = v
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
@@ -235,7 +236,7 @@ def set_video_state(
|
||||
) -> dict:
|
||||
status = body.status
|
||||
if status not in VALID_STATES:
|
||||
raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATES}")
|
||||
raise refused(400, "invalid_status")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
@@ -256,9 +257,11 @@ def set_video_state(
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the documented lost race — a concurrent player checkpoint already removed
|
||||
# or moved this row. The state the caller asked for is the state that holds either way.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new"}
|
||||
return {"ok": True, "video_id": video_id, "status": "new"}
|
||||
|
||||
# watched / hidden: atomic upsert. The player marks "watched" at end-of-playback at
|
||||
# the same instant a progress checkpoint may DELETE the in-progress row; a plain
|
||||
@@ -279,7 +282,7 @@ def set_video_state(
|
||||
)
|
||||
db.execute(stmt)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "status": status}
|
||||
return {"ok": True, "video_id": video_id, "status": status}
|
||||
|
||||
|
||||
@router.delete("/videos/{video_id}/state")
|
||||
@@ -298,9 +301,10 @@ def clear_video_state(
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
|
||||
|
||||
class VideoProgressIn(BaseModel):
|
||||
@@ -355,9 +359,10 @@ def set_video_progress(
|
||||
# have removed/updated this row already — tolerate the lost race.
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: exactly the race the comment above describes; the response stays true.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
||||
|
||||
if row is None:
|
||||
row = VideoState(user_id=user.id, video_id=video_id)
|
||||
@@ -365,7 +370,7 @@ def set_video_progress(
|
||||
row.position_seconds = position
|
||||
row.progress_updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "position_seconds": position}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
||||
|
||||
|
||||
@router.get("/videos/{video_id}")
|
||||
@@ -398,7 +403,7 @@ def get_video_detail(
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit, so the
|
||||
# demo account can't reach it — it just sees "unknown" for a video we don't already store.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
try:
|
||||
with (
|
||||
@@ -407,9 +412,9 @@ def get_video_detail(
|
||||
):
|
||||
items = yt.get_videos([video_id])
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube lookup failed: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
if not items:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
snippet = items[0].get("snippet", {})
|
||||
stats = items[0].get("statistics", {})
|
||||
|
||||
+74
-24
@@ -1,5 +1,7 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, RootModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -7,6 +9,7 @@ from app.auth import (
|
||||
count_admins,
|
||||
count_pending_activations,
|
||||
current_user,
|
||||
drop_from_wallet,
|
||||
google_enabled,
|
||||
has_read_scope,
|
||||
has_write_scope,
|
||||
@@ -14,10 +17,16 @@ from app.auth import (
|
||||
optional_current_user,
|
||||
purge_user,
|
||||
)
|
||||
from app import debugsession, diagnostics
|
||||
from app.issues import gitea
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import sysconfig
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/me", tags=["me"])
|
||||
|
||||
@@ -64,28 +73,28 @@ def switch_account(
|
||||
here) are allowed, and access is re-checked in case the invite was revoked since."""
|
||||
target = body.user_id
|
||||
if target not in (request.session.get("account_ids") or []):
|
||||
raise HTTPException(status_code=403, detail="Not an account you've signed into here.")
|
||||
raise refused(403, "not_your_account")
|
||||
# An account `_session_user` would refuse — gone, deactivated, suspended, or its session revoked — is
|
||||
# refused here too, and dropped from the switch list the way `_session_user` drops it. Made the session
|
||||
# default instead, the next request's current_user would refuse it as the DEFAULT and clear the whole
|
||||
# wallet session, logging out every account here. (B-core-9)
|
||||
u = db.get(User, target)
|
||||
if u is None:
|
||||
raise HTTPException(status_code=404, detail="That account no longer exists.")
|
||||
drop_from_wallet(request.session, target)
|
||||
raise not_found("account")
|
||||
if not is_allowed(db, u.email):
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
raise refused(403, "account_no_access")
|
||||
if not u.is_active:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_no_access")
|
||||
if u.is_suspended:
|
||||
# Don't make a suspended account the active one — the next request's current_user would
|
||||
# see the suspension and clear the WHOLE wallet session, logging out every account here.
|
||||
raise HTTPException(status_code=403, detail="That account is suspended.")
|
||||
# Same reasoning for a REVOKED session (a password reset/change or "log out everywhere" bumped the
|
||||
# account's epoch): making it the session default would trip current_user's epoch check on the next
|
||||
# request and clear the whole wallet. Mirror the suspend guard — drop the stale account from the
|
||||
# switch list and refuse, rather than logging everyone out. (B-core-9)
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_suspended")
|
||||
# Revoked: a password reset/change or "log out everywhere" bumped the account's epoch.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(target), 0)) != (u.session_epoch or 0):
|
||||
request.session["account_ids"] = [
|
||||
a for a in (request.session.get("account_ids") or []) if a != target
|
||||
]
|
||||
raise HTTPException(
|
||||
status_code=403, detail="That account was signed out elsewhere — sign in again."
|
||||
)
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_signed_out")
|
||||
request.session["user_id"] = target
|
||||
return {"ok": True, "user_id": target}
|
||||
|
||||
@@ -131,6 +140,9 @@ def get_me(
|
||||
user.google_email if user.google_email and user.google_email != user.email else None
|
||||
),
|
||||
"has_password": user.password_hash is not None,
|
||||
# The account's latest sign-in on any device — shown in Settings → Account, so a sign-in the
|
||||
# user does not recognise is something they can see (null: none since 0.88, or the demo).
|
||||
"last_login_at": iso(user.last_login_at),
|
||||
# Instance-wide: whether Google sign-in / YouTube connect is available at all (the UI hides
|
||||
# YouTube-access affordances and the onboarding nudge when it isn't configured).
|
||||
"google_enabled": google_enabled(db),
|
||||
@@ -144,9 +156,21 @@ def get_me(
|
||||
"pending_activations": pending_activations,
|
||||
"email_configured": email_configured,
|
||||
"preferences": user.preferences or {},
|
||||
# The consented diagnostics level in force and when it lapses (E22 S5). The client applies
|
||||
# it to its own log and streams that log while it is raised.
|
||||
"diagnostics": diagnostics.state(db, user),
|
||||
# The live debug session, if one is recording (E16) — a reload resumes it.
|
||||
"debug_session": _live_session(db, user),
|
||||
# Where a bug report goes (E16 S4): the tracker's repository, or None — download only.
|
||||
"issues": gitea.target(db),
|
||||
}
|
||||
|
||||
|
||||
def _live_session(db: Session, user: User) -> dict | None:
|
||||
session = debugsession.open_session(db, user)
|
||||
return None if session is None else debugsession.view(db, session)
|
||||
|
||||
|
||||
@router.delete("/account")
|
||||
def delete_my_account(
|
||||
request: Request,
|
||||
@@ -167,10 +191,7 @@ def delete_my_account(
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "delete_account"})
|
||||
if user.role == "admin" and count_admins(db) <= 1:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="You're the only admin — promote another admin before deleting your account.",
|
||||
)
|
||||
raise refused(400, "only_admin_delete")
|
||||
user_id = user.id
|
||||
# Full erasure (cascades) + access-request cleanup + Google-grant revocation; shared with the
|
||||
# admin delete path. Capture the id first — `user` is detached once purge_user deletes the row.
|
||||
@@ -185,16 +206,45 @@ def delete_my_account(
|
||||
return {"deleted": True}
|
||||
|
||||
|
||||
class PreferencesIn(RootModel[dict[str, Any]]):
|
||||
"""Free-form UI preferences — a JSON object of any keys (theme, filters, …), merged into the
|
||||
stored ones. A model rather than a bare `dict` parameter so a non-object body is a 422 at the
|
||||
boundary like every other request body."""
|
||||
|
||||
|
||||
@router.put("/preferences")
|
||||
def update_preferences(
|
||||
preferences: dict,
|
||||
preferences: PreferencesIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# Merge so partial updates don't wipe other keys.
|
||||
merged = dict(user.preferences or {})
|
||||
merged.update(preferences)
|
||||
merged.update(preferences.root)
|
||||
user.preferences = merged
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return {"preferences": merged}
|
||||
return {"ok": True, "preferences": merged}
|
||||
|
||||
|
||||
class DiagnosticsIn(BaseModel):
|
||||
level: diagnostics.DiagnosticsLevel
|
||||
|
||||
|
||||
@router.put("/diagnostics")
|
||||
def update_diagnostics(
|
||||
body: DiagnosticsIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Raise the diagnostics level (the user consented in Settings → Diagnosztika) or return to the
|
||||
base. The expiry is the server's: a raised level always lapses after
|
||||
`diagnostics.ELEVATION_HOURS`, whatever the client asks. The shared demo account cannot raise
|
||||
it — its log would interleave every visitor's session under one account."""
|
||||
if user.is_demo and body.level != diagnostics.BASE:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "diagnostics"})
|
||||
diagnostics.set_level(user, body.level)
|
||||
db.commit()
|
||||
# The rest of THIS request, and the process's elevation registry, follow the new level now.
|
||||
logs.set_user(user.id, diagnostics.effective_level(user), user.diagnostics_until)
|
||||
return {"ok": True, "diagnostics": diagnostics.state(db, user)}
|
||||
|
||||
@@ -16,7 +16,7 @@ routing and unread counts.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
|
||||
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -29,6 +29,8 @@ from app.realtime import manager
|
||||
from app.security import verify_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/messages", tags=["messages"])
|
||||
|
||||
@@ -46,7 +48,7 @@ WELCOME = {
|
||||
"This is your private message inbox. Conversations with other members are end-to-end "
|
||||
"encrypted — only you and the person you're chatting with can read them, not even an "
|
||||
"admin. To start messaging someone, set up secure messaging with a passphrase first.\n\n"
|
||||
"Enjoy your feed!"
|
||||
"Enjoy your Feed!"
|
||||
),
|
||||
"hu": (
|
||||
"Üdv a Siftlode-on! 👋\n\n"
|
||||
@@ -152,7 +154,7 @@ def setup_keys(
|
||||
"""Store this user's public key + passphrase-wrapped private key. Set-once: changing the
|
||||
key would orphan every existing conversation key, so re-setup is rejected here."""
|
||||
if db.get(MessageKey, user.id) is not None:
|
||||
raise HTTPException(status_code=409, detail="Secure messaging is already set up.")
|
||||
raise refused(409, "messaging_set_up")
|
||||
db.add(
|
||||
MessageKey(
|
||||
user_id=user.id,
|
||||
@@ -164,7 +166,7 @@ def setup_keys(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"configured": True}
|
||||
return {"ok": True, "configured": True}
|
||||
|
||||
|
||||
@router.post("/keys/reset")
|
||||
@@ -185,11 +187,11 @@ def reset_keys(
|
||||
less (Google-only) account has nothing to re-verify, so it relies on its live session plus the
|
||||
client-side danger confirmation."""
|
||||
if not _reset_key_limiter.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again shortly.")
|
||||
raise refused(429, "too_many_attempts")
|
||||
if user.password_hash and not verify_password(
|
||||
payload.current_password or "", user.password_hash
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Current password is incorrect.")
|
||||
raise refused(403, "wrong_password")
|
||||
# Delete the now-dead ciphertext first (both directions), then the wrapped keypair itself so a
|
||||
# fresh setup is allowed. Order doesn't matter (no FK between them), but do it in one commit.
|
||||
db.execute(
|
||||
@@ -202,7 +204,7 @@ def reset_keys(
|
||||
if key is not None:
|
||||
db.delete(key)
|
||||
db.commit()
|
||||
return {"configured": False}
|
||||
return {"ok": True, "configured": False}
|
||||
|
||||
|
||||
@router.get("/keys/{user_id}")
|
||||
@@ -223,10 +225,10 @@ def public_key(
|
||||
p = db.get(User, user_id)
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
raise not_found("user")
|
||||
k = db.get(MessageKey, user_id)
|
||||
if k is None:
|
||||
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
|
||||
raise refused(404, "no_messaging")
|
||||
return {"user_id": user_id, "public_key": k.public_key}
|
||||
|
||||
|
||||
@@ -344,7 +346,7 @@ def get_thread(
|
||||
# still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle.
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
raise not_found("user")
|
||||
partner = _serialize_user(p)
|
||||
msgs = (
|
||||
db.execute(select(Message).where(cond).order_by(Message.created_at.asc())).scalars().all()
|
||||
@@ -374,20 +376,20 @@ def send_message(
|
||||
"""Send an end-to-end-encrypted message. The server stores only the ciphertext the client
|
||||
produced, then pushes it live to the recipient (and the sender's other devices)."""
|
||||
if not _send_limiter.allow(f"msg:{user.id}"):
|
||||
raise HTTPException(status_code=429, detail="Too many messages — slow down a moment.")
|
||||
raise refused(429, "message_rate")
|
||||
ct = (payload.ciphertext or "").strip()
|
||||
iv = (payload.iv or "").strip()
|
||||
if not ct or not iv:
|
||||
raise HTTPException(status_code=400, detail="Message is empty.")
|
||||
raise refused(400, "message_empty")
|
||||
if len(ct) > MAX_CIPHERTEXT:
|
||||
raise HTTPException(status_code=400, detail="Message is too long.")
|
||||
raise refused(400, "message_too_long")
|
||||
if payload.recipient_id == user.id:
|
||||
raise HTTPException(status_code=400, detail="You can't message yourself.")
|
||||
raise refused(400, "message_self")
|
||||
recipient = db.get(User, payload.recipient_id)
|
||||
if not is_messageable_user(recipient):
|
||||
raise HTTPException(status_code=404, detail="Recipient not available.")
|
||||
raise not_found("recipient")
|
||||
if db.get(MessageKey, recipient.id) is None:
|
||||
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
|
||||
raise refused(404, "no_messaging")
|
||||
m = Message(kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv)
|
||||
db.add(m)
|
||||
db.commit()
|
||||
@@ -423,9 +425,10 @@ async def messages_ws(ws: WebSocket) -> None:
|
||||
"""Live push channel: authenticated by the session cookie, registers the connection so
|
||||
sent messages reach this user's open tabs instantly. We don't consume client→server frames
|
||||
(sending goes through POST /api/messages); the receive loop only detects disconnect."""
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (wallet-gated
|
||||
# ?account= override, session default otherwise). A browser WebSocket can't send the
|
||||
# X-Siftlode-Account header, so resolved_user_id falls through to the ?account= query param.
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (a wallet-gated
|
||||
# ?account=, the session default when none is named, nobody for an account no longer signed in
|
||||
# here). A browser WebSocket can't send the X-Siftlode-Account header, so resolved_user_id falls
|
||||
# through to the ?account= query param.
|
||||
uid, _ = resolved_user_id(ws)
|
||||
if not uid:
|
||||
await ws.close(code=1008)
|
||||
@@ -448,6 +451,8 @@ async def messages_ws(ws: WebSocket) -> None:
|
||||
try:
|
||||
while True:
|
||||
await ws.receive_text()
|
||||
# silent-ok: the ordinary way a chat socket ends — a closed tab. The `finally` below is the
|
||||
# whole handling, and nothing failed.
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
|
||||
@@ -5,7 +5,7 @@ All endpoints use `current_user` (not `require_human`): notifications and their
|
||||
state are personal UI state with no quota or YouTube involvement, so the shared demo account
|
||||
manages its own inbox normally."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.db import get_db
|
||||
from app.models import Notification, User
|
||||
from app.notifications import trim_read
|
||||
from app.utils import iso
|
||||
from app.routes._common import owned_or_404
|
||||
|
||||
router = APIRouter(prefix="/api/me/notifications", tags=["notifications"])
|
||||
|
||||
@@ -72,20 +73,13 @@ def unread_count(user: User = Depends(current_user), db: Session = Depends(get_d
|
||||
return {"count": count or 0}
|
||||
|
||||
|
||||
def _owned(db: Session, notification_id: int, user: User) -> Notification:
|
||||
n = db.get(Notification, notification_id)
|
||||
if n is None or n.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Notification not found")
|
||||
return n
|
||||
|
||||
|
||||
@router.post("/{notification_id}/read")
|
||||
def mark_read(
|
||||
notification_id: int,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
n = _owned(db, notification_id, user)
|
||||
n = owned_or_404(db, Notification, notification_id, user, what="notification")
|
||||
n.read = True
|
||||
db.commit()
|
||||
trim_read(db, user.id)
|
||||
@@ -110,7 +104,7 @@ def dismiss(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
n = _owned(db, notification_id, user)
|
||||
n = owned_or_404(db, Notification, notification_id, user, what="notification")
|
||||
# Dismiss implies read (it no longer counts toward the unread badge).
|
||||
n.dismissed = True
|
||||
n.read = True
|
||||
|
||||
@@ -3,11 +3,10 @@ CRUD (create / rename / delete, add / remove / reorder items) plus YouTube two-w
|
||||
mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert local edits back
|
||||
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, and_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app import quota
|
||||
@@ -23,7 +22,7 @@ from app.sync.playlists import (
|
||||
sync_user_playlists,
|
||||
)
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import get_or_404, owned_or_404
|
||||
from app.routes._common import get_or_404, owned_or_404, youtube_refusal
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||
@@ -48,10 +47,6 @@ def _check_pull_cooldown(user: User) -> None:
|
||||
raise HTTPException(status_code=429, detail={"code": "sync_cooldown"})
|
||||
|
||||
|
||||
def _own_playlist(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
return owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
|
||||
|
||||
def _refused(status: int, reason: str) -> HTTPException:
|
||||
"""A playlist action the server refuses, as a reason the UI translates
|
||||
(`playlists.errors.<reason>`) — a bare English detail reached the Hungarian UI verbatim."""
|
||||
@@ -64,8 +59,11 @@ def _touch(pl: Playlist) -> None:
|
||||
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
|
||||
reorder / move write the item rows instead — so without this the rail's "recently changed"
|
||||
sort would only ever see renames. The assignment is what makes the column mean "last time
|
||||
this list changed", which is what the rail promises the reader."""
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
this list changed", which is what the rail promises the reader.
|
||||
|
||||
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
|
||||
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
|
||||
pl.updated_at = func.now()
|
||||
|
||||
|
||||
def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
@@ -108,7 +106,7 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
@@ -126,7 +124,7 @@ def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
@@ -280,7 +278,7 @@ def _watch_later(db: Session, user: User) -> Playlist:
|
||||
position=_next_playlist_position(db, user.id),
|
||||
)
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
db.flush() # the id, for the item the caller adds next; the handler commits
|
||||
return pl
|
||||
|
||||
|
||||
@@ -295,7 +293,8 @@ def add_watch_later(
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
pl = _watch_later(db, user)
|
||||
_add_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": True, "playlist_id": pl.id}
|
||||
db.commit()
|
||||
return {"ok": True, "saved": True, "playlist_id": pl.id}
|
||||
|
||||
|
||||
@router.delete("/watch-later/{video_id}")
|
||||
@@ -309,7 +308,8 @@ def remove_watch_later(
|
||||
)
|
||||
if pl is not None:
|
||||
_remove_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": False}
|
||||
db.commit()
|
||||
return {"ok": True, "saved": False}
|
||||
|
||||
|
||||
@router.post("/sync-youtube")
|
||||
@@ -322,8 +322,12 @@ def sync_youtube(
|
||||
if not has_read_scope(user):
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return sync_user_playlists(db, user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **sync_user_playlists(db, user)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/revert-youtube")
|
||||
@@ -335,7 +339,7 @@ def revert_youtube(
|
||||
"""Reset one YouTube-linked playlist to its current YouTube state, discarding local
|
||||
edits (and clearing the dirty flag). The user-facing 'undo all my changes' for a list
|
||||
once the in-session undo history is gone."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_read_scope(user):
|
||||
@@ -343,16 +347,16 @@ def revert_youtube(
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return repull_playlist(db, user, pl)
|
||||
except YouTubeError:
|
||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise _refused(422, "youtube_failed")
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
"""A local playlist the user may push to YouTube (create or update). The built-in
|
||||
Watch later and read-only YouTube mirrors are never pushable."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_not_pushable")
|
||||
return pl
|
||||
@@ -371,8 +375,8 @@ def push_plan(
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
plan = plan_merge(db, user, pl)
|
||||
except YouTubeError:
|
||||
raise _refused(422, "youtube_failed")
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
plan["remaining_today"] = quota.remaining_today(db)
|
||||
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
||||
return plan
|
||||
@@ -403,10 +407,10 @@ def push(
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return merge_playlist(db, user, pl, live=live)
|
||||
except YouTubeError:
|
||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise _refused(422, "youtube_failed")
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.get("/{playlist_id}")
|
||||
@@ -415,7 +419,7 @@ def get_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
state = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, state))
|
||||
@@ -447,7 +451,7 @@ def rename_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if "name" in body.model_fields_set:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
@@ -484,7 +488,7 @@ def delete_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_undeletable")
|
||||
if on_youtube:
|
||||
@@ -496,8 +500,8 @@ def delete_playlist(
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError:
|
||||
raise _refused(422, "youtube_failed")
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
return {"deleted": playlist_id}
|
||||
@@ -510,11 +514,12 @@ def add_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
video_id = body.video_id
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
_add_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
|
||||
@router.delete("/{playlist_id}/items/{video_id}")
|
||||
@@ -524,9 +529,10 @@ def remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
_remove_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
|
||||
class MoveItemIn(BaseModel):
|
||||
@@ -548,7 +554,7 @@ def move_item(
|
||||
exactly the kind of half-applied state this codebase treats as a bug rather than an edge
|
||||
case. Idempotent: if the video is already in the target, the source row is simply dropped
|
||||
(moving into a list that already has it is a removal, not a duplicate)."""
|
||||
src = _own_playlist(db, user, playlist_id)
|
||||
src = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
# Its own reason: "this playlist doesn't exist any more" about a TARGET that is gone was read as
|
||||
# the list on screen — which is still there.
|
||||
dst = owned_or_404(db, Playlist, body.target_playlist_id, user, what="move_target")
|
||||
@@ -571,19 +577,33 @@ def move_item(
|
||||
PlaylistItem.playlist_id == dst.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
if already is not None:
|
||||
db.delete(item) # the uq_playlist_video constraint would reject a second row anyway
|
||||
else:
|
||||
dst_changed = False
|
||||
if already is None:
|
||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||
# survives the move, and there is no window where the video belongs to neither list.
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
# Both sides changed, so both get the dirty recompute and the timestamp — the asymmetric
|
||||
# version of this (touching only the source) is the recurring smell CLAUDE.md warns about.
|
||||
# survives the move, and there is no window where the video belongs to neither list. The
|
||||
# check above is not a lock: a second tab's add or the scheduler's YouTube merge can insert
|
||||
# the same (playlist, video) in between, and uq_playlist_video then refuses the re-parent.
|
||||
# That is the already-present outcome arriving late, not a failure — answer it the same way
|
||||
# instead of a raw 500.
|
||||
try:
|
||||
with db.begin_nested():
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
db.flush()
|
||||
dst_changed = True
|
||||
# silent-ok: the conflict IS the answer — the target already holds the video, which is the
|
||||
# idempotent outcome; the savepoint put the row back in the source list, dropped below.
|
||||
except IntegrityError:
|
||||
pass
|
||||
if not dst_changed:
|
||||
db.delete(item)
|
||||
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
||||
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
|
||||
recompute_dirty(db, src)
|
||||
recompute_dirty(db, dst)
|
||||
_touch(src)
|
||||
_touch(dst)
|
||||
if dst_changed:
|
||||
recompute_dirty(db, dst)
|
||||
_touch(dst)
|
||||
db.commit()
|
||||
return {"ok": True, "from_playlist_id": src.id, "to_playlist_id": dst.id, "video_id": video_id}
|
||||
|
||||
@@ -599,7 +619,7 @@ def reorder_items(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
order = body.video_ids
|
||||
items = {
|
||||
it.video_id: it
|
||||
@@ -625,4 +645,4 @@ def reorder_items(
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
return {"playlist_id": pl.id, "count": pos}
|
||||
return {"ok": True, "playlist_id": pl.id, "count": pos}
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
"""Shared helpers, serializers and constants for the plex route package."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexItem
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex item")
|
||||
raise not_found("plex_item")
|
||||
return it
|
||||
|
||||
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -20,6 +20,9 @@ from app.plex import sync as plex_sync
|
||||
from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -35,10 +38,15 @@ def test_connection(_: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
info = plex.server_info()
|
||||
sections = plex.sections()
|
||||
media_check = _media_check(db, plex, sections)
|
||||
except PlexNotConfigured as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except PlexNotConfigured:
|
||||
raise refused(400, "plex_not_configured") from None
|
||||
except PlexError as e:
|
||||
raise HTTPException(status_code=422, detail=f"Plex connection failed: {e}")
|
||||
# What the upstream said is for the log, not the toast: it can quote the server URL back.
|
||||
log.warning(
|
||||
"the Plex connection test failed",
|
||||
extra=logs.event("plex.admin.connection_failed", reason=type(e).__name__, error=str(e)),
|
||||
)
|
||||
raise refused(422, "plex_unreachable") from None
|
||||
return {
|
||||
"ok": True,
|
||||
"server_name": info.get("friendlyName") or info.get("title1") or "Plex",
|
||||
@@ -95,7 +103,7 @@ def _media_check(db: Session, plex: PlexClient, sections: list[dict]) -> dict:
|
||||
def trigger_sync(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Admin: mirror the enabled Plex sections into the local catalog. Synchronous (can take a
|
||||
while on a large library); a background/scheduled sync also runs periodically."""
|
||||
return plex_sync.sync(db)
|
||||
return {"ok": True, **plex_sync.sync(db)}
|
||||
|
||||
|
||||
def _link_dict(link: PlexLink | None) -> dict:
|
||||
@@ -129,7 +137,7 @@ def watch_link_set(
|
||||
admin token, so no separate Plex login. Enabling it for the first time runs the one-time
|
||||
"Plex is master" import immediately so the owner's existing Plex history shows up in Siftlode."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
raise HTTPException(status_code=409, detail="The Plex module is disabled.")
|
||||
raise refused(409, "plex_disabled")
|
||||
enabled = body.enabled
|
||||
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
|
||||
if link is None:
|
||||
@@ -141,7 +149,7 @@ def watch_link_set(
|
||||
result: dict = {}
|
||||
if enabled and not link.initial_import_done:
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
return {**_link_dict(link), "import": result or None}
|
||||
return {"ok": True, **_link_dict(link), "import": result or None}
|
||||
|
||||
|
||||
@router.post("/watch/import")
|
||||
@@ -150,6 +158,6 @@ def watch_import_now(user: User = Depends(admin_user), db: Session = Depends(get
|
||||
intersection; Siftlode-only states are preserved. Idempotent."""
|
||||
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
|
||||
if link is None or not link.sync_enabled:
|
||||
raise HTTPException(status_code=409, detail="Enable Plex watch-sync first.")
|
||||
raise refused(409, "watch_sync_off")
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
return {**_link_dict(link), "import": result}
|
||||
return {"ok": True, **_link_dict(link), "import": result}
|
||||
|
||||
@@ -9,6 +9,7 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -24,7 +25,12 @@ from app.plex import sync as plex_sync
|
||||
from app.plex.cards import movie_card, show_card
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.query import content_libs, like_escape, runtime_by_key, top_art_keys
|
||||
from app.plex.query import content_libs, runtime_by_key, top_art_keys
|
||||
from app.utils import like_escape
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# The most-restrictive → least ordering used to pick a collection's effective content rating from its
|
||||
# members (Plex shows one cert on the collection; we don't store it, so derive it).
|
||||
@@ -233,7 +239,7 @@ def collection_detail(
|
||||
endpoints below."""
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
raise not_found("collection")
|
||||
movies = (
|
||||
db.execute(
|
||||
select(PlexItem)
|
||||
@@ -324,9 +330,9 @@ def collection_detail(
|
||||
def _editable_collection_or_403(db: Session, rating_key: str) -> PlexCollection:
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
raise not_found("collection")
|
||||
if not _collection_editable(col):
|
||||
raise HTTPException(status_code=403, detail="This collection is read-only")
|
||||
raise refused(403, "collection_read_only")
|
||||
return col
|
||||
|
||||
|
||||
@@ -361,14 +367,13 @@ def _write_or_pending_error(wrote: bool, e: Exception) -> HTTPException:
|
||||
Both use 422, NOT a 5xx: the frontend treats 502/503/504 as "gateway unreachable" and raises a
|
||||
sticky offline banner (see api/core.ts RETRIABLE_GATEWAY). A 422 surfaces the detail as an ordinary
|
||||
action-error toast, which is what both of these are — the app answered, it didn't go unreachable."""
|
||||
if wrote:
|
||||
detail = (
|
||||
"Collection updated in Plex, but refreshing the local copy failed; "
|
||||
f"it will reconcile on the next sync ({e})"
|
||||
)
|
||||
else:
|
||||
detail = f"Plex write failed: {e}"
|
||||
return HTTPException(status_code=422, detail=detail)
|
||||
log.warning(
|
||||
"a Plex collection write did not settle",
|
||||
extra=logs.event(
|
||||
"plex.collections.write_failed", wrote=wrote, reason=type(e).__name__, error=str(e)
|
||||
),
|
||||
)
|
||||
return refused(422, "collection_pending_sync") if wrote else refused(422, "plex_write_failed")
|
||||
|
||||
|
||||
class CreateCollectionIn(BaseModel):
|
||||
@@ -388,7 +393,7 @@ def create_collection(
|
||||
title = body.title.strip()
|
||||
seed = body.item_rating_key
|
||||
if not title or not seed:
|
||||
raise HTTPException(status_code=400, detail="title and item_rating_key are required")
|
||||
raise refused(400, "collection_fields_required")
|
||||
if library:
|
||||
lib = (
|
||||
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == library)).scalars().first()
|
||||
@@ -398,7 +403,7 @@ def create_collection(
|
||||
seed_item = by_rating_key(db, PlexItem, seed) or by_rating_key(db, PlexShow, seed)
|
||||
lib = db.get(PlexLibrary, seed_item.library_id) if seed_item else None
|
||||
if lib is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown library")
|
||||
raise not_found("library")
|
||||
_reject_duplicate_name(db, lib.id, title)
|
||||
wrote = False
|
||||
try:
|
||||
@@ -472,7 +477,7 @@ def collection_rename(
|
||||
col = _editable_collection_or_403(db, rating_key)
|
||||
title = body.title.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="title is required")
|
||||
raise refused(400, "title_required")
|
||||
_reject_duplicate_name(db, col.library_id, title, exclude_rk=rating_key)
|
||||
lib = db.get(PlexLibrary, col.library_id)
|
||||
wrote = False
|
||||
@@ -501,7 +506,7 @@ def collection_delete(
|
||||
db.commit()
|
||||
except (PlexError, PlexNotConfigured) as e:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail=f"Plex write failed: {e}")
|
||||
raise _write_or_pending_error(False, e) from None
|
||||
return {"deleted": rating_key}
|
||||
|
||||
|
||||
@@ -520,13 +525,10 @@ def collection_set_editable(
|
||||
(IMDb/TMDb/…) can never be made editable — they're regenerated outside Plex."""
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
raise not_found("collection")
|
||||
want = body.editable
|
||||
if want and (col.smart or _collection_source(col) != "collection"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Smart / auto-list collections can't be made editable",
|
||||
)
|
||||
raise refused(400, "collection_smart")
|
||||
col.editable = want
|
||||
db.commit()
|
||||
return _collection_card(col)
|
||||
|
||||
@@ -5,10 +5,11 @@ import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -29,6 +30,9 @@ from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes.plex._shared import _norm_lang, item_or_404
|
||||
from app.routes.plex.collections import _collection_source
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -65,7 +69,21 @@ def _show_enrichment(db: Session, rating_key: str) -> tuple[dict, list]:
|
||||
related_f = ex.submit(plex.related, rating_key) # swallows its own errors → []
|
||||
meta = meta_f.result() or {}
|
||||
related_raw = related_f.result()
|
||||
except (PlexError, PlexNotConfigured):
|
||||
except PlexNotConfigured:
|
||||
# silent-ok: not a failure but a state — this instance has no Plex server, and the page
|
||||
# is meant to render from the local rows alone.
|
||||
return {}, []
|
||||
except PlexError as exc:
|
||||
# The detail page still renders, from the rows the sync left behind: no cast, no related
|
||||
# strip, no rich detail. Indistinguishable on screen from a show that simply has none.
|
||||
log.warning(
|
||||
"live Plex enrichment failed for a show",
|
||||
extra=logs.event(
|
||||
"plex.detail.show_enrich_failed",
|
||||
rating_key=rating_key,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return {}, []
|
||||
# Don't cache an EMPTY related list: plex.related() swallows a transient failure as [] (same as a
|
||||
# genuinely no-related show), so caching it would serve an empty "related" section for the whole TTL
|
||||
@@ -92,7 +110,7 @@ def show_detail(
|
||||
hu = _norm_lang(lang) == "hu"
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex show")
|
||||
raise not_found("show")
|
||||
seasons = (
|
||||
db.execute(
|
||||
select(PlexSeason)
|
||||
@@ -145,12 +163,26 @@ def show_detail(
|
||||
sh.people_enriched_at = datetime.now(timezone.utc)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a failed enrich must not fail the page
|
||||
# The rollback un-does the enrichment AND the `people_enriched_at` stamp, so the
|
||||
# next view tries again — but if the commit keeps failing nobody ever hears, and
|
||||
# the cast section stays empty for good.
|
||||
log.warning(
|
||||
"the show enrichment could not be committed",
|
||||
extra=logs.event(
|
||||
"plex.detail.enrich_commit_failed", rating_key=sh.rating_key, kind="show"
|
||||
),
|
||||
)
|
||||
db.rollback()
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — the local episode list must render whatever Plex does
|
||||
# Best-effort only — the local episode list is already assembled, so any live-Plex failure
|
||||
# (not configured, HTTP/JSON shape) must degrade, not 500.
|
||||
pass
|
||||
# (not configured, HTTP/JSON shape) must degrade, not 500. With the traceback, because what
|
||||
# is left in here after the Plex errors above are handled is our own shape bugs.
|
||||
log.warning(
|
||||
"the live part of a show page could not be built",
|
||||
extra=logs.event("plex.detail.show_extras_failed", rating_key=sh.rating_key),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
imdb_id = sh.imdb_id
|
||||
return {
|
||||
@@ -295,6 +327,14 @@ def _episode_nav(db: Session, it: PlexItem) -> tuple[str | None, str | None]:
|
||||
try:
|
||||
i = keys.index(it.rating_key)
|
||||
except ValueError:
|
||||
# An episode missing from its own show's ordered list — the two came out of the same sync,
|
||||
# so this is a gap in it. The page loses its previous/next arrows and says nothing.
|
||||
log.info(
|
||||
"an episode is not in its show's episode list; no previous/next",
|
||||
extra=logs.event(
|
||||
"plex.detail.episode_nav_missing", rating_key=it.rating_key, show_id=it.show_id
|
||||
),
|
||||
)
|
||||
return None, None
|
||||
return (keys[i - 1] if i > 0 else None), (keys[i + 1] if i < len(keys) - 1 else None)
|
||||
|
||||
@@ -366,8 +406,32 @@ def item_detail(
|
||||
"text": codec not in subtitles.BITMAP_SUBS,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass # live extras (markers/streams) are best-effort; any live-Plex failure must degrade, not 500 — playback works without them
|
||||
# Best-effort, but not free: what either branch drops is the subtitle stream list and the intro
|
||||
# markers, so the player offers no subtitles and no skip button — and looks like a file that has
|
||||
# neither.
|
||||
except PlexNotConfigured:
|
||||
# silent-ok: not a failure but a state — no Plex server, and nothing to play from anyway.
|
||||
pass
|
||||
except PlexError as exc:
|
||||
log.warning(
|
||||
"the live extras for an item could not be fetched",
|
||||
extra=logs.event(
|
||||
"plex.detail.extras_failed",
|
||||
rating_key=it.rating_key,
|
||||
kind=it.kind,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001 — playback works without the extras; never 500 here
|
||||
# Plex answered and WE could not read it — a shape our parsing does not expect. The same
|
||||
# split `show_detail` makes: the stack is the only thing that says where.
|
||||
log.warning(
|
||||
"the live extras for an item could not be read",
|
||||
extra=logs.event(
|
||||
"plex.detail.extras_unreadable", rating_key=it.rating_key, kind=it.kind
|
||||
),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Lazy enrichment (movies only): the live fetch above carries the FULL cast + rich detail (the sync
|
||||
# only stored the listing's top ~3), so fill a not-yet-enriched movie — its cast becomes filterable
|
||||
@@ -380,7 +444,15 @@ def item_detail(
|
||||
it.people_enriched_at = datetime.now(timezone.utc)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a failed enrich must not fail the page
|
||||
# Same as the show path above: the info page then serves the three names the listing
|
||||
# sync stored, and the full cast it just fetched is thrown away unreported.
|
||||
log.warning(
|
||||
"the movie enrichment could not be committed",
|
||||
extra=logs.event(
|
||||
"plex.detail.enrich_commit_failed", rating_key=it.rating_key, kind="movie"
|
||||
),
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
hu = lang == "hu"
|
||||
|
||||
@@ -32,12 +32,12 @@ from app.plex.query import (
|
||||
meta_filter_conds,
|
||||
top_art_keys,
|
||||
)
|
||||
from app.discover.queries import RAIL_LIMIT
|
||||
from app.plex.tmdb import resolve_recs
|
||||
from app.routes.plex._shared import _norm_lang
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
RAIL_LIMIT = 20
|
||||
GENRE_TILES = 12
|
||||
SURPRISE_PER_KIND = 3 # Surprise-me: at most this many movies + this many shows (6 total)
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Plex people: cast/crew filter-chip cards + a single person's filmography page."""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
@@ -8,6 +7,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.auth import current_user
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import (
|
||||
@@ -22,7 +22,7 @@ from app.plex.cards import movie_card, show_card
|
||||
from app.plex.query import content_libs, split_csv
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_known_for(
|
||||
|
||||
@@ -17,7 +17,8 @@ from fastapi import (
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, resolved_user_id
|
||||
@@ -38,6 +39,8 @@ from app.plex import stream as plex_stream
|
||||
from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.routes.plex._shared import item_or_404
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -48,7 +51,7 @@ def _require_session(request: Request) -> int:
|
||||
low-sensitivity; a valid session is enough."""
|
||||
uid, _ = resolved_user_id(request)
|
||||
if not uid:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
return uid
|
||||
|
||||
|
||||
@@ -77,15 +80,17 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
|
||||
with SessionLocal() as db:
|
||||
key = imgcache.key_for(db, str(rating_key), v)
|
||||
if not key:
|
||||
raise HTTPException(status_code=404, detail="No image")
|
||||
raise not_found("image")
|
||||
plex = PlexClient(db) # reads base URL + token now; the fetch below needs no DB
|
||||
except PlexNotConfigured as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except PlexNotConfigured:
|
||||
raise refused(400, "plex_not_configured") from None
|
||||
try:
|
||||
with plex:
|
||||
data, ct = plex.image_bytes(key, w, h)
|
||||
except PlexError as e:
|
||||
raise HTTPException(status_code=502, detail=f"Plex image fetch failed: {e}")
|
||||
# silent-ok: a byte proxy for an <img>; the browser shows the placeholder and no reader sees the
|
||||
# detail — the 502 is the whole answer, as for the person-image proxy below.
|
||||
except PlexError:
|
||||
raise refused(502, "image_fetch_failed") from None
|
||||
imgcache.write(cache_key, data, ct)
|
||||
return imgcache.response(data, ct)
|
||||
|
||||
@@ -120,7 +125,7 @@ def subtitle_status(
|
||||
|
||||
`at` is the viewer's absolute position: the extractor windows around it, so what gets read first
|
||||
is the part actually about to be watched."""
|
||||
return subtitles.ensure(rating_key, sub_ord, at)
|
||||
return subtitles.ensure(rating_key, sub_ord, at, viewer=user.id)
|
||||
|
||||
|
||||
@router.get("/subtitle/{rating_key}/{sub_ord}")
|
||||
@@ -213,7 +218,7 @@ def person_image(u: str, request: Request) -> Response:
|
||||
Signed-session auth (no DB) so it, like /image, never holds a pool connection during the fetch."""
|
||||
_require_session(request)
|
||||
if not u.startswith(plex_people.PERSON_IMG_HOST):
|
||||
raise HTTPException(status_code=400, detail="Unsupported image host")
|
||||
raise refused(400, "image_host")
|
||||
cache_key = "person_" + hashlib.sha1(u.encode()).hexdigest()
|
||||
hit = imgcache.read(cache_key)
|
||||
if hit:
|
||||
@@ -223,7 +228,7 @@ def person_image(u: str, request: Request) -> Response:
|
||||
r = c.get(u)
|
||||
r.raise_for_status()
|
||||
except httpx.HTTPError:
|
||||
raise HTTPException(status_code=502, detail="Image fetch failed")
|
||||
raise refused(502, "image_fetch_failed")
|
||||
ct = r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(cache_key, r.content, ct)
|
||||
return imgcache.response(r.content, ct)
|
||||
@@ -295,6 +300,37 @@ def _push_watch(
|
||||
)
|
||||
|
||||
|
||||
def _mark_finished(db: Session, user_id: int, item_id: int, now: datetime) -> bool:
|
||||
"""Mark the item watched (position cleared) and say whether THIS call made it so. Two near-end
|
||||
checkpoints arrive together routinely (the 10 s tick and the debounced `seeked` flush), and a
|
||||
read-then-write let both see "not watched" and both scrobble. The transition is decided by the
|
||||
write itself: an insert that did not conflict, or an update conditional on the old status —
|
||||
under READ COMMITTED the second writer re-reads the committed row and matches nothing."""
|
||||
values = {
|
||||
"status": "watched",
|
||||
"watched_at": now,
|
||||
"position_seconds": 0,
|
||||
"progress_updated_at": now,
|
||||
"synced_to_plex": False,
|
||||
}
|
||||
inserted = db.execute(
|
||||
pg_insert(PlexState)
|
||||
.values(user_id=user_id, item_id=item_id, **values)
|
||||
.on_conflict_do_nothing(constraint="uq_plex_user_item")
|
||||
.returning(PlexState.id)
|
||||
).first()
|
||||
if inserted is not None:
|
||||
return True
|
||||
mine = (PlexState.user_id == user_id) & (PlexState.item_id == item_id)
|
||||
if db.execute(
|
||||
update(PlexState).where(mine, PlexState.status != "watched").values(values)
|
||||
).rowcount:
|
||||
return True
|
||||
# Already watched: a re-finish still restamps the row, as it always did — just no scrobble.
|
||||
db.execute(update(PlexState).where(mine).values(values))
|
||||
return False
|
||||
|
||||
|
||||
class ItemProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
@@ -330,11 +366,27 @@ def item_progress(
|
||||
# finish latch is up, or before the first position is known. Those are the states in which
|
||||
# reaping is the right answer anyway, and a viewer who scrubs back into a reaped session gets
|
||||
# the same 404 → reclaim the player already handles.
|
||||
plex_stream.touch_session(str(rating_key))
|
||||
plex_stream.touch_session(user.id, str(rating_key))
|
||||
position = max(0, int(body.position_seconds))
|
||||
duration = int(body.duration_seconds)
|
||||
final = body.final
|
||||
now = datetime.now(timezone.utc)
|
||||
near_end = duration > 0 and position > duration - _FINISH_MARGIN_S
|
||||
if near_end:
|
||||
flipped = _mark_finished(db, user.id, it.id, now)
|
||||
# The finish supersedes every resume push still in flight for this item: stamping it as the
|
||||
# latest push makes `_resume_push_superseded` drop them. Popping the entry instead (to keep
|
||||
# the map small) removed the very guard, and an older resume push could then land at Plex
|
||||
# AFTER the scrobble and revive a resume point on a watched item. Eviction bounds the map.
|
||||
with _push_throttle_lock:
|
||||
_last_push_at[(user.id, it.id)] = now
|
||||
db.commit()
|
||||
# Only the checkpoint that flipped the row scrobbles: a later checkpoint of the same finished
|
||||
# item — or a second one racing this one — would otherwise bump Plex's viewCount again.
|
||||
if flipped:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "watched")
|
||||
return {"ok": True, "status": "watched", "position_seconds": 0}
|
||||
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
|
||||
@@ -342,28 +394,6 @@ def item_progress(
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
was_watched = st is not None and st.status == "watched"
|
||||
|
||||
near_end = duration > 0 and position > duration - _FINISH_MARGIN_S
|
||||
if near_end:
|
||||
if st is None:
|
||||
st = PlexState(user_id=user.id, item_id=it.id)
|
||||
db.add(st)
|
||||
st.status = "watched"
|
||||
st.watched_at = now
|
||||
st.position_seconds = 0
|
||||
st.progress_updated_at = now
|
||||
st.synced_to_plex = False
|
||||
# Terminal state for resume write-back: drop the throttle/ordering entry so the map doesn't
|
||||
# retain a dead key for every finished item.
|
||||
with _push_throttle_lock:
|
||||
_last_push_at.pop((user.id, it.id), None)
|
||||
db.commit()
|
||||
# Immediate on the watched transition; skip a repeat scrobble if it was already watched
|
||||
# (a later checkpoint of the same finished item would otherwise bump Plex's viewCount again).
|
||||
if not was_watched:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "watched")
|
||||
return {"status": "watched", "position_seconds": 0}
|
||||
|
||||
if position < _PROGRESS_MIN_S:
|
||||
if st is not None and st.status == "new":
|
||||
@@ -372,7 +402,7 @@ def item_progress(
|
||||
st.position_seconds = 0
|
||||
st.progress_updated_at = now
|
||||
db.commit()
|
||||
return {"position_seconds": 0}
|
||||
return {"ok": True, "position_seconds": 0}
|
||||
|
||||
if st is None:
|
||||
st = PlexState(user_id=user.id, item_id=it.id)
|
||||
@@ -397,7 +427,7 @@ def item_progress(
|
||||
duration * 1000,
|
||||
intent=now,
|
||||
)
|
||||
return {"position_seconds": position}
|
||||
return {"ok": True, "position_seconds": position}
|
||||
|
||||
|
||||
class ItemStateIn(BaseModel):
|
||||
@@ -418,7 +448,7 @@ def item_state(
|
||||
it = item_or_404(db, rating_key)
|
||||
status = body.status
|
||||
if status not in ("new", "watched", "hidden"):
|
||||
raise HTTPException(status_code=400, detail="Invalid status")
|
||||
raise refused(400, "invalid_status")
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
|
||||
@@ -443,7 +473,7 @@ def item_state(
|
||||
db.commit()
|
||||
if mirrored_on_plex:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "unwatched")
|
||||
return {"status": "new"}
|
||||
return {"ok": True, "status": "new"}
|
||||
if st is None:
|
||||
st = PlexState(user_id=user.id, item_id=it.id)
|
||||
db.add(st)
|
||||
@@ -455,7 +485,7 @@ def item_state(
|
||||
db.commit()
|
||||
if status == "watched" and prev_status != "watched":
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "watched")
|
||||
return {"status": status}
|
||||
return {"ok": True, "status": status}
|
||||
|
||||
|
||||
def _bulk_state(
|
||||
@@ -497,7 +527,9 @@ def _bulk_state(
|
||||
db.delete(st)
|
||||
if prev != "new":
|
||||
to_push.append((e.id, e.rating_key, "unwatched"))
|
||||
db.commit()
|
||||
# Flush, not commit: the handler commits, and it does so before the response — which is when
|
||||
# the background push below runs, so the push always reads the committed marks.
|
||||
db.flush()
|
||||
if pushable and to_push:
|
||||
# One coalesced background task for the whole batch (reuses a single session + Plex client),
|
||||
# not one task per episode — a 200-ep show mark is 1 task, not 200.
|
||||
@@ -520,7 +552,7 @@ def show_state(
|
||||
"""Mark a whole show watched/unwatched (all its episodes) for the current user + push to Plex."""
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex show")
|
||||
raise not_found("show")
|
||||
watched = body.watched
|
||||
eps = (
|
||||
db.execute(select(PlexItem).where(PlexItem.show_id == sh.id, PlexItem.kind == "episode"))
|
||||
@@ -528,7 +560,8 @@ def show_state(
|
||||
.all()
|
||||
)
|
||||
changed = _bulk_state(background, db, user.id, eps, watched)
|
||||
return {"changed": changed, "watched": watched}
|
||||
db.commit()
|
||||
return {"ok": True, "changed": changed, "watched": watched}
|
||||
|
||||
|
||||
@router.post("/season/{rating_key}/state")
|
||||
@@ -542,7 +575,7 @@ def season_state(
|
||||
"""Mark a whole season watched/unwatched (all its episodes) for the current user + push to Plex."""
|
||||
se = by_rating_key(db, PlexSeason, rating_key)
|
||||
if se is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex season")
|
||||
raise not_found("season")
|
||||
watched = body.watched
|
||||
eps = (
|
||||
db.execute(select(PlexItem).where(PlexItem.season_id == se.id, PlexItem.kind == "episode"))
|
||||
@@ -550,7 +583,8 @@ def season_state(
|
||||
.all()
|
||||
)
|
||||
changed = _bulk_state(background, db, user.id, eps, watched)
|
||||
return {"changed": changed, "watched": watched}
|
||||
db.commit()
|
||||
return {"ok": True, "changed": changed, "watched": watched}
|
||||
|
||||
|
||||
@router.post("/stream/{rating_key}/session")
|
||||
@@ -575,21 +609,23 @@ def stream_session(
|
||||
# the client can switch audio without a restart — even for otherwise direct-playable files.
|
||||
if it.playable == "direct" and audio is None and not aoff and not multi:
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "direct",
|
||||
"url": f"/api/plex/stream/{it.rating_key}/file",
|
||||
"start_s": 0.0,
|
||||
}
|
||||
if plex_paths.local_media_path(db, it.file_path) is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
s = plex_stream.start_session(db, it, start, audio, aoff, multi, transcode=transcode)
|
||||
raise refused(404, "media_missing")
|
||||
s = plex_stream.start_session(db, it, user.id, start, audio, aoff, multi, transcode=transcode)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
raise refused(404, "media_missing")
|
||||
# Report the REAL media start (the keyframe ffmpeg landed on, which hls.js zero-bases to), not the
|
||||
# requested offset — the client's absolute clock and the subtitle cue-shift both key off this.
|
||||
# `transcode`/`hw`/`tonemap` let the player show an honest cold-start ("Preparing stream…") and
|
||||
# surface the active mode + encoder in the stats overlay (`tonemap` = the HW HDR→SDR path); both
|
||||
# segment streams are otherwise identical.
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "hls",
|
||||
"start_s": s.media_start_s,
|
||||
# The session id is part of the path (not a query param) so the playlist's RELATIVE segment
|
||||
@@ -614,7 +650,7 @@ def stream_file(
|
||||
it = item_or_404(db, rating_key)
|
||||
src = plex_paths.local_media_path(db, it.file_path)
|
||||
if src is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
raise refused(404, "media_missing")
|
||||
if download:
|
||||
# A clean, safe download name from the title + the real file extension.
|
||||
ext = src.suffix or (f".{it.container}" if it.container else "")
|
||||
@@ -651,9 +687,9 @@ def stream_hls_file(
|
||||
references its segments/subtitles by relative name, all resolving back here.
|
||||
|
||||
The session id in the path is what makes "back here" mean the same session. Sessions are keyed by
|
||||
rating_key alone and a seek-restart or a second viewer of the same item replaces the registered
|
||||
one, while the DIRECTORY is per (key, offset, tag) — so resolving by rating_key handed the old
|
||||
viewer the new session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong
|
||||
(viewer, rating_key) and a seek-restart replaces the viewer's registered one, while the DIRECTORY
|
||||
is per (key, viewer, offset, tag) — so resolving by the key alone handed the old player the new
|
||||
session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong
|
||||
bytes with a 200, which the player has no way to notice (deep-review round 5). A request for a
|
||||
session that is no longer the registered one is a 404, and the player reclaims at the playhead.
|
||||
|
||||
@@ -668,10 +704,10 @@ def stream_hls_file(
|
||||
or "\\" in filename
|
||||
or ".." in filename
|
||||
):
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
s = plex_stream.current_session(str(rating_key))
|
||||
raise not_found("file")
|
||||
s = plex_stream.current_session(user.id, str(rating_key))
|
||||
if s is None or s.id != session_id:
|
||||
raise HTTPException(status_code=404, detail="No active playback session")
|
||||
raise refused(404, "session_gone")
|
||||
path = s.dir / filename
|
||||
# Only a name THIS session can actually produce is worth waiting for: its own playlists, and a
|
||||
# segment of one of its own renditions whose predecessor is already on disk (ffmpeg writes them
|
||||
@@ -681,14 +717,14 @@ def stream_hls_file(
|
||||
# past the produced region — or a deliberate flood of hopeless names — from parking threadpool
|
||||
# workers and their DB connections (see `artifact_may_arrive`).
|
||||
if not plex_stream.artifact_may_arrive(s, filename):
|
||||
raise HTTPException(status_code=404, detail="Not available yet")
|
||||
raise refused(404, "not_ready")
|
||||
timeout = 30 if ext == ".m3u8" else 20
|
||||
# `proc` so a session whose ffmpeg has already exited answers at once instead of polling out the
|
||||
# whole timeout: this is a SYNC route on the ~40-slot threadpool, holding a pooled DB connection
|
||||
# with it, so every such wait is a worker (and a connection) parked for 20 s — an ordinary seek
|
||||
# past the produced region was enough to do it (round 5).
|
||||
if not plex_stream.wait_for(path, timeout, proc=s.proc):
|
||||
raise HTTPException(status_code=404, detail="Not available yet")
|
||||
raise refused(404, "not_ready")
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError:
|
||||
@@ -699,7 +735,7 @@ def stream_hls_file(
|
||||
# restarting at the playhead. Ordinary trigger: `loadSession` awaits the new session — which
|
||||
# kills and rmtrees the old one — before destroying the old hls instance, so the outgoing
|
||||
# instance's in-flight segment request races the rmtree on every seek-restart.
|
||||
raise HTTPException(status_code=404, detail="No active playback session") from None
|
||||
raise refused(404, "session_gone") from None
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=_HLS_CT[ext],
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -19,15 +19,12 @@ from app.models import (
|
||||
)
|
||||
from app.plex.cards import leaf_card
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
|
||||
def _own_playlist_or_404(db: Session, pid: int, user: User) -> PlexPlaylist:
|
||||
return owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
|
||||
|
||||
def _playlist_card(pl: PlexPlaylist, count: int, thumb_rk: str | None) -> dict:
|
||||
return {
|
||||
"id": pl.id,
|
||||
@@ -120,7 +117,7 @@ def create_playlist(
|
||||
) -> dict:
|
||||
title = body.title.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="title is required")
|
||||
raise refused(400, "title_required")
|
||||
pl = PlexPlaylist(user_id=user.id, title=title)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
@@ -140,7 +137,7 @@ def create_playlist(
|
||||
def playlist_detail(
|
||||
pid: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
rows = db.execute(_playlist_item_query(pid)).all()
|
||||
items = [it for (_pi, it) in rows]
|
||||
states = {
|
||||
@@ -183,10 +180,10 @@ def rename_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
title = body.title.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="title is required")
|
||||
raise refused(400, "title_required")
|
||||
pl.title = title
|
||||
db.commit()
|
||||
count = (
|
||||
@@ -203,7 +200,7 @@ def rename_playlist(
|
||||
def delete_playlist(
|
||||
pid: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
db.delete(pl)
|
||||
db.commit()
|
||||
return {"deleted": pid}
|
||||
@@ -220,10 +217,10 @@ def playlist_add_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
it = by_rating_key(db, PlexItem, body.item_rating_key)
|
||||
if it is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
exists = (
|
||||
db.execute(
|
||||
select(PlexPlaylistItem).where(
|
||||
@@ -252,7 +249,7 @@ def playlist_remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
it = by_rating_key(db, PlexItem, item_rating_key)
|
||||
if it is not None:
|
||||
db.execute(
|
||||
@@ -297,7 +294,7 @@ def playlist_add_bulk(
|
||||
) -> dict:
|
||||
"""Append many items at once (a whole season/show). Already-present items are skipped; the input
|
||||
order is preserved for the newly-added ones. Returns how many were added + the new total."""
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
items = _items_by_rks(db, body.item_rating_keys)
|
||||
existing = set(
|
||||
db.execute(
|
||||
@@ -328,7 +325,7 @@ def playlist_add_bulk(
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {"added": added, "item_count": total}
|
||||
return {"ok": True, "added": added, "item_count": total}
|
||||
|
||||
|
||||
@router.post("/playlists/{pid}/items/remove-bulk")
|
||||
@@ -340,7 +337,7 @@ def playlist_remove_bulk(
|
||||
) -> dict:
|
||||
"""Remove many items at once (a whole season/show, or a multi-select). Returns how many were
|
||||
removed + the new total."""
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
ids = [it.id for it in _items_by_rks(db, body.item_rating_keys)]
|
||||
removed = 0
|
||||
if ids:
|
||||
@@ -359,7 +356,7 @@ def playlist_remove_bulk(
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {"removed": removed, "item_count": total}
|
||||
return {"ok": True, "removed": removed, "item_count": total}
|
||||
|
||||
|
||||
@router.put("/playlists/{pid}/order")
|
||||
@@ -370,7 +367,7 @@ def reorder_playlist(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Set the playlist order from an explicit list of item rating_keys (0-based positions)."""
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
order = body.item_rating_keys
|
||||
id_by_rk = {
|
||||
it.rating_key: it.id
|
||||
|
||||
@@ -6,7 +6,7 @@ the app or any account data. Password-protected links exchange the password once
|
||||
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
@@ -17,19 +17,29 @@ from app.db import get_db
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import verify_password
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/public", tags=["public"])
|
||||
|
||||
# Slow password guessing per link (the token itself is 192-bit, so it needs no throttle).
|
||||
# Slow password guessing on a link (the token itself is 192-bit, so it needs no throttle). Counted per
|
||||
# viewer — the link AND the client (`client_key`) — so a client guessing wrong shuts out only itself, and a
|
||||
# busy link's own unlocks don't count against its next viewer.
|
||||
_unlock_limit = RateLimiter(max_events=10, window_seconds=300)
|
||||
# ...under a ceiling on WRONG passwords for the link from every client together, so guessing spread over
|
||||
# many addresses stays slow too; unlocking with the password never adds to it. The price of the ceiling:
|
||||
# someone with the link but not its password, guessing from 50 / 10 = 5 networks, can keep the link shut
|
||||
# for everyone until the window passes. It is checked before the password and counted after it, so guesses
|
||||
# already running when it fills are still checked — as many as the route's threadpool runs at once.
|
||||
_unlock_failures = RateLimiter(max_events=50, window_seconds=300)
|
||||
|
||||
|
||||
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
||||
if link is None or linksmod.is_expired(link):
|
||||
raise HTTPException(status_code=404, detail="This link is no longer available.")
|
||||
raise refused(404, "link_gone")
|
||||
return link
|
||||
|
||||
|
||||
@@ -37,12 +47,12 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||
if job is None or asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
raise refused(404, "video_gone")
|
||||
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
|
||||
# loads a player for a missing file — meta and file agree.
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
raise refused(404, "video_gone")
|
||||
return job, asset
|
||||
|
||||
|
||||
@@ -83,25 +93,33 @@ class WatchUnlockIn(BaseModel):
|
||||
|
||||
|
||||
@router.post("/watch/{token}/unlock")
|
||||
def watch_unlock(token: str, body: WatchUnlockIn, db: Session = Depends(get_db)) -> dict:
|
||||
def watch_unlock(
|
||||
token: str, body: WatchUnlockIn, request: Request, db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Exchange the link password for a signed grant carried by the media URL."""
|
||||
if not _unlock_limit.allow(token):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again later.")
|
||||
# The link first: a token naming none must leave no limiter entry behind, or posting made-up tokens
|
||||
# would grow the limiter without bound.
|
||||
link = _link_or_404(db, token)
|
||||
viewer = f"{token}|{client_key(request)}"
|
||||
if _unlock_failures.exhausted(token) or not _unlock_limit.allow(viewer):
|
||||
raise refused(429, "too_many_attempts")
|
||||
if not link.password_hash:
|
||||
# Not a password link — nothing to unlock; just hand back the plain meta.
|
||||
job, asset = _ready_target(db, link)
|
||||
return {
|
||||
"ok": True,
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
||||
}
|
||||
if not verify_password(body.password, link.password_hash):
|
||||
raise HTTPException(status_code=403, detail="Wrong password.")
|
||||
_unlock_failures.allow(token)
|
||||
raise refused(403, "wrong_share_password")
|
||||
job, asset = _ready_target(db, link)
|
||||
link.view_count += 1
|
||||
db.commit()
|
||||
grant = linksmod.make_grant(token, link.password_version)
|
||||
return {
|
||||
"ok": True,
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(
|
||||
link, job, asset, _file_url(token, grant), _poster_url(token, asset, grant)
|
||||
@@ -115,11 +133,11 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||
`allow_download` link serves it as an attachment. Password links require a valid grant."""
|
||||
link = _link_or_404(db, token)
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise HTTPException(status_code=403, detail="This link needs a password.")
|
||||
raise refused(403, "share_password_required")
|
||||
asset = _ready_asset(db, link)
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
raise refused(404, "video_gone")
|
||||
|
||||
filename = storage.download_filename(asset.title or "video", asset.container, path)
|
||||
# The served Content-Type comes from the asset's ffprobe-verified media_kind + a container
|
||||
@@ -149,11 +167,11 @@ def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)
|
||||
and the link-preview image). Password links require a valid grant, like the file."""
|
||||
link = _link_or_404(db, token)
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise HTTPException(status_code=403, detail="This link needs a password.")
|
||||
raise refused(403, "share_password_required")
|
||||
job, asset = _ready_target(db, link)
|
||||
if not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -8,6 +8,7 @@ from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.models import SavedView, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.refusal import refused
|
||||
|
||||
# Saved "smart views": per-user named snapshots of a module's filters (feed or plex, keyed by
|
||||
# `source`). Gated behind require_human so the shared demo account can't pollute a communal list
|
||||
@@ -66,7 +67,7 @@ def create_view(
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name is required")
|
||||
raise refused(400, "name_required")
|
||||
source = _norm_source(body.source)
|
||||
# Position is per (user, source), so each module's views number 0..N independently.
|
||||
maxpos = db.execute(
|
||||
@@ -86,14 +87,10 @@ def create_view(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a view named “{name}”.")
|
||||
raise refused(409, "view_name_taken", name=name)
|
||||
return _serialize(view)
|
||||
|
||||
|
||||
def _own_view(db: Session, user: User, view_id: int) -> SavedView:
|
||||
return owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
|
||||
|
||||
class UpdateViewIn(BaseModel):
|
||||
# PATCH: presence decides what's touched (read via model_fields_set). filters, when sent,
|
||||
# must be a real object — an explicit null is rejected (a view can't have null filters).
|
||||
@@ -109,16 +106,16 @@ def update_view(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = _own_view(db, user, view_id)
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name cannot be empty")
|
||||
raise refused(400, "name_required")
|
||||
view.name = name[:80]
|
||||
if "filters" in sent:
|
||||
if body.filters is None:
|
||||
raise HTTPException(status_code=400, detail="filters must be an object")
|
||||
raise refused(400, "filters_not_object")
|
||||
view.filters = body.filters
|
||||
if "is_default" in sent:
|
||||
make_default = bool(body.is_default)
|
||||
@@ -138,7 +135,7 @@ def update_view(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a view named “{view.name}”.")
|
||||
raise refused(409, "view_name_taken", name=view.name)
|
||||
return _serialize(view)
|
||||
|
||||
|
||||
@@ -148,7 +145,7 @@ def delete_view(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = _own_view(db, user, view_id)
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
db.delete(view)
|
||||
db.commit()
|
||||
return {"deleted": view_id}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
per-job run activity (from the in-process scheduler), the work still queued (DB-derived),
|
||||
and the shared quota picture."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -22,8 +22,11 @@ from app.scheduler import (
|
||||
trigger_all,
|
||||
trigger_job,
|
||||
)
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
from app.routes._common import not_found
|
||||
from app.refusal import refused
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"])
|
||||
|
||||
@@ -45,10 +48,7 @@ def set_job_interval(
|
||||
raise not_found("job")
|
||||
minutes = body.interval_minutes
|
||||
if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"interval must be between {MIN_INTERVAL} and {MAX_INTERVAL} minutes",
|
||||
)
|
||||
raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL)
|
||||
applied = apply_interval(db, job_id, minutes)
|
||||
audit.record(
|
||||
db,
|
||||
@@ -60,7 +60,7 @@ def set_job_interval(
|
||||
after={"interval_minutes": applied},
|
||||
)
|
||||
db.commit()
|
||||
return {"id": job_id, "interval_minutes": applied}
|
||||
return {"ok": True, "id": job_id, "interval_minutes": applied}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/run")
|
||||
@@ -76,14 +76,14 @@ def run_job_now(
|
||||
except KeyError:
|
||||
raise not_found("job")
|
||||
if result == "already_running":
|
||||
raise HTTPException(status_code=409, detail="Job is already running")
|
||||
return {"id": job_id, "started": True}
|
||||
raise refused(409, "job_running")
|
||||
return {"ok": True, "id": job_id, "started": True}
|
||||
|
||||
|
||||
@router.post("/run-all")
|
||||
def run_all_now(user: User = Depends(admin_user)) -> dict:
|
||||
"""Trigger every job that isn't already running. Returns the ids actually started."""
|
||||
return {"started": trigger_all(actor_id=user.id)}
|
||||
return {"ok": True, "started": trigger_all(actor_id=user.id)}
|
||||
|
||||
|
||||
class MaintenanceSettingsIn(BaseModel):
|
||||
@@ -101,8 +101,8 @@ def set_maintenance_settings(
|
||||
# set_value validates against the spec's min/max (one source of bounds), mirroring the config PATCH.
|
||||
try:
|
||||
sysconfig.set_value(db, "maintenance_revalidate_batch", body.revalidate_batch)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
new = sysconfig.get_int(db, "maintenance_revalidate_batch")
|
||||
audit.record(
|
||||
db,
|
||||
@@ -112,7 +112,7 @@ def set_maintenance_settings(
|
||||
after={"revalidate_batch": new},
|
||||
)
|
||||
db.commit()
|
||||
return {"revalidate_batch": new}
|
||||
return {"ok": True, "revalidate_batch": new}
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -137,9 +137,17 @@ def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
|
||||
queue = {
|
||||
"channels_recent_pending": db.scalar(
|
||||
select(func.count()).select_from(Channel).where(Channel.recent_synced_at.is_(None))
|
||||
# The same selection run_recent_backfill works through, or the card counts channels
|
||||
# the job will never touch and never reaches zero.
|
||||
select(func.count())
|
||||
.select_from(Channel)
|
||||
.where(Channel.recent_synced_at.is_(None), followed_channel())
|
||||
)
|
||||
or 0,
|
||||
# The whole opted-in BACKLOG, deliberately not `run_deep_backfill`'s next-pass selection
|
||||
# (which also wants `recent_synced_at`): the same figure /api/sync/my-status shows the user,
|
||||
# and a channel missing its recent pull is one the same job tick fetches first, not one the
|
||||
# job will never touch. Narrowing it here would under-report work that is genuinely queued.
|
||||
"channels_deep_pending": deep_eta["channels_pending"],
|
||||
"deep_eta_seconds": deep_eta["eta_seconds"],
|
||||
"videos_pending_enrich": db.scalar(
|
||||
|
||||
@@ -13,17 +13,16 @@ this search-discovered "noise" by default. Shorts and currently-live/upcoming vi
|
||||
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
|
||||
dropped from the search response up front)."""
|
||||
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, delete, func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app import quota, sysconfig
|
||||
from app import logs, quota, sysconfig
|
||||
from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -39,14 +38,16 @@ from app.models import (
|
||||
VideoState,
|
||||
)
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.utils import like_escape
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.videos import insert_stubs, apply_video_details, parse_dt
|
||||
from app.youtube import search_scrape
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
from app.refusal import refused
|
||||
from app.routes._common import youtube_refusal
|
||||
|
||||
log = logging.getLogger("siftlode.search")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
|
||||
# search.list costs this many quota units per page.
|
||||
@@ -132,6 +133,13 @@ def _ingest_candidates(
|
||||
apply_channel_details(db, details)
|
||||
db.commit()
|
||||
except YouTubeError:
|
||||
# The stubs remain and the details can be backfilled later — but until they are, those
|
||||
# channels sit in the results with no thumbnail and no subscriber count, looking like
|
||||
# channels that have neither.
|
||||
log.warning(
|
||||
"the channels found by a search could not be enriched",
|
||||
extra=logs.event("search.channel_enrich_failed", channels=len(new_channel_ids)),
|
||||
)
|
||||
db.rollback() # details can be backfilled later; the stubs remain
|
||||
|
||||
# 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO
|
||||
@@ -161,6 +169,12 @@ def _ingest_candidates(
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH):
|
||||
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
|
||||
except YouTubeError:
|
||||
# Every candidate then stays unenriched, and an unenriched video cannot be judged live or
|
||||
# Short — so it is dropped. A working search returns nothing and reads as "no results".
|
||||
log.warning(
|
||||
"the videos found by a search could not be enriched",
|
||||
extra=logs.event("search.video_enrich_failed", candidates=len(ids)),
|
||||
)
|
||||
detail_by_id = {}
|
||||
enriched_at = datetime.now(
|
||||
timezone.utc
|
||||
@@ -213,6 +227,25 @@ def _new_first(db: Session, user: User, ids: list[str]) -> list[str]:
|
||||
return [i for i in ids if i not in have] + [i for i in ids if i in have]
|
||||
|
||||
|
||||
def _fold_term(db: Session, video_ids: list[str], term: str) -> None:
|
||||
"""Append `term` to each video's space-joined `search_terms`, unless it is already one of them.
|
||||
|
||||
"Already one of them" is a whole-term match, as a literal: a substring test skipped `cat` on a
|
||||
video already found for `category`, and an unescaped one read the `_` in `my_channel` as a
|
||||
wildcard — either way the term was never folded in, and local search lost that relevance."""
|
||||
padded = func.concat(" ", func.coalesce(Video.search_terms, ""), " ")
|
||||
db.execute(
|
||||
update(Video)
|
||||
.where(Video.id.in_(video_ids))
|
||||
.where(padded.notilike(f"% {like_escape(term)} %"))
|
||||
.values(
|
||||
search_terms=func.trim(
|
||||
func.coalesce(Video.search_terms, "").op("||")(" ").op("||")(term)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/youtube")
|
||||
def search_youtube(
|
||||
q: str | None = None,
|
||||
@@ -227,29 +260,31 @@ def search_youtube(
|
||||
page, so it returns a single page (≤50) regardless. `cursor` lets the caller fetch further."""
|
||||
term = (q or "").strip()
|
||||
if not term:
|
||||
raise HTTPException(status_code=400, detail="A search term is required.")
|
||||
raise refused(400, "search_term_required")
|
||||
limit = max(1, min(limit, 100))
|
||||
|
||||
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
||||
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
|
||||
if source != "scrape":
|
||||
source = "api"
|
||||
source = sysconfig.get_str(db, "search_source")
|
||||
|
||||
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
||||
# since the search itself spends no quota.
|
||||
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
||||
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
if used >= daily_limit:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
|
||||
# The 429 is in the request line either way; which of the two caps produced it is not, and
|
||||
# they are answered by different things — this one by tomorrow, the other by the admin.
|
||||
log.info(
|
||||
"a live search was refused: the per-user daily cap is spent",
|
||||
extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit),
|
||||
)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
|
||||
log.info(
|
||||
"a live search was refused: the shared YouTube budget is spent",
|
||||
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
||||
)
|
||||
raise refused(429, "quota_spent")
|
||||
|
||||
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
|
||||
blocked = set(
|
||||
@@ -274,9 +309,14 @@ def search_youtube(
|
||||
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
|
||||
# which hides the real message and burns a second quota-costed search. Upstream failure
|
||||
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube search failed: {exc}"
|
||||
) from exc
|
||||
# A Data API failure is logged where it happened; the scraper's is not, and its text
|
||||
# no longer rides the answer — so it is said here.
|
||||
if isinstance(exc, search_scrape.ScrapeError):
|
||||
log.warning(
|
||||
"the YouTube search scrape failed",
|
||||
extra=logs.event("search.scrape_failed", error=str(exc)),
|
||||
)
|
||||
raise youtube_refusal(exc) from exc
|
||||
|
||||
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
||||
candidates = [
|
||||
@@ -293,12 +333,32 @@ def search_youtube(
|
||||
collected.extend(_ingest_candidates(db, yt, user, candidates))
|
||||
except Exception as exc: # noqa: BLE001 — any ingest failure, not just YouTubeError
|
||||
db.rollback()
|
||||
log.exception("live search ingest failed for %r", term)
|
||||
# The term is NOT in this line. A search string is exactly what the privacy
|
||||
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
||||
# themselves, and `redact_text` does not cover it — it scrubs addresses and
|
||||
# credential params, not what someone typed into the search box. What the
|
||||
# ERROR line carries instead is the shape of the failure, which is what tells
|
||||
# an ingest bug from an upstream one; the term follows at TRACE, and only
|
||||
# under this user's own consent — not whenever the process runs at TRACE.
|
||||
log.exception(
|
||||
"live search ingest failed",
|
||||
extra=logs.event(
|
||||
"search.live.ingest_failed",
|
||||
source=source,
|
||||
candidates=len(candidates),
|
||||
collected=len(collected),
|
||||
),
|
||||
)
|
||||
if logs.consented_trace():
|
||||
log.log(
|
||||
logs.TRACE,
|
||||
"the term the ingest failed on",
|
||||
# log-pii-ok: only under this user's own Teljes consent
|
||||
extra=logs.event("search.live.term", term=term),
|
||||
)
|
||||
if collected:
|
||||
break # keep what earlier pages already gathered
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube search failed: {exc}"
|
||||
) from exc
|
||||
raise youtube_refusal(exc) from exc
|
||||
next_cursor = page["next_page_token"]
|
||||
# The API source pages cost 100 units each — never auto-page; one page per request.
|
||||
if source == "api" or len(collected) >= limit or not next_cursor:
|
||||
@@ -323,16 +383,7 @@ def search_youtube(
|
||||
.values([{"user_id": user.id, "video_id": vid} for vid in ordered])
|
||||
.on_conflict_do_nothing(index_elements=["user_id", "video_id"])
|
||||
)
|
||||
db.execute(
|
||||
update(Video)
|
||||
.where(Video.id.in_(ordered))
|
||||
.where(func.coalesce(Video.search_terms, "").notilike(f"%{term}%"))
|
||||
.values(
|
||||
search_terms=func.trim(
|
||||
func.coalesce(Video.search_terms, "").op("||")(" ").op("||")(term)
|
||||
)
|
||||
)
|
||||
)
|
||||
_fold_term(db, ordered, term)
|
||||
db.commit()
|
||||
|
||||
ordered = _new_first(db, user, ordered)
|
||||
@@ -360,7 +411,7 @@ def clear_search(
|
||||
find. Kept videos and organically-owned ones are left untouched."""
|
||||
ids = body.video_ids
|
||||
if not ids:
|
||||
return {"removed": 0}
|
||||
return {"ok": True, "removed": 0}
|
||||
db.execute(
|
||||
delete(SearchFind).where(SearchFind.user_id == user.id, SearchFind.video_id.in_(ids))
|
||||
)
|
||||
@@ -380,4 +431,4 @@ def clear_search(
|
||||
)
|
||||
).rowcount
|
||||
db.commit()
|
||||
return {"removed": removed}
|
||||
return {"ok": True, "removed": removed}
|
||||
|
||||
+14
-14
@@ -8,7 +8,7 @@ X-Setup-Token header).
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from fastapi import APIRouter, Depends, Header
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -21,6 +21,8 @@ from app.auth import PASSWORD_MIN_LEN
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.security import hash_password
|
||||
from app.refusal import refused
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/setup", tags=["setup"])
|
||||
|
||||
@@ -31,9 +33,9 @@ def require_setup(
|
||||
) -> Session:
|
||||
"""Gate for setup steps. Returns the db session for the handler to reuse."""
|
||||
if state.is_configured(db):
|
||||
raise HTTPException(status_code=409, detail="This instance is already set up.")
|
||||
raise refused(409, "already_set_up")
|
||||
if not state.check_setup_token(db, x_setup_token):
|
||||
raise HTTPException(status_code=403, detail="Invalid or missing setup token.")
|
||||
raise refused(403, "setup_token")
|
||||
return db
|
||||
|
||||
|
||||
@@ -66,11 +68,9 @@ def setup_admin(body: SetupAdminIn, db: Session = Depends(require_setup)) -> dic
|
||||
email = body.email.strip().lower()
|
||||
password = body.password
|
||||
if "@" not in email:
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if len(password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Password must be at least {PASSWORD_MIN_LEN} characters."
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
# Resolve canonically, not by the literal string (DX-17). The wizard can be re-run, and E9.5's
|
||||
# whole point is that two spellings of one Gmail mailbox are one person — matching literally here
|
||||
# would let a second run under the other spelling create the DUPLICATE admin row the epic exists
|
||||
@@ -97,16 +97,16 @@ def setup_config(settings_map: dict[str, Any], db: Session = Depends(require_set
|
||||
saved: list[str] = []
|
||||
for key, value in settings_map.items():
|
||||
if sysconfig.spec(key) is None:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown setting: {key}")
|
||||
raise refused(400, "unknown_setting", key=key)
|
||||
if value is None or value == "":
|
||||
continue
|
||||
try:
|
||||
sysconfig.set_value(db, key, value)
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
saved.append(key)
|
||||
db.commit() # sysconfig.set_value only flushes now — persist the wizard's config in one commit
|
||||
return {"saved": saved}
|
||||
return {"ok": True, "saved": saved}
|
||||
|
||||
|
||||
class SetupTestEmailIn(BaseModel):
|
||||
@@ -121,9 +121,9 @@ def setup_test_email(body: SetupTestEmailIn, db: Session = Depends(require_setup
|
||||
"""Send a test email to confirm the SMTP settings the wizard just saved."""
|
||||
to = body.to.strip()
|
||||
if "@" not in to:
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if not email_mod.send_test(to, mailtext.normalize(body.lang)):
|
||||
raise HTTPException(status_code=400, detail="Couldn't send — check the SMTP settings.")
|
||||
raise refused(400, "smtp_failed")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@@ -133,6 +133,6 @@ def setup_finish(db: Session = Depends(require_setup)) -> dict:
|
||||
invalidates the setup token and disables the wizard)."""
|
||||
has_admin = db.execute(select(User).where(User.role == "admin")).first() is not None
|
||||
if not has_admin:
|
||||
raise HTTPException(status_code=400, detail="Create the admin account first.")
|
||||
raise refused(400, "admin_first")
|
||||
state.mark_configured(db)
|
||||
return {"ok": True}
|
||||
|
||||
+16
-10
@@ -7,10 +7,13 @@ from app.audit import AuditAction
|
||||
from app.auth import admin_user, current_user, has_read_scope, require_human
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.routes._common import youtube_refusal
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.scheduler import running_job_ids
|
||||
from app.youtube.client import YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/sync", tags=["sync"])
|
||||
|
||||
@@ -26,17 +29,20 @@ _sync_limiter = RateLimiter(max_events=3, window_seconds=600)
|
||||
def sync_subscriptions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
if not has_read_scope(user):
|
||||
# No YouTube read grant yet — the onboarding wizard hasn't been completed.
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Connect your YouTube account (read access) to import subscriptions.",
|
||||
)
|
||||
raise refused(403, "youtube_read_scope")
|
||||
if not _sync_limiter.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail={"code": "sync_cooldown"})
|
||||
with quota.measured(db, user.id, quota.QuotaAction.SUBSCRIPTIONS_PULL) as m:
|
||||
result = import_subscriptions(db, user)
|
||||
try:
|
||||
with quota.measured(db, user.id, quota.QuotaAction.SUBSCRIPTIONS_PULL) as m:
|
||||
result = import_subscriptions(db, user)
|
||||
except YouTubeError as exc:
|
||||
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
||||
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
result["quota_used_estimate"] = m.used
|
||||
result["quota_remaining_today"] = m.remaining
|
||||
return result
|
||||
return {"ok": True, **result}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
@@ -137,7 +143,7 @@ def deep_all(
|
||||
update(Subscription).where(Subscription.user_id == user.id).values(deep_requested=on)
|
||||
)
|
||||
db.commit()
|
||||
return {"updated": result.rowcount, "deep_requested": on}
|
||||
return {"ok": True, "updated": result.rowcount, "deep_requested": on}
|
||||
|
||||
|
||||
@router.post("/pause")
|
||||
@@ -145,7 +151,7 @@ def pause_sync(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
state.set_sync_paused(db, True)
|
||||
audit.record(db, admin.id, AuditAction.SYNC_PAUSE, summary="Background sync paused")
|
||||
db.commit()
|
||||
return {"paused": True}
|
||||
return {"ok": True, "paused": True}
|
||||
|
||||
|
||||
@router.post("/resume")
|
||||
@@ -153,4 +159,4 @@ def resume_sync(admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
state.set_sync_paused(db, False)
|
||||
audit.record(db, admin.id, AuditAction.SYNC_RESUME, summary="Background sync resumed")
|
||||
db.commit()
|
||||
return {"paused": False}
|
||||
return {"ok": True, "paused": False}
|
||||
|
||||
+24
-14
@@ -1,4 +1,4 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -9,6 +9,7 @@ from app.db import get_db
|
||||
from app.models import ChannelTag, Tag, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.sync.tagsuggest import compute_tag_suggestions
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/tags", tags=["tags"])
|
||||
|
||||
@@ -60,7 +61,7 @@ def create_tag(
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name is required")
|
||||
raise refused(400, "name_required")
|
||||
# An unknown category silently falls back to "other" (unchanged behaviour) rather than 422.
|
||||
category = body.category if body.category in ("language", "topic", "other") else "other"
|
||||
tag = Tag(
|
||||
@@ -74,13 +75,14 @@ def create_tag(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a tag named “{name}”.")
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "category": tag.category}
|
||||
|
||||
|
||||
def _own_tag(db: Session, user: User, tag_id: int) -> Tag:
|
||||
# System tags (user_id NULL) are not user-editable → owned_or_404 404s them too.
|
||||
return owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
raise refused(409, "tag_name_taken", name=name)
|
||||
return {
|
||||
"ok": True,
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"category": tag.category,
|
||||
}
|
||||
|
||||
|
||||
class TagUpdateIn(BaseModel):
|
||||
@@ -97,12 +99,13 @@ def update_tag(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
tag = _own_tag(db, user, tag_id)
|
||||
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
|
||||
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name cannot be empty")
|
||||
raise refused(400, "name_required")
|
||||
tag.name = name
|
||||
if "color" in sent:
|
||||
tag.color = body.color
|
||||
@@ -110,8 +113,14 @@ def update_tag(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a tag named “{tag.name}”.")
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "category": tag.category}
|
||||
raise refused(409, "tag_name_taken", name=tag.name)
|
||||
return {
|
||||
"ok": True,
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"category": tag.category,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{tag_id}")
|
||||
@@ -120,7 +129,8 @@ def delete_tag(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
tag = _own_tag(db, user, tag_id)
|
||||
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
|
||||
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
db.delete(tag) # ChannelTag rows cascade on tag delete.
|
||||
db.commit()
|
||||
return {"deleted": tag_id}
|
||||
|
||||
+29
-12
@@ -2,7 +2,6 @@
|
||||
recent-first / deep backfill. Runs inside the API process (single worker)."""
|
||||
|
||||
import contextvars
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable
|
||||
@@ -10,7 +9,7 @@ from typing import Callable
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import audit, progress, quota
|
||||
from app import audit, debugsession, logs, logsink, progress, quota
|
||||
from app.audit import AuditAction
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
@@ -35,8 +34,7 @@ from app.sync.runner import (
|
||||
run_subscription_resync,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("siftlode.scheduler")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_scheduler: BackgroundScheduler | None = None
|
||||
|
||||
# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the
|
||||
@@ -72,6 +70,7 @@ JOB_INTERVALS: dict[str, int] = {
|
||||
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
||||
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
||||
"audit_gc": settings.audit_gc_minutes,
|
||||
"log_gc": settings.log_gc_minutes,
|
||||
}
|
||||
|
||||
# Sane bounds for an admin-set interval (minutes).
|
||||
@@ -103,7 +102,7 @@ def apply_interval(db, job_id: str, minutes: int) -> int:
|
||||
db.commit()
|
||||
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
|
||||
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
|
||||
logger.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||
log.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||
return minutes
|
||||
|
||||
|
||||
@@ -130,7 +129,7 @@ def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | Non
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("failed to write completion notification for job %s", job_id)
|
||||
log.exception("failed to write completion notification for job %s", job_id)
|
||||
|
||||
|
||||
def _run_changed(result) -> bool:
|
||||
@@ -166,7 +165,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("failed to write audit row for job %s", job_id)
|
||||
log.exception("failed to write audit row for job %s", job_id)
|
||||
|
||||
|
||||
def _job(name: str, fn) -> None:
|
||||
@@ -174,7 +173,7 @@ def _job(name: str, fn) -> None:
|
||||
# HERE, atomically: whichever reaches this first wins, the other skips. This is the real
|
||||
# single-run guard — trigger_job's pre-check is only a fast best-effort answer for the UI.
|
||||
if not _claim_running(name):
|
||||
logger.info("job %s already running — skipping this trigger", name)
|
||||
log.info("job %s already running — skipping this trigger", name)
|
||||
return
|
||||
db = SessionLocal()
|
||||
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
|
||||
@@ -187,7 +186,7 @@ def _job(name: str, fn) -> None:
|
||||
|
||||
try:
|
||||
if is_sync_paused(db):
|
||||
logger.info("job %s skipped (sync paused)", name)
|
||||
log.info("job %s skipped (sync paused)", name)
|
||||
_record(
|
||||
name,
|
||||
status="skipped",
|
||||
@@ -202,14 +201,14 @@ def _job(name: str, fn) -> None:
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
summary = _summarize(result)
|
||||
logger.info("job %s -> %s", name, result)
|
||||
log.info("job %s -> %s", name, result)
|
||||
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "ok", summary)
|
||||
_audit_run(db, actor_id, name, "ok", summary, result)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception("job %s failed", name)
|
||||
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
|
||||
if actor_id is not None:
|
||||
@@ -381,6 +380,23 @@ def _audit_gc_job() -> None:
|
||||
_job("audit_gc", work)
|
||||
|
||||
|
||||
def _log_gc_job() -> None:
|
||||
# Prune log_events rows older than log_retention_days — the limit the privacy policy states, so
|
||||
# this runs hourly rather than daily. Pure DB, no quota.
|
||||
def work(db):
|
||||
from app import sysconfig
|
||||
|
||||
days = sysconfig.get_int(db, "log_retention_days")
|
||||
reaped = logsink.gc(db, days)
|
||||
# Debug sessions (E16) go with the rows they bundle; an open one past its cap is closed.
|
||||
closed = debugsession.sweep(db)
|
||||
sessions = debugsession.gc(db, days)
|
||||
db.commit()
|
||||
return {"reaped": reaped, "sessions_closed": closed, "sessions_reaped": sessions}
|
||||
|
||||
_job("log_gc", work)
|
||||
|
||||
|
||||
# job_id -> wrapper. The single source of truth for which jobs exist and how to run one,
|
||||
# shared by start_scheduler (recurring registration) and trigger_job (manual "run now").
|
||||
JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
@@ -400,6 +416,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
"plex_watch_reconcile": _plex_watch_reconcile_job,
|
||||
"tmdb_enrich": _tmdb_enrich_job,
|
||||
"audit_gc": _audit_gc_job,
|
||||
"log_gc": _log_gc_job,
|
||||
}
|
||||
|
||||
|
||||
@@ -478,7 +495,7 @@ def start_scheduler() -> None:
|
||||
scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id)
|
||||
scheduler.start()
|
||||
_scheduler = scheduler
|
||||
logger.info("scheduler started")
|
||||
log.info("scheduler started")
|
||||
|
||||
|
||||
def shutdown_scheduler() -> None:
|
||||
|
||||
+23
-3
@@ -1,10 +1,14 @@
|
||||
import hashlib
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# argon2id with the library defaults (sensible memory/time cost). One shared hasher instance.
|
||||
_ph = PasswordHasher()
|
||||
|
||||
@@ -24,8 +28,16 @@ def verify_password(password: str, hashed: str | None) -> bool:
|
||||
return False
|
||||
try:
|
||||
return _ph.verify(hashed, password)
|
||||
except Exception:
|
||||
# Any failure (mismatch, malformed hash) is a non-match — never raise to the caller.
|
||||
except VerifyMismatchError:
|
||||
return False # silent-ok: a wrong password. The commonest path through this function.
|
||||
except Exception as exc: # noqa: BLE001 — never raise to the caller, whatever went wrong
|
||||
# Everything else is a hash this process cannot USE — truncated in the column, written by
|
||||
# a different argon2 build. It is answered as "wrong password", so the owner of that
|
||||
# account is locked out by a bug and told they mistyped.
|
||||
log.warning(
|
||||
"a stored password hash could not be verified",
|
||||
extra=logs.event("auth.password.hash_unusable", reason=type(exc).__name__),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -60,5 +72,13 @@ def decrypt_optional(value: str | None) -> str | None:
|
||||
return None
|
||||
try:
|
||||
return _require_fernet().decrypt(value.encode()).decode()
|
||||
except Exception:
|
||||
except Exception as exc: # noqa: BLE001 — callers fall back to a refresh, never to an error
|
||||
# Info rather than warning because the docstring's legacy-plaintext case comes through
|
||||
# here too and is harmless. What is NOT harmless looks identical: a rotated
|
||||
# TOKEN_ENCRYPTION_KEY turns every stored token unreadable at once, and every account
|
||||
# then reads as "not connected" with nothing saying the key is the reason.
|
||||
log.info(
|
||||
"a stored ciphertext could not be decrypted",
|
||||
extra=logs.event("security.decrypt.failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
"""Global admin-controlled app state (e.g. pausing background sync, first-run setup)."""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import AppState
|
||||
from app.security import hash_token
|
||||
|
||||
log = logging.getLogger("siftlode.state")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
||||
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
||||
_configured_cache = False
|
||||
|
||||
@@ -6,17 +6,16 @@ topicDetails categories and the channel's dominant video category. System tags a
|
||||
regenerated freely; user tags are never touched here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from sqlalchemy import and_, exists, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
log = logging.getLogger("siftlode.autotag")
|
||||
|
||||
from app import progress, sysconfig
|
||||
from app import logs, progress, sysconfig
|
||||
from app.models import Channel, ChannelTag, Tag, Video
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# --- language id (offline, deterministic, small) ---
|
||||
try:
|
||||
from py3langid.langid import MODEL_FILE, LanguageIdentifier
|
||||
@@ -29,7 +28,13 @@ try:
|
||||
def _restrict(codes: list[str]) -> None:
|
||||
_identifier.set_languages(codes)
|
||||
|
||||
except Exception: # pragma: no cover - fallback to module-level API
|
||||
except Exception: # noqa: BLE001 — pragma: no cover, fallback to the module-level API
|
||||
# The fallback classifies without the normalisation the identifier applies, so language tags
|
||||
# come out subtly different — and nothing else in the app would ever hint at which path ran.
|
||||
log.info(
|
||||
"the langid model could not be loaded; using the module-level classifier",
|
||||
extra=logs.event("autotag.langid.fallback"),
|
||||
)
|
||||
import py3langid as _langid
|
||||
|
||||
def _classify(text: str):
|
||||
|
||||
@@ -8,13 +8,12 @@ stay private to users who explored (or subscribe to) the channel — the feed qu
|
||||
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
|
||||
grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.models import (
|
||||
Channel,
|
||||
ExploredChannel,
|
||||
@@ -28,7 +27,24 @@ from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_d
|
||||
from app.utils import now_utc as _now
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
log = logging.getLogger("siftlode.explore")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def followed_channel():
|
||||
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
|
||||
grace window (`purge_unkept_explored` deletes the expired ExploredChannel rows first). ONE
|
||||
definition for both halves of the question: what the scheduled RSS poll and recent backfill cover
|
||||
(sync/runner.py) and what the explore purge may delete.
|
||||
|
||||
The poll and backfill cover only these (owner decision, 2026-09-17). They used to cover EVERY
|
||||
channel row, but rows also come from search results, playlist sync and dropped subscriptions, and
|
||||
nothing deletes those: on prod 582 of 979 polled channels had no follower, three of them dead
|
||||
feeds answering 404 on every poll. The cost of narrowing is deliberate: the shared Library stops
|
||||
receiving new uploads from channels nobody follows."""
|
||||
return or_(
|
||||
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
|
||||
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
|
||||
)
|
||||
|
||||
|
||||
def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
|
||||
@@ -98,12 +114,8 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
).rowcount
|
||||
db.commit()
|
||||
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
|
||||
exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
|
||||
orphan_ids = (
|
||||
db.execute(
|
||||
select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists)
|
||||
)
|
||||
db.execute(select(Channel.id).where(Channel.from_explore.is_(True), ~followed_channel()))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user