fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
@@ -313,6 +313,29 @@ describe("backend ids have their names in both languages", () => {
expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours);
});
it("the Takeout import's files and outcomes are the server's", () => {
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
const ours = (name: string) =>
[
...(
new RegExp(`^export const ${name} = \\[([^\\]]*)\\]`, "m").exec(feedTs)?.[1] ?? ""
).matchAll(/"(\w+)"/g),
].map((m) => m[1]);
const models = backend("reader/models.py");
const theirs = (cls: string) =>
[
...(
new RegExp(`^class ${cls}\\(StrEnum\\):\\n([\\s\\S]*?)(?=\\n\\S)`, "m").exec(
models,
)?.[1] ?? ""
).matchAll(/^\s+[A-Z_]+ = "(\w+)"/gm),
].map((m) => m[1]);
expect(theirs("TakeoutKind").length).toBeGreaterThan(0);
expect(ours("TAKEOUT_KINDS")).toEqual(theirs("TakeoutKind"));
expect(theirs("TakeoutOutcome").length).toBeGreaterThan(0);
expect(ours("TAKEOUT_OUTCOMES")).toEqual(theirs("TakeoutOutcome"));
});
for (const lang of ["en", "hu"]) {
const keys = keysOf(lang);
const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k));
+5 -1
View File
@@ -252,6 +252,10 @@
"api_token_name_required": "Give the token a name",
"api_token_limit": "You already have {{limit}} tokens — revoke one first",
"takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.",
"takeout_no_videos": "This file has no videos to import"
"takeout_no_videos": "This file has no videos to import",
"takeout_busy": "Another of your files is still being uploaded — wait for it to finish",
"takeout_cooldown": "You have uploaded many files in the last hour — try again later",
"takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds",
"takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can"
}
}
+4 -2
View File
@@ -187,7 +187,8 @@
"watch_later_present": "already in Watch later",
"watch_later_missing": "no longer on YouTube",
"watch_later_waiting": "Watch later waiting",
"takeout_watched": "marked watched from Takeout"
"takeout_watched": "marked watched from Takeout",
"takeout_watch_later_added": "put into Watch later from Takeout"
},
"skip": {
"disabled": "disabled",
@@ -205,7 +206,8 @@
},
"failure": {
"rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.",
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}."
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}.",
"unexpected": "The run ran into an unexpected error."
},
"rssCause": {
"http": "HTTP {{status}}",
+2 -1
View File
@@ -52,7 +52,8 @@
"missing_other": "{{n}} no longer on YouTube",
"laterWaiting_one": "{{n}} waiting for Watch later",
"laterWaiting_other": "{{n}} waiting for Watch later"
}
},
"tooLarge": "{{name}} is larger than {{limit}} — the import takes files up to that size"
},
"save": {
"saving": "Saving…",
+5 -1
View File
@@ -252,6 +252,10 @@
"api_token_name_required": "Adj nevet a tokennek",
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet",
"takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.",
"takeout_no_videos": "Ebben a fájlban nincs importálható videó"
"takeout_no_videos": "Ebben a fájlban nincs importálható videó",
"takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik",
"takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később",
"takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van",
"takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet"
}
}
+4 -2
View File
@@ -187,7 +187,8 @@
"watch_later_present": "már a „Megnézem később” listán",
"watch_later_missing": "már nincs a YouTube-on",
"watch_later_waiting": "a „Megnézem később” listára vár",
"takeout_watched": "Takeoutból megnézettre jelölve"
"takeout_watched": "Takeoutból megnézettre jelölve",
"takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve"
},
"skip": {
"disabled": "kikapcsolva",
@@ -205,7 +206,8 @@
},
"failure": {
"rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}."
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
"unexpected": "A futás váratlan hibába ütközött."
},
"rssCause": {
"http": "HTTP {{status}}",
+4 -3
View File
@@ -44,15 +44,16 @@
"short_other": "{{n}} Short kihagyva",
"kept_one": "{{n}} úgy maradt, ahogy itt beállítottad",
"kept_other": "{{n}} úgy maradt, ahogy itt beállítottad",
"added_one": "{{n}} a „Megnézem később” listába",
"added_other": "{{n}} a „Megnézem később” listába",
"added_one": "{{n}} a „Megnézem később” listába téve",
"added_other": "{{n}} a „Megnézem később” listába téve",
"present_one": "{{n}} már a „Megnézem később” listán",
"present_other": "{{n}} már a „Megnézem később” listán",
"missing_one": "{{n}} már nincs fent a YouTube-on",
"missing_other": "{{n}} már nincs fent a YouTube-on",
"laterWaiting_one": "{{n}} vár a „Megnézem később” listára",
"laterWaiting_other": "{{n}} vár a „Megnézem később” listára"
}
},
"tooLarge": "{{name}}: nagyobb, mint {{limit}} — az import legfeljebb ekkora fájlt fogad"
},
"save": {
"saving": "Mentés…",