fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
@@ -313,6 +313,29 @@ describe("backend ids have their names in both languages", () => {
|
||||
expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours);
|
||||
});
|
||||
|
||||
it("the Takeout import's files and outcomes are the server's", () => {
|
||||
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
|
||||
const ours = (name: string) =>
|
||||
[
|
||||
...(
|
||||
new RegExp(`^export const ${name} = \\[([^\\]]*)\\]`, "m").exec(feedTs)?.[1] ?? ""
|
||||
).matchAll(/"(\w+)"/g),
|
||||
].map((m) => m[1]);
|
||||
const models = backend("reader/models.py");
|
||||
const theirs = (cls: string) =>
|
||||
[
|
||||
...(
|
||||
new RegExp(`^class ${cls}\\(StrEnum\\):\\n([\\s\\S]*?)(?=\\n\\S)`, "m").exec(
|
||||
models,
|
||||
)?.[1] ?? ""
|
||||
).matchAll(/^\s+[A-Z_]+ = "(\w+)"/gm),
|
||||
].map((m) => m[1]);
|
||||
expect(theirs("TakeoutKind").length).toBeGreaterThan(0);
|
||||
expect(ours("TAKEOUT_KINDS")).toEqual(theirs("TakeoutKind"));
|
||||
expect(theirs("TakeoutOutcome").length).toBeGreaterThan(0);
|
||||
expect(ours("TAKEOUT_OUTCOMES")).toEqual(theirs("TakeoutOutcome"));
|
||||
});
|
||||
|
||||
for (const lang of ["en", "hu"]) {
|
||||
const keys = keysOf(lang);
|
||||
const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k));
|
||||
|
||||
@@ -252,6 +252,10 @@
|
||||
"api_token_name_required": "Give the token a name",
|
||||
"api_token_limit": "You already have {{limit}} tokens — revoke one first",
|
||||
"takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.",
|
||||
"takeout_no_videos": "This file has no videos to import"
|
||||
"takeout_no_videos": "This file has no videos to import",
|
||||
"takeout_busy": "Another of your files is still being uploaded — wait for it to finish",
|
||||
"takeout_cooldown": "You have uploaded many files in the last hour — try again later",
|
||||
"takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds",
|
||||
"takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,7 +187,8 @@
|
||||
"watch_later_present": "already in Watch later",
|
||||
"watch_later_missing": "no longer on YouTube",
|
||||
"watch_later_waiting": "Watch later waiting",
|
||||
"takeout_watched": "marked watched from Takeout"
|
||||
"takeout_watched": "marked watched from Takeout",
|
||||
"takeout_watch_later_added": "put into Watch later from Takeout"
|
||||
},
|
||||
"skip": {
|
||||
"disabled": "disabled",
|
||||
@@ -205,7 +206,8 @@
|
||||
},
|
||||
"failure": {
|
||||
"rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.",
|
||||
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}."
|
||||
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}.",
|
||||
"unexpected": "The run ran into an unexpected error."
|
||||
},
|
||||
"rssCause": {
|
||||
"http": "HTTP {{status}}",
|
||||
|
||||
@@ -52,7 +52,8 @@
|
||||
"missing_other": "{{n}} no longer on YouTube",
|
||||
"laterWaiting_one": "{{n}} waiting for Watch later",
|
||||
"laterWaiting_other": "{{n}} waiting for Watch later"
|
||||
}
|
||||
},
|
||||
"tooLarge": "{{name}} is larger than {{limit}} — the import takes files up to that size"
|
||||
},
|
||||
"save": {
|
||||
"saving": "Saving…",
|
||||
|
||||
@@ -252,6 +252,10 @@
|
||||
"api_token_name_required": "Adj nevet a tokennek",
|
||||
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet",
|
||||
"takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.",
|
||||
"takeout_no_videos": "Ebben a fájlban nincs importálható videó"
|
||||
"takeout_no_videos": "Ebben a fájlban nincs importálható videó",
|
||||
"takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik",
|
||||
"takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később",
|
||||
"takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van",
|
||||
"takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,7 +187,8 @@
|
||||
"watch_later_present": "már a „Megnézem később” listán",
|
||||
"watch_later_missing": "már nincs a YouTube-on",
|
||||
"watch_later_waiting": "a „Megnézem később” listára vár",
|
||||
"takeout_watched": "Takeoutból megnézettre jelölve"
|
||||
"takeout_watched": "Takeoutból megnézettre jelölve",
|
||||
"takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve"
|
||||
},
|
||||
"skip": {
|
||||
"disabled": "kikapcsolva",
|
||||
@@ -205,7 +206,8 @@
|
||||
},
|
||||
"failure": {
|
||||
"rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
|
||||
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}."
|
||||
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
|
||||
"unexpected": "A futás váratlan hibába ütközött."
|
||||
},
|
||||
"rssCause": {
|
||||
"http": "HTTP {{status}}",
|
||||
|
||||
@@ -44,15 +44,16 @@
|
||||
"short_other": "{{n}} Short kihagyva",
|
||||
"kept_one": "{{n}} úgy maradt, ahogy itt beállítottad",
|
||||
"kept_other": "{{n}} úgy maradt, ahogy itt beállítottad",
|
||||
"added_one": "{{n}} a „Megnézem később” listába",
|
||||
"added_other": "{{n}} a „Megnézem később” listába",
|
||||
"added_one": "{{n}} a „Megnézem később” listába téve",
|
||||
"added_other": "{{n}} a „Megnézem később” listába téve",
|
||||
"present_one": "{{n}} már a „Megnézem később” listán",
|
||||
"present_other": "{{n}} már a „Megnézem később” listán",
|
||||
"missing_one": "{{n}} már nincs fent a YouTube-on",
|
||||
"missing_other": "{{n}} már nincs fent a YouTube-on",
|
||||
"laterWaiting_one": "{{n}} vár a „Megnézem később” listára",
|
||||
"laterWaiting_other": "{{n}} vár a „Megnézem később” listára"
|
||||
}
|
||||
},
|
||||
"tooLarge": "{{name}}: nagyobb, mint {{limit}} — az import legfeljebb ekkora fájlt fogad"
|
||||
},
|
||||
"save": {
|
||||
"saving": "Mentés…",
|
||||
|
||||
Reference in New Issue
Block a user