fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
+4 -2
View File
@@ -1,9 +1,9 @@
import { test as setup, expect, type Page } from "@playwright/test";
import { execFileSync } from "node:child_process";
import { mkdirSync, readFileSync } from "node:fs";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { MEMBER_STORAGE } from "./helpers";
import { MEMBER_STORAGE, SEED_FILE } from "./helpers";
import { retryConnect } from "./retryConnect";
/**
@@ -99,6 +99,8 @@ setup("seed + authenticate", async ({ page, browser }) => {
{ input: readFileSync(SEED_SCRIPT) },
);
console.log(`[e2e seed] ${out.toString().trim()}`);
// What the seed chose, for the specs that need one of its videos (helpers.seeded).
writeFileSync(SEED_FILE, out);
} catch (err: unknown) {
const e = err as { stderr?: Buffer; message?: string };
throw new Error(`E2E seed failed: ${e.stderr?.toString() || e.message}`);
+10
View File
@@ -1,8 +1,18 @@
import { readFileSync } from "node:fs";
import { expect, type Page } from "@playwright/test";
/** The plain member's saved session (global.setup.ts signs it in); `member.spec.ts` and
* `devices.spec.ts` run from it. */
export const MEMBER_STORAGE = "e2e/.auth/member.json";
// The seed's own answer (scripts/e2e_seed.py), written by global.setup: the videos it chose.
export const SEED_FILE = "e2e/.auth/seed.json";
/** The videos the seed chose for the main account under `key` in its answer. */
export function seeded(key: string): string[] {
const ids = (JSON.parse(readFileSync(SEED_FILE, "utf-8")) as Record<string, unknown>)[key];
if (!Array.isArray(ids) || !ids.length) throw new Error(`the seed chose no ${key} videos`);
return ids as string[];
}
/** Open the app and wait for the authenticated shell (the nav rail) to be ready. */
export async function openApp(page: Page, path = "/"): Promise<void> {
+7 -7
View File
@@ -1,20 +1,16 @@
import { test, expect } from "@playwright/test";
import { openApp, gotoPage } from "./helpers";
import { openApp, gotoPage, seeded } from "./helpers";
// Settings › Account › YouTube history: a Google Takeout watch history, uploaded through the real
// picker, is read, imported in the background, and the section says what it did — a video the
// catalog knows is marked watched, one it does not know is remembered for later. The seed clears
// the account's kept Takeout videos each run, but the projects share one account: a later one's
// feed no longer holds the video an earlier one marked, so what it adds is counted against what was
// there before it.
// import of the same video adds nothing new, so what each adds is counted against what was there
// before it.
test("a Takeout watch history uploads, imports, and the section says what it did", async ({
page,
}) => {
await openApp(page);
// A video the catalog knows: one on the account's feed (not a Short — the feed leaves them out).
const known = await page.getByTestId("video-card").first().getAttribute("data-video-id");
expect(known).toBeTruthy();
await gotoPage(page, "settings");
await page.getByRole("tab", { name: "Account" }).click();
await expect(page.getByRole("heading", { name: "YouTube history" })).toBeVisible();
@@ -28,6 +24,10 @@ test("a Takeout watch history uploads, imports, and the section says what it did
watched?: Record<string, number>;
};
const watchedBefore = before.watched?.watched ?? 0;
// A video the catalog knows and the Shorts probe has cleared, chosen by the seed (one not probed
// yet would wait for the probe): the next one no earlier project of this run has imported.
const known = seeded("takeout_known")[watchedBefore];
expect(known, "the seed chose too few videos for every project").toBeTruthy();
// Watched now: later than any state the seed set, which the import would otherwise leave alone.
const now = new Date().toISOString();