fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
+4 -2
View File
@@ -1,9 +1,9 @@
import { test as setup, expect, type Page } from "@playwright/test";
import { execFileSync } from "node:child_process";
import { mkdirSync, readFileSync } from "node:fs";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { MEMBER_STORAGE } from "./helpers";
import { MEMBER_STORAGE, SEED_FILE } from "./helpers";
import { retryConnect } from "./retryConnect";
/**
@@ -99,6 +99,8 @@ setup("seed + authenticate", async ({ page, browser }) => {
{ input: readFileSync(SEED_SCRIPT) },
);
console.log(`[e2e seed] ${out.toString().trim()}`);
// What the seed chose, for the specs that need one of its videos (helpers.seeded).
writeFileSync(SEED_FILE, out);
} catch (err: unknown) {
const e = err as { stderr?: Buffer; message?: string };
throw new Error(`E2E seed failed: ${e.stderr?.toString() || e.message}`);
+10
View File
@@ -1,8 +1,18 @@
import { readFileSync } from "node:fs";
import { expect, type Page } from "@playwright/test";
/** The plain member's saved session (global.setup.ts signs it in); `member.spec.ts` and
* `devices.spec.ts` run from it. */
export const MEMBER_STORAGE = "e2e/.auth/member.json";
// The seed's own answer (scripts/e2e_seed.py), written by global.setup: the videos it chose.
export const SEED_FILE = "e2e/.auth/seed.json";
/** The videos the seed chose for the main account under `key` in its answer. */
export function seeded(key: string): string[] {
const ids = (JSON.parse(readFileSync(SEED_FILE, "utf-8")) as Record<string, unknown>)[key];
if (!Array.isArray(ids) || !ids.length) throw new Error(`the seed chose no ${key} videos`);
return ids as string[];
}
/** Open the app and wait for the authenticated shell (the nav rail) to be ready. */
export async function openApp(page: Page, path = "/"): Promise<void> {
+7 -7
View File
@@ -1,20 +1,16 @@
import { test, expect } from "@playwright/test";
import { openApp, gotoPage } from "./helpers";
import { openApp, gotoPage, seeded } from "./helpers";
// Settings › Account › YouTube history: a Google Takeout watch history, uploaded through the real
// picker, is read, imported in the background, and the section says what it did — a video the
// catalog knows is marked watched, one it does not know is remembered for later. The seed clears
// the account's kept Takeout videos each run, but the projects share one account: a later one's
// feed no longer holds the video an earlier one marked, so what it adds is counted against what was
// there before it.
// import of the same video adds nothing new, so what each adds is counted against what was there
// before it.
test("a Takeout watch history uploads, imports, and the section says what it did", async ({
page,
}) => {
await openApp(page);
// A video the catalog knows: one on the account's feed (not a Short — the feed leaves them out).
const known = await page.getByTestId("video-card").first().getAttribute("data-video-id");
expect(known).toBeTruthy();
await gotoPage(page, "settings");
await page.getByRole("tab", { name: "Account" }).click();
await expect(page.getByRole("heading", { name: "YouTube history" })).toBeVisible();
@@ -28,6 +24,10 @@ test("a Takeout watch history uploads, imports, and the section says what it did
watched?: Record<string, number>;
};
const watchedBefore = before.watched?.watched ?? 0;
// A video the catalog knows and the Shorts probe has cleared, chosen by the seed (one not probed
// yet would wait for the probe): the next one no earlier project of this run has imported.
const known = seeded("takeout_known")[watchedBefore];
expect(known, "the seed chose too few videos for every project").toBeTruthy();
// Watched now: later than any state the seed set, which the import would otherwise leave alone.
const now = new Date().toISOString();
+5 -1
View File
@@ -1,4 +1,4 @@
import { useRef, useState, type ReactNode } from "react";
import { useRef, useState, type ReactNode, type Ref } from "react";
import clsx from "clsx";
import { Pressable } from "./Pressable";
@@ -8,6 +8,7 @@ import { Pressable } from "./Pressable";
// children (an icon, a line of text), so a plain boolean flickers — enter and leave are counted
// and the highlight clears at depth 0. The caller draws the inside, told whether a drag is over it.
export function DropZone({
ref,
onFiles,
multiple = false,
accept,
@@ -17,6 +18,8 @@ export function DropZone({
className,
children,
}: {
/** The zone's button, for a caller that moves the focus to it. */
ref?: Ref<HTMLButtonElement>;
/** The files dropped or picked, never empty. One unless `multiple`. */
onFiles: (files: [File, ...File[]]) => void;
multiple?: boolean;
@@ -40,6 +43,7 @@ export function DropZone({
return (
<>
<Pressable
ref={ref}
align="center"
onClick={() => inputRef.current?.click()}
onDragEnter={(e) => {
@@ -313,6 +313,29 @@ describe("backend ids have their names in both languages", () => {
expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours);
});
it("the Takeout import's files and outcomes are the server's", () => {
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
const ours = (name: string) =>
[
...(
new RegExp(`^export const ${name} = \\[([^\\]]*)\\]`, "m").exec(feedTs)?.[1] ?? ""
).matchAll(/"(\w+)"/g),
].map((m) => m[1]);
const models = backend("reader/models.py");
const theirs = (cls: string) =>
[
...(
new RegExp(`^class ${cls}\\(StrEnum\\):\\n([\\s\\S]*?)(?=\\n\\S)`, "m").exec(
models,
)?.[1] ?? ""
).matchAll(/^\s+[A-Z_]+ = "(\w+)"/gm),
].map((m) => m[1]);
expect(theirs("TakeoutKind").length).toBeGreaterThan(0);
expect(ours("TAKEOUT_KINDS")).toEqual(theirs("TakeoutKind"));
expect(theirs("TakeoutOutcome").length).toBeGreaterThan(0);
expect(ours("TAKEOUT_OUTCOMES")).toEqual(theirs("TakeoutOutcome"));
});
for (const lang of ["en", "hu"]) {
const keys = keysOf(lang);
const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k));
+5 -1
View File
@@ -252,6 +252,10 @@
"api_token_name_required": "Give the token a name",
"api_token_limit": "You already have {{limit}} tokens — revoke one first",
"takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.",
"takeout_no_videos": "This file has no videos to import"
"takeout_no_videos": "This file has no videos to import",
"takeout_busy": "Another of your files is still being uploaded — wait for it to finish",
"takeout_cooldown": "You have uploaded many files in the last hour — try again later",
"takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds",
"takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can"
}
}
+4 -2
View File
@@ -187,7 +187,8 @@
"watch_later_present": "already in Watch later",
"watch_later_missing": "no longer on YouTube",
"watch_later_waiting": "Watch later waiting",
"takeout_watched": "marked watched from Takeout"
"takeout_watched": "marked watched from Takeout",
"takeout_watch_later_added": "put into Watch later from Takeout"
},
"skip": {
"disabled": "disabled",
@@ -205,7 +206,8 @@
},
"failure": {
"rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.",
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}."
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}.",
"unexpected": "The run ran into an unexpected error."
},
"rssCause": {
"http": "HTTP {{status}}",
+2 -1
View File
@@ -52,7 +52,8 @@
"missing_other": "{{n}} no longer on YouTube",
"laterWaiting_one": "{{n}} waiting for Watch later",
"laterWaiting_other": "{{n}} waiting for Watch later"
}
},
"tooLarge": "{{name}} is larger than {{limit}} — the import takes files up to that size"
},
"save": {
"saving": "Saving…",
+5 -1
View File
@@ -252,6 +252,10 @@
"api_token_name_required": "Adj nevet a tokennek",
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet",
"takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.",
"takeout_no_videos": "Ebben a fájlban nincs importálható videó"
"takeout_no_videos": "Ebben a fájlban nincs importálható videó",
"takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik",
"takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később",
"takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van",
"takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet"
}
}
+4 -2
View File
@@ -187,7 +187,8 @@
"watch_later_present": "már a „Megnézem később” listán",
"watch_later_missing": "már nincs a YouTube-on",
"watch_later_waiting": "a „Megnézem később” listára vár",
"takeout_watched": "Takeoutból megnézettre jelölve"
"takeout_watched": "Takeoutból megnézettre jelölve",
"takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve"
},
"skip": {
"disabled": "kikapcsolva",
@@ -205,7 +206,8 @@
},
"failure": {
"rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}."
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
"unexpected": "A futás váratlan hibába ütközött."
},
"rssCause": {
"http": "HTTP {{status}}",
+4 -3
View File
@@ -44,15 +44,16 @@
"short_other": "{{n}} Short kihagyva",
"kept_one": "{{n}} úgy maradt, ahogy itt beállítottad",
"kept_other": "{{n}} úgy maradt, ahogy itt beállítottad",
"added_one": "{{n}} a „Megnézem később” listába",
"added_other": "{{n}} a „Megnézem később” listába",
"added_one": "{{n}} a „Megnézem később” listába téve",
"added_other": "{{n}} a „Megnézem később” listába téve",
"present_one": "{{n}} már a „Megnézem később” listán",
"present_other": "{{n}} már a „Megnézem később” listán",
"missing_one": "{{n}} már nincs fent a YouTube-on",
"missing_other": "{{n}} már nincs fent a YouTube-on",
"laterWaiting_one": "{{n}} vár a „Megnézem később” listára",
"laterWaiting_other": "{{n}} vár a „Megnézem később” listára"
}
},
"tooLarge": "{{name}}: nagyobb, mint {{limit}} — az import legfeljebb ekkora fájlt fogad"
},
"save": {
"saving": "Mentés…",
+19 -5
View File
@@ -164,12 +164,26 @@ export interface TakeoutUpload {
other: number;
malformed: number;
}
export type TakeoutKind = "watched" | "watch_later";
// A kept video's outcome (`reader.models.TakeoutOutcome`), or "waiting" while the catalog does not
// know it yet: watched / kept (the user's own state here stood) / short for the history; added /
// present / missing for Watch later.
export type TakeoutCounts = Partial<Record<string, number>>;
// The two files (`reader.models.TakeoutKind`) and a kept video's outcome (`TakeoutOutcome`): watched
// / kept (the user's own state here stood) / short for the history; added / present / missing for
// Watch later. backendIds.contract.test holds both lists to the backend's enums.
export const TAKEOUT_KINDS = ["watched", "watch_later"] as const;
export const TAKEOUT_OUTCOMES = [
"watched",
"kept",
"short",
"added",
"present",
"missing",
] as const;
export type TakeoutKind = (typeof TAKEOUT_KINDS)[number];
// Each outcome's count, and "waiting" for the videos the import could not settle yet.
export type TakeoutCounts = Partial<Record<(typeof TAKEOUT_OUTCOMES)[number] | "waiting", number>>;
export interface TakeoutStatus {
/** An import is queued and its run not started: `run` is still the one before it. */
queued: boolean;
/** The largest file the upload takes. */
max_bytes: number;
run: {
status: RunStatus;
started_at: string | null;
@@ -56,6 +56,14 @@ const uploaded = (videos: number): TakeoutUpload => ({
malformed: 0,
});
// A whole status: nothing queued and room for any file, unless the test says otherwise.
const st = (s: Partial<TakeoutStatus>): TakeoutStatus => ({
queued: false,
max_bytes: 1_000_000,
run: null,
...s,
});
function mount(status: TakeoutStatus | Error | Promise<never>) {
const spy = vi.spyOn(api, "takeoutStatus");
if (status instanceof Error) spy.mockRejectedValue(status);
@@ -77,11 +85,13 @@ const drop = (files: File[]) =>
act(async () => {
fireEvent.drop(zone()!, { dataTransfer: { files } });
});
const file = (name: string) => new File(["[]"], name);
const file = (name: string, size = 2) => new File(["x".repeat(size)], name);
const unavailable = (el: HTMLElement | null) =>
!!el && ((el as HTMLButtonElement).disabled || el.getAttribute("aria-disabled") === "true");
describe("YouTube history import · what it says", () => {
it("before the first import: the drop zone, and no line about an import", async () => {
mount({ run: null, watched: {}, watch_later: {} });
mount(st({ run: null, watched: {}, watch_later: {} }));
await waitFor(() => expect(zone()).toBeTruthy());
expect(screen.queryByTestId("takeout-last")).toBeNull();
});
@@ -92,11 +102,13 @@ describe("YouTube history import · what it says", () => {
});
it("after one: when, then only the counts it produced, the history's first", async () => {
mount({
run: run("ok"),
watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 },
watch_later: { added: 41, waiting: 2 },
});
mount(
st({
run: run("ok"),
watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 },
watch_later: { added: 41, waiting: 2 },
}),
);
await waitFor(() => expect(screen.getByTestId("takeout-last")).toBeTruthy());
expect(line()).toBe(
[
@@ -114,20 +126,20 @@ describe("YouTube history import · what it says", () => {
});
it("a failed run and one a restart cut off say so, not counts", async () => {
mount({ run: run("error"), watched: { watched: 3 } });
mount(st({ run: run("error"), watched: { watched: 3 } }));
await waitFor(() => expect(line()).toBe("settings.takeout.failed(at F)"));
cleanup();
mount({ run: run("interrupted"), watched: { watched: 3 } });
mount(st({ run: run("interrupted"), watched: { watched: 3 } }));
await waitFor(() => expect(line()).toBe("settings.takeout.interrupted(at S)"));
});
it("a running import: the running line, no drop zone, asked again until it ends", async () => {
const spy = mount({ run: run("running"), watched: {} });
it("a running import: the running line, the zone unavailable, asked again until it ends", async () => {
const spy = mount(st({ run: run("running"), watched: {} }));
await waitFor(() =>
expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"),
);
expect(zone()).toBeNull();
spy.mockResolvedValue({ run: run("ok"), watched: { watched: 5 } });
expect(unavailable(zone())).toBe(true);
spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 5 } }));
await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(5,5)"), {
timeout: 3000,
});
@@ -143,7 +155,7 @@ describe("YouTube history import · what it says", () => {
});
it("a re-read that fails does not leave the previous answer standing under it", async () => {
const status = mount({ run: run("ok", { finished_at: "OLD" }), watched: {} });
const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
await waitFor(() => expect(line()).toContain("OLD"));
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
@@ -158,7 +170,7 @@ describe("YouTube history import · what it says", () => {
describe("YouTube history import · the upload", () => {
it("sends each dropped file on its own, says each outcome, and reads the new run", async () => {
const status = mount({ run: null, watched: {} });
const status = mount(st({ run: null, watched: {} }));
await waitFor(() => expect(zone()).toBeTruthy());
const refused = new HttpError(422, "refused");
const sent: string[] = [];
@@ -167,7 +179,7 @@ describe("YouTube history import · the upload", () => {
if (f.name === "bad.html") throw refused;
return uploaded(f.name === "watch-history.json" ? 9034 : 56);
});
status.mockResolvedValue({ run: run("ok"), watched: { watched: 7 } });
status.mockResolvedValue(st({ run: run("ok"), watched: { watched: 7 } }));
await drop([file("watch-history.json"), file("bad.html"), file("Watch later-videos.csv")]);
await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(7,7)"));
@@ -183,7 +195,7 @@ describe("YouTube history import · the upload", () => {
});
it("names the file being sent, then never shows the previous import's line before the new one", async () => {
const status = mount({ run: run("ok", { finished_at: "OLD" }), watched: {} });
const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
await waitFor(() => expect(line()).toContain("OLD"));
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
let finish!: (v: TakeoutUpload) => void;
@@ -197,7 +209,115 @@ describe("YouTube history import · the upload", () => {
await act(async () => finish(uploaded(1)));
// The new run is still being read: the upload's line stays, the old import's does not return.
expect(screen.queryByTestId("takeout-last")).toBeNull();
await act(async () => answer({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
await act(async () => answer(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })));
await waitFor(() => expect(line()).toContain("NEW"));
});
});
describe("YouTube history import · while it runs", () => {
it("a queued import is this upload's, not the last one: the running line until its run reads", async () => {
const spy = mount(st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }));
await waitFor(() =>
expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"),
);
expect(screen.queryByTestId("takeout-last")).toBeNull();
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
});
it("a re-read that fails mid-run does not leave the spinner claiming progress under it", async () => {
const spy = mount(st({ run: run("running"), watched: {} }));
await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy());
spy.mockRejectedValue(new HttpError(500, "boom"));
await waitFor(
() => expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"),
{ timeout: 3000 },
);
expect(screen.queryByTestId("takeout-running")).toBeNull();
spy.mockResolvedValue(st({ run: run("ok"), watched: {} }));
await waitFor(() => expect(screen.queryByRole("alert")).toBeNull(), { timeout: 3000 });
});
it("one live region says the run and then its outcome", async () => {
const spy = mount(st({ run: run("running"), watched: {} }));
await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy());
const region = screen.getByRole("status");
spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 2 } }));
await waitFor(() => expect(line()).toContain("watched(2,2)"), { timeout: 3000 });
expect(screen.getByRole("status")).toBe(region);
expect(region.contains(screen.getByTestId("takeout-last"))).toBe(true);
});
it("a file dropped beside the zone is kept from the browser, which would open it", async () => {
mount(st({ run: null, watched: {} }));
await waitFor(() => expect(zone()).toBeTruthy());
const missed = (type: string, types: string[]) => {
const e = new Event(type, { bubbles: true, cancelable: true });
Object.defineProperty(e, "dataTransfer", { value: { types, dropEffect: "copy" } });
document.body.dispatchEvent(e);
return e;
};
expect(missed("drop", ["Files"]).defaultPrevented).toBe(true);
expect(missed("dragover", ["Files"]).defaultPrevented).toBe(true);
// A drag of something else (a link, text) is the page's own business.
expect(missed("drop", ["text/plain"]).defaultPrevented).toBe(false);
cleanup();
expect(missed("drop", ["Files"]).defaultPrevented).toBe(false);
});
it("a file larger than the import takes is refused before it is sent, naming the limit", async () => {
mount(st({ max_bytes: 10, run: run("ok"), watched: {} }));
await waitFor(() => expect(line()).toBeTruthy());
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
const send = vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
await drop([file("watch-history.json", 11)]);
expect(send).not.toHaveBeenCalled();
expect(mocks.notify).toHaveBeenCalledWith({
level: "error",
message: "settings.takeout.tooLarge(watch-history.json,10 B)",
});
// Nothing was taken: the zone stays open for the right file, and so after a refusal.
await waitFor(() => expect(zone()).toBeTruthy());
send.mockRejectedValue(new HttpError(422, "refused"));
await drop([file("bad.html")]);
await waitFor(() => expect(mocks.notifyActionError).toHaveBeenCalled());
expect(zone()).toBeTruthy();
});
it("the focus follows: New import hands it to the zone, the zone back to New import", async () => {
const spy = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
await waitFor(() => expect(line()).toContain("OLD"));
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
expect(document.activeElement).toBe(zone());
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
spy.mockResolvedValue(
st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }),
);
await drop([file("watch-history.json")]);
// While it runs the zone stays, unavailable, and keeps the focus.
expect(document.activeElement).toBe(zone());
expect(unavailable(zone())).toBe(true);
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
expect(zone()).toBeNull();
expect(document.activeElement).toBe(
screen.getByRole("button", { name: "settings.takeout.newImport" }),
);
});
it("a focus the user moved elsewhere meanwhile is not taken back", async () => {
const spy = mount(st({ run: null, watched: {} }));
await waitFor(() => expect(zone()).toBeTruthy());
const elsewhere = document.body.appendChild(document.createElement("button"));
zone()!.focus();
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
spy.mockResolvedValue(st({ queued: true, run: null, watched: {} }));
await drop([file("watch-history.json")]);
elsewhere.focus();
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
expect(document.activeElement).toBe(elsewhere);
elsewhere.remove();
});
});
+140 -67
View File
@@ -1,4 +1,4 @@
import { useEffect, useState } from "react";
import { useEffect, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { RefreshCw, UploadCloud } from "lucide-react";
@@ -6,19 +6,19 @@ import { Section } from "../../components/ui/form";
import { Button } from "../../components/ui/Button";
import { DropZone } from "../../components/ui/DropZone";
import { StateMessage } from "../../components/ui/QueryState";
import { api, type TakeoutStatus } from "../../lib/api";
import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api";
import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed";
import { qk } from "../../lib/queryKeys";
import { notify } from "../../lib/notifications";
import { notifyActionError } from "../../lib/actionErrors";
import { formatDateTime, formatNumber } from "../../lib/format";
import { formatBytes, formatDateTime, formatNumber } from "../../lib/format";
import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch";
// Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and
// their YouTube Watch later list reach no API, so the export's two files are uploaded here, one
// request each; the server tells them apart and runs the import in the background, and its outcome
// also reaches the inbox. The import has no steps to count (the whole history is applied in one
// go), so while it runs the section says so and asks again until it is done.
const POLL_MS = 1500;
// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`).
// The outcome counts, in the order they are said: the history's, then Watch later's. A count the
// import did not produce is not said at all.
@@ -35,6 +35,12 @@ const WATCH_LATER = [
["waiting", "laterWaiting"],
] as const;
// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it).
type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number];
type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0];
const everyOutcomeSaid: [Exclude<TakeoutOutcome, Said>] extends [never] ? true : never = true;
void everyOutcomeSaid;
function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] {
const parts: string[] = [];
for (const [outcome, key] of HISTORY) {
@@ -52,58 +58,111 @@ export default function TakeoutImport() {
const { t, i18n } = useTranslation();
const lang = i18n.language;
const qc = useQueryClient();
const status = useQuery({
queryKey: qk.takeout(),
queryFn: api.takeoutStatus,
refetchInterval: (q) => (q.state.data?.run?.status === "running" ? POLL_MS : false),
});
const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus });
const [uploading, setUploading] = useState<string | null>(null);
// "New import" reopens the drop zone over a finished one; before the first import it is open.
const [reopened, setReopened] = useState(false);
const run = status.data?.run ?? null;
const running = uploading !== null || run?.status === "running";
// Queued counts too: until its run row is written, `run` is still the import before it.
const busy = importBusy(status.data);
const running = uploading !== null || busy;
const finishedAt = run?.finished_at ?? null;
// A run that ended has changed what the feed and Watch later show: those lists are read again
// when the user next opens them, not 30 s later from the cache.
// A run under way is watched to its end, even if the user leaves Settings meanwhile.
useEffect(() => {
if (!finishedAt) return;
void qc.invalidateQueries({ queryKey: qk.feed() });
void qc.invalidateQueries({ queryKey: qk.playlist() });
if (busy) void watchTakeout(qc);
}, [busy, qc]);
// One that ended while nothing here watched it (a reload, another tab) has changed what the feed
// and Watch later show too: those lists are read again when next opened, not from the cache.
useEffect(() => {
if (finishedAt) invalidateImported(qc);
}, [finishedAt, qc]);
// A file dropped beside the zone, or while it is not there, is kept from the browser: it would
// open the file in the tab, unloading the app and the upload under way. The zone takes its own.
useEffect(() => {
const missed = (e: DragEvent) => {
if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return;
e.preventDefault();
if (e.type === "dragover") e.dataTransfer.dropEffect = "none";
};
window.addEventListener("dragover", missed);
window.addEventListener("drop", missed);
return () => {
window.removeEventListener("dragover", missed);
window.removeEventListener("drop", missed);
};
}, []);
// Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and
// the zone, gone once the import it started is done, hands it to "New import".
const zone = useRef<HTMLButtonElement | null>(null);
const focusNext = useRef<"zone" | "newImport" | null>(null);
const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => {
if (!el || focusNext.current !== which) return;
focusNext.current = null;
// Only focus that was lost with the control that held it: never taken from where the user went.
if (document.activeElement === null || document.activeElement === document.body) el.focus();
};
const zoneRef = (el: HTMLButtonElement | null) => {
zone.current = el;
focusIfNext("zone", el);
};
const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el);
const upload = async (files: File[]) => {
for (const file of files) await uploadOne(file);
if (document.activeElement === zone.current) focusNext.current = "newImport";
let taken = false;
for (const file of files) if (await uploadOne(file)) taken = true;
// The new run read first, then the upload's line let go: the other way round the previous
// import's line showed for a moment, as if this upload had done nothing.
await qc.invalidateQueries({ queryKey: qk.takeout() });
setUploading(null);
setReopened(false);
// Nothing taken, nothing imported: the zone stays open for the right file.
if (taken) setReopened(false);
else focusNext.current = null;
};
const uploadOne = async (file: File) => {
setUploading(file.name);
try {
const res = await api.takeoutUpload(file);
/** Send one file; whether the server took it. */
const uploadOne = async (file: File): Promise<boolean> => {
const max = status.data?.max_bytes ?? Infinity;
if (file.size > max) {
// Refused here, not after minutes of upload, and with the limit named.
notify({
level: "success",
message: t("settings.takeout.read", {
name: file.name,
count: res.videos,
n: formatNumber(res.videos, lang),
}),
level: "error",
message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }),
});
return false;
}
setUploading(file.name);
let res: TakeoutUpload | null = null;
try {
res = await api.takeoutUpload(file);
} catch (e) {
// A refusal names its reason (not a Takeout file, nothing in it); the next file still goes.
notifyActionError(e, t("errors.writeFailed.takeoutUpload"));
}
if (res === null) return false;
notify({
level: "success",
message: t("settings.takeout.read", {
name: file.name,
count: res.videos,
n: formatNumber(res.videos, lang),
}),
});
return true;
};
const say = (key: string, n: number) =>
t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) });
const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : "");
// Not while the last import is still being asked for: a zone that flashes before the line that
// replaces it would offer a first import to someone who has done one.
const showZone = status.isSuccess && !running && (run === null || reopened);
// replaces it would offer a first import to someone who has done one. While an import runs the
// zone stays, unavailable: the control that started it keeps the focus.
const showZone = status.isSuccess && (running || run === null || reopened);
// A failed re-read keeps the previous answer: it is not shown under the line saying the read
// failed, as if it were current — a run's spinner no more than its outcome.
const current = !status.isError || uploading !== null;
return (
<Section title={t("settings.takeout.title")}>
@@ -117,7 +176,14 @@ export default function TakeoutImport() {
{showZone && (
<>
<DropZone className="px-4 py-5" multiple accept=".json,.csv" onFiles={upload}>
<DropZone
ref={zoneRef}
className="px-4 py-5"
multiple
accept=".json,.csv"
disabled={running}
onFiles={upload}
>
{(dragOver) => (
<>
<UploadCloud className="w-6 h-6 text-muted" />
@@ -138,42 +204,49 @@ export default function TakeoutImport() {
</>
)}
{running ? (
<div
className="mt-2 flex items-center gap-2 text-xs text-muted"
role="status"
data-testid="takeout-running"
>
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
{uploading !== null
? t("settings.takeout.uploading", { name: uploading })
: t("settings.takeout.running")}
<div className="mt-2 flex items-center justify-between gap-3">
{/* One live region for the run and its outcome, mounted throughout: what replaces
"Processing…" is announced, which a region mounted with its text is not. */}
<div role="status" className="min-w-0 text-xs text-muted">
{running && current ? (
<span className="flex items-center gap-2" data-testid="takeout-running">
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
{uploading !== null
? t("settings.takeout.uploading", { name: uploading })
: t("settings.takeout.running")}
</span>
) : (
!running &&
run &&
status.data &&
current && (
<p data-testid="takeout-last">
{run.status === "error"
? t("settings.takeout.failed", { when: when(run.finished_at) })
: run.status === "interrupted"
? t("settings.takeout.interrupted", { when: when(run.started_at) })
: [
t("settings.takeout.last", { when: when(run.finished_at) }),
...countParts(status.data, say),
].join(" · ")}
</p>
)
)}
</div>
) : (
// A failed re-read keeps the previous answer: it is not shown under the line saying the
// read failed, as if it were current.
run &&
status.data &&
!status.isError && (
<div className="mt-2 flex items-center justify-between gap-3">
<p className="text-xs text-muted" data-testid="takeout-last">
{run.status === "error"
? t("settings.takeout.failed", { when: when(run.finished_at) })
: run.status === "interrupted"
? t("settings.takeout.interrupted", { when: when(run.started_at) })
: [
t("settings.takeout.last", { when: when(run.finished_at) }),
...countParts(status.data, say),
].join(" · ")}
</p>
{!reopened && (
<Button variant="outline" size="sm" onClick={() => setReopened(true)}>
{t("settings.takeout.newImport")}
</Button>
)}
</div>
)
)}
{!running && run && status.data && current && !reopened && (
<Button
ref={newImportRef}
variant="outline"
size="sm"
onClick={() => {
focusNext.current = "zone";
setReopened(true);
}}
>
{t("settings.takeout.newImport")}
</Button>
)}
</div>
</Section>
);
}
@@ -0,0 +1,67 @@
// @vitest-environment jsdom
//
// A Takeout import is watched to its end whether or not its section is still on screen, and what it
// changed is read afresh then: the feed and its counts, Watch later, and the playlists' sizes.
import { afterEach, describe, expect, it, vi } from "vitest";
import { QueryClient } from "@tanstack/react-query";
import { api, type TakeoutStatus } from "../../lib/api";
import { qk } from "../../lib/queryKeys";
import { importBusy, watchTakeout } from "./takeoutWatch";
afterEach(() => {
vi.restoreAllMocks();
vi.useRealTimers();
});
const st = (s: Partial<TakeoutStatus>): TakeoutStatus => ({
queued: false,
max_bytes: 1,
run: null,
...s,
});
const running = {
status: "running",
started_at: "S",
finished_at: null,
result: null,
failure: null,
};
describe("the Takeout import's watch", () => {
it("a queued import is busy as much as a running one", () => {
expect(importBusy(st({ queued: true }))).toBe(true);
expect(importBusy(st({ run: { ...running, status: "running" } as TakeoutStatus["run"] }))).toBe(
true,
);
expect(importBusy(st({ run: { ...running, status: "ok" } as TakeoutStatus["run"] }))).toBe(
false,
);
expect(importBusy(undefined)).toBe(false);
});
it("asks until the import is done, then reads again everything it changed", async () => {
vi.useFakeTimers();
const qc = new QueryClient();
qc.setQueryData(qk.takeout(), st({ queued: true }));
const answers = [st({ run: running as TakeoutStatus["run"] }), st({})];
const read = vi.spyOn(api, "takeoutStatus").mockImplementation(async () => answers.shift()!);
const stale = vi.spyOn(qc, "invalidateQueries");
const done = watchTakeout(qc);
void watchTakeout(qc); // a second call while it watches is the same watch
await vi.advanceTimersByTimeAsync(1500);
expect(read).toHaveBeenCalledTimes(1);
expect(stale).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1500);
await done;
expect(read).toHaveBeenCalledTimes(2);
expect(stale.mock.calls.map(([f]) => f?.queryKey)).toEqual([
qk.feed(),
qk.feedSummary(),
qk.feedYears(),
qk.playlist(),
qk.playlists(),
]);
});
});
@@ -0,0 +1,57 @@
import type { QueryClient } from "@tanstack/react-query";
import { api, type TakeoutStatus } from "../../lib/api";
import { qk } from "../../lib/queryKeys";
// A Takeout import, watched to its end: its status is asked again until no run is queued or going,
// then the lists it changed are read afresh. It lives outside the section on purpose: the user may
// leave Settings while the import runs, and the feed they go to must not keep showing as new what
// the import has just marked watched.
const POLL_MS = 1500;
export const importBusy = (s: TakeoutStatus | undefined): boolean =>
!!s && (s.queued || s.run?.status === "running");
let watching = false;
const pause = () => new Promise<void>((done) => setTimeout(done, POLL_MS));
async function read(qc: QueryClient): Promise<TakeoutStatus | undefined> {
try {
return await qc.fetchQuery({
queryKey: qk.takeout(),
queryFn: api.takeoutStatus,
staleTime: 0,
});
} catch {
// A read that failed is the section's to say (its query holds the error); the watch asks again.
return qc.getQueryData<TakeoutStatus>(qk.takeout());
}
}
/** Mark stale everything an import may have changed: the feed and its counts, Watch later, and the
* playlists that show its size. */
export function invalidateImported(qc: QueryClient): void {
for (const queryKey of [
qk.feed(),
qk.feedSummary(),
qk.feedYears(),
qk.playlist(),
qk.playlists(),
])
void qc.invalidateQueries({ queryKey });
}
/** Watch the user's import until it is done (once: a second call while it watches is that watch),
* then mark stale what it changed. */
export async function watchTakeout(qc: QueryClient): Promise<void> {
if (watching) return;
watching = true;
let status = qc.getQueryData<TakeoutStatus>(qk.takeout());
while (importBusy(status)) {
await pause();
status = await read(qc);
}
watching = false;
invalidateImported(qc);
}