fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
@@ -56,6 +56,14 @@ const uploaded = (videos: number): TakeoutUpload => ({
|
||||
malformed: 0,
|
||||
});
|
||||
|
||||
// A whole status: nothing queued and room for any file, unless the test says otherwise.
|
||||
const st = (s: Partial<TakeoutStatus>): TakeoutStatus => ({
|
||||
queued: false,
|
||||
max_bytes: 1_000_000,
|
||||
run: null,
|
||||
...s,
|
||||
});
|
||||
|
||||
function mount(status: TakeoutStatus | Error | Promise<never>) {
|
||||
const spy = vi.spyOn(api, "takeoutStatus");
|
||||
if (status instanceof Error) spy.mockRejectedValue(status);
|
||||
@@ -77,11 +85,13 @@ const drop = (files: File[]) =>
|
||||
act(async () => {
|
||||
fireEvent.drop(zone()!, { dataTransfer: { files } });
|
||||
});
|
||||
const file = (name: string) => new File(["[]"], name);
|
||||
const file = (name: string, size = 2) => new File(["x".repeat(size)], name);
|
||||
const unavailable = (el: HTMLElement | null) =>
|
||||
!!el && ((el as HTMLButtonElement).disabled || el.getAttribute("aria-disabled") === "true");
|
||||
|
||||
describe("YouTube history import · what it says", () => {
|
||||
it("before the first import: the drop zone, and no line about an import", async () => {
|
||||
mount({ run: null, watched: {}, watch_later: {} });
|
||||
mount(st({ run: null, watched: {}, watch_later: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
});
|
||||
@@ -92,11 +102,13 @@ describe("YouTube history import · what it says", () => {
|
||||
});
|
||||
|
||||
it("after one: when, then only the counts it produced, the history's first", async () => {
|
||||
mount({
|
||||
run: run("ok"),
|
||||
watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 },
|
||||
watch_later: { added: 41, waiting: 2 },
|
||||
});
|
||||
mount(
|
||||
st({
|
||||
run: run("ok"),
|
||||
watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 },
|
||||
watch_later: { added: 41, waiting: 2 },
|
||||
}),
|
||||
);
|
||||
await waitFor(() => expect(screen.getByTestId("takeout-last")).toBeTruthy());
|
||||
expect(line()).toBe(
|
||||
[
|
||||
@@ -114,20 +126,20 @@ describe("YouTube history import · what it says", () => {
|
||||
});
|
||||
|
||||
it("a failed run and one a restart cut off say so, not counts", async () => {
|
||||
mount({ run: run("error"), watched: { watched: 3 } });
|
||||
mount(st({ run: run("error"), watched: { watched: 3 } }));
|
||||
await waitFor(() => expect(line()).toBe("settings.takeout.failed(at F)"));
|
||||
cleanup();
|
||||
mount({ run: run("interrupted"), watched: { watched: 3 } });
|
||||
mount(st({ run: run("interrupted"), watched: { watched: 3 } }));
|
||||
await waitFor(() => expect(line()).toBe("settings.takeout.interrupted(at S)"));
|
||||
});
|
||||
|
||||
it("a running import: the running line, no drop zone, asked again until it ends", async () => {
|
||||
const spy = mount({ run: run("running"), watched: {} });
|
||||
it("a running import: the running line, the zone unavailable, asked again until it ends", async () => {
|
||||
const spy = mount(st({ run: run("running"), watched: {} }));
|
||||
await waitFor(() =>
|
||||
expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"),
|
||||
);
|
||||
expect(zone()).toBeNull();
|
||||
spy.mockResolvedValue({ run: run("ok"), watched: { watched: 5 } });
|
||||
expect(unavailable(zone())).toBe(true);
|
||||
spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 5 } }));
|
||||
await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(5,5)"), {
|
||||
timeout: 3000,
|
||||
});
|
||||
@@ -143,7 +155,7 @@ describe("YouTube history import · what it says", () => {
|
||||
});
|
||||
|
||||
it("a re-read that fails does not leave the previous answer standing under it", async () => {
|
||||
const status = mount({ run: run("ok", { finished_at: "OLD" }), watched: {} });
|
||||
const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("OLD"));
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
@@ -158,7 +170,7 @@ describe("YouTube history import · what it says", () => {
|
||||
|
||||
describe("YouTube history import · the upload", () => {
|
||||
it("sends each dropped file on its own, says each outcome, and reads the new run", async () => {
|
||||
const status = mount({ run: null, watched: {} });
|
||||
const status = mount(st({ run: null, watched: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
const refused = new HttpError(422, "refused");
|
||||
const sent: string[] = [];
|
||||
@@ -167,7 +179,7 @@ describe("YouTube history import · the upload", () => {
|
||||
if (f.name === "bad.html") throw refused;
|
||||
return uploaded(f.name === "watch-history.json" ? 9034 : 56);
|
||||
});
|
||||
status.mockResolvedValue({ run: run("ok"), watched: { watched: 7 } });
|
||||
status.mockResolvedValue(st({ run: run("ok"), watched: { watched: 7 } }));
|
||||
await drop([file("watch-history.json"), file("bad.html"), file("Watch later-videos.csv")]);
|
||||
|
||||
await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(7,7)"));
|
||||
@@ -183,7 +195,7 @@ describe("YouTube history import · the upload", () => {
|
||||
});
|
||||
|
||||
it("names the file being sent, then never shows the previous import's line before the new one", async () => {
|
||||
const status = mount({ run: run("ok", { finished_at: "OLD" }), watched: {} });
|
||||
const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("OLD"));
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
let finish!: (v: TakeoutUpload) => void;
|
||||
@@ -197,7 +209,115 @@ describe("YouTube history import · the upload", () => {
|
||||
await act(async () => finish(uploaded(1)));
|
||||
// The new run is still being read: the upload's line stays, the old import's does not return.
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
await act(async () => answer({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await act(async () => answer(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })));
|
||||
await waitFor(() => expect(line()).toContain("NEW"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("YouTube history import · while it runs", () => {
|
||||
it("a queued import is this upload's, not the last one: the running line until its run reads", async () => {
|
||||
const spy = mount(st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() =>
|
||||
expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"),
|
||||
);
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
|
||||
});
|
||||
|
||||
it("a re-read that fails mid-run does not leave the spinner claiming progress under it", async () => {
|
||||
const spy = mount(st({ run: run("running"), watched: {} }));
|
||||
await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy());
|
||||
spy.mockRejectedValue(new HttpError(500, "boom"));
|
||||
await waitFor(
|
||||
() => expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"),
|
||||
{ timeout: 3000 },
|
||||
);
|
||||
expect(screen.queryByTestId("takeout-running")).toBeNull();
|
||||
spy.mockResolvedValue(st({ run: run("ok"), watched: {} }));
|
||||
await waitFor(() => expect(screen.queryByRole("alert")).toBeNull(), { timeout: 3000 });
|
||||
});
|
||||
|
||||
it("one live region says the run and then its outcome", async () => {
|
||||
const spy = mount(st({ run: run("running"), watched: {} }));
|
||||
await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy());
|
||||
const region = screen.getByRole("status");
|
||||
spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 2 } }));
|
||||
await waitFor(() => expect(line()).toContain("watched(2,2)"), { timeout: 3000 });
|
||||
expect(screen.getByRole("status")).toBe(region);
|
||||
expect(region.contains(screen.getByTestId("takeout-last"))).toBe(true);
|
||||
});
|
||||
|
||||
it("a file dropped beside the zone is kept from the browser, which would open it", async () => {
|
||||
mount(st({ run: null, watched: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
const missed = (type: string, types: string[]) => {
|
||||
const e = new Event(type, { bubbles: true, cancelable: true });
|
||||
Object.defineProperty(e, "dataTransfer", { value: { types, dropEffect: "copy" } });
|
||||
document.body.dispatchEvent(e);
|
||||
return e;
|
||||
};
|
||||
expect(missed("drop", ["Files"]).defaultPrevented).toBe(true);
|
||||
expect(missed("dragover", ["Files"]).defaultPrevented).toBe(true);
|
||||
// A drag of something else (a link, text) is the page's own business.
|
||||
expect(missed("drop", ["text/plain"]).defaultPrevented).toBe(false);
|
||||
cleanup();
|
||||
expect(missed("drop", ["Files"]).defaultPrevented).toBe(false);
|
||||
});
|
||||
|
||||
it("a file larger than the import takes is refused before it is sent, naming the limit", async () => {
|
||||
mount(st({ max_bytes: 10, run: run("ok"), watched: {} }));
|
||||
await waitFor(() => expect(line()).toBeTruthy());
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
const send = vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
await drop([file("watch-history.json", 11)]);
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(mocks.notify).toHaveBeenCalledWith({
|
||||
level: "error",
|
||||
message: "settings.takeout.tooLarge(watch-history.json,10 B)",
|
||||
});
|
||||
// Nothing was taken: the zone stays open for the right file, and so after a refusal.
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
send.mockRejectedValue(new HttpError(422, "refused"));
|
||||
await drop([file("bad.html")]);
|
||||
await waitFor(() => expect(mocks.notifyActionError).toHaveBeenCalled());
|
||||
expect(zone()).toBeTruthy();
|
||||
});
|
||||
|
||||
it("the focus follows: New import hands it to the zone, the zone back to New import", async () => {
|
||||
const spy = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("OLD"));
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
expect(document.activeElement).toBe(zone());
|
||||
|
||||
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
spy.mockResolvedValue(
|
||||
st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }),
|
||||
);
|
||||
await drop([file("watch-history.json")]);
|
||||
// While it runs the zone stays, unavailable, and keeps the focus.
|
||||
expect(document.activeElement).toBe(zone());
|
||||
expect(unavailable(zone())).toBe(true);
|
||||
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
|
||||
expect(zone()).toBeNull();
|
||||
expect(document.activeElement).toBe(
|
||||
screen.getByRole("button", { name: "settings.takeout.newImport" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("a focus the user moved elsewhere meanwhile is not taken back", async () => {
|
||||
const spy = mount(st({ run: null, watched: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
const elsewhere = document.body.appendChild(document.createElement("button"));
|
||||
zone()!.focus();
|
||||
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
spy.mockResolvedValue(st({ queued: true, run: null, watched: {} }));
|
||||
await drop([file("watch-history.json")]);
|
||||
elsewhere.focus();
|
||||
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
|
||||
expect(document.activeElement).toBe(elsewhere);
|
||||
elsewhere.remove();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { RefreshCw, UploadCloud } from "lucide-react";
|
||||
@@ -6,19 +6,19 @@ import { Section } from "../../components/ui/form";
|
||||
import { Button } from "../../components/ui/Button";
|
||||
import { DropZone } from "../../components/ui/DropZone";
|
||||
import { StateMessage } from "../../components/ui/QueryState";
|
||||
import { api, type TakeoutStatus } from "../../lib/api";
|
||||
import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api";
|
||||
import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { notify } from "../../lib/notifications";
|
||||
import { notifyActionError } from "../../lib/actionErrors";
|
||||
import { formatDateTime, formatNumber } from "../../lib/format";
|
||||
import { formatBytes, formatDateTime, formatNumber } from "../../lib/format";
|
||||
import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch";
|
||||
|
||||
// Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and
|
||||
// their YouTube Watch later list reach no API, so the export's two files are uploaded here, one
|
||||
// request each; the server tells them apart and runs the import in the background, and its outcome
|
||||
// also reaches the inbox. The import has no steps to count (the whole history is applied in one
|
||||
// go), so while it runs the section says so and asks again until it is done.
|
||||
|
||||
const POLL_MS = 1500;
|
||||
// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`).
|
||||
|
||||
// The outcome counts, in the order they are said: the history's, then Watch later's. A count the
|
||||
// import did not produce is not said at all.
|
||||
@@ -35,6 +35,12 @@ const WATCH_LATER = [
|
||||
["waiting", "laterWaiting"],
|
||||
] as const;
|
||||
|
||||
// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it).
|
||||
type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number];
|
||||
type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0];
|
||||
const everyOutcomeSaid: [Exclude<TakeoutOutcome, Said>] extends [never] ? true : never = true;
|
||||
void everyOutcomeSaid;
|
||||
|
||||
function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] {
|
||||
const parts: string[] = [];
|
||||
for (const [outcome, key] of HISTORY) {
|
||||
@@ -52,58 +58,111 @@ export default function TakeoutImport() {
|
||||
const { t, i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
const qc = useQueryClient();
|
||||
const status = useQuery({
|
||||
queryKey: qk.takeout(),
|
||||
queryFn: api.takeoutStatus,
|
||||
refetchInterval: (q) => (q.state.data?.run?.status === "running" ? POLL_MS : false),
|
||||
});
|
||||
const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus });
|
||||
const [uploading, setUploading] = useState<string | null>(null);
|
||||
// "New import" reopens the drop zone over a finished one; before the first import it is open.
|
||||
const [reopened, setReopened] = useState(false);
|
||||
const run = status.data?.run ?? null;
|
||||
const running = uploading !== null || run?.status === "running";
|
||||
// Queued counts too: until its run row is written, `run` is still the import before it.
|
||||
const busy = importBusy(status.data);
|
||||
const running = uploading !== null || busy;
|
||||
const finishedAt = run?.finished_at ?? null;
|
||||
|
||||
// A run that ended has changed what the feed and Watch later show: those lists are read again
|
||||
// when the user next opens them, not 30 s later from the cache.
|
||||
// A run under way is watched to its end, even if the user leaves Settings meanwhile.
|
||||
useEffect(() => {
|
||||
if (!finishedAt) return;
|
||||
void qc.invalidateQueries({ queryKey: qk.feed() });
|
||||
void qc.invalidateQueries({ queryKey: qk.playlist() });
|
||||
if (busy) void watchTakeout(qc);
|
||||
}, [busy, qc]);
|
||||
// One that ended while nothing here watched it (a reload, another tab) has changed what the feed
|
||||
// and Watch later show too: those lists are read again when next opened, not from the cache.
|
||||
useEffect(() => {
|
||||
if (finishedAt) invalidateImported(qc);
|
||||
}, [finishedAt, qc]);
|
||||
|
||||
// A file dropped beside the zone, or while it is not there, is kept from the browser: it would
|
||||
// open the file in the tab, unloading the app and the upload under way. The zone takes its own.
|
||||
useEffect(() => {
|
||||
const missed = (e: DragEvent) => {
|
||||
if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return;
|
||||
e.preventDefault();
|
||||
if (e.type === "dragover") e.dataTransfer.dropEffect = "none";
|
||||
};
|
||||
window.addEventListener("dragover", missed);
|
||||
window.addEventListener("drop", missed);
|
||||
return () => {
|
||||
window.removeEventListener("dragover", missed);
|
||||
window.removeEventListener("drop", missed);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and
|
||||
// the zone, gone once the import it started is done, hands it to "New import".
|
||||
const zone = useRef<HTMLButtonElement | null>(null);
|
||||
const focusNext = useRef<"zone" | "newImport" | null>(null);
|
||||
const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => {
|
||||
if (!el || focusNext.current !== which) return;
|
||||
focusNext.current = null;
|
||||
// Only focus that was lost with the control that held it: never taken from where the user went.
|
||||
if (document.activeElement === null || document.activeElement === document.body) el.focus();
|
||||
};
|
||||
const zoneRef = (el: HTMLButtonElement | null) => {
|
||||
zone.current = el;
|
||||
focusIfNext("zone", el);
|
||||
};
|
||||
const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el);
|
||||
|
||||
const upload = async (files: File[]) => {
|
||||
for (const file of files) await uploadOne(file);
|
||||
if (document.activeElement === zone.current) focusNext.current = "newImport";
|
||||
let taken = false;
|
||||
for (const file of files) if (await uploadOne(file)) taken = true;
|
||||
// The new run read first, then the upload's line let go: the other way round the previous
|
||||
// import's line showed for a moment, as if this upload had done nothing.
|
||||
await qc.invalidateQueries({ queryKey: qk.takeout() });
|
||||
setUploading(null);
|
||||
setReopened(false);
|
||||
// Nothing taken, nothing imported: the zone stays open for the right file.
|
||||
if (taken) setReopened(false);
|
||||
else focusNext.current = null;
|
||||
};
|
||||
const uploadOne = async (file: File) => {
|
||||
setUploading(file.name);
|
||||
try {
|
||||
const res = await api.takeoutUpload(file);
|
||||
/** Send one file; whether the server took it. */
|
||||
const uploadOne = async (file: File): Promise<boolean> => {
|
||||
const max = status.data?.max_bytes ?? Infinity;
|
||||
if (file.size > max) {
|
||||
// Refused here, not after minutes of upload, and with the limit named.
|
||||
notify({
|
||||
level: "success",
|
||||
message: t("settings.takeout.read", {
|
||||
name: file.name,
|
||||
count: res.videos,
|
||||
n: formatNumber(res.videos, lang),
|
||||
}),
|
||||
level: "error",
|
||||
message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
setUploading(file.name);
|
||||
let res: TakeoutUpload | null = null;
|
||||
try {
|
||||
res = await api.takeoutUpload(file);
|
||||
} catch (e) {
|
||||
// A refusal names its reason (not a Takeout file, nothing in it); the next file still goes.
|
||||
notifyActionError(e, t("errors.writeFailed.takeoutUpload"));
|
||||
}
|
||||
if (res === null) return false;
|
||||
notify({
|
||||
level: "success",
|
||||
message: t("settings.takeout.read", {
|
||||
name: file.name,
|
||||
count: res.videos,
|
||||
n: formatNumber(res.videos, lang),
|
||||
}),
|
||||
});
|
||||
return true;
|
||||
};
|
||||
|
||||
const say = (key: string, n: number) =>
|
||||
t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) });
|
||||
const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : "");
|
||||
// Not while the last import is still being asked for: a zone that flashes before the line that
|
||||
// replaces it would offer a first import to someone who has done one.
|
||||
const showZone = status.isSuccess && !running && (run === null || reopened);
|
||||
// replaces it would offer a first import to someone who has done one. While an import runs the
|
||||
// zone stays, unavailable: the control that started it keeps the focus.
|
||||
const showZone = status.isSuccess && (running || run === null || reopened);
|
||||
// A failed re-read keeps the previous answer: it is not shown under the line saying the read
|
||||
// failed, as if it were current — a run's spinner no more than its outcome.
|
||||
const current = !status.isError || uploading !== null;
|
||||
|
||||
return (
|
||||
<Section title={t("settings.takeout.title")}>
|
||||
@@ -117,7 +176,14 @@ export default function TakeoutImport() {
|
||||
|
||||
{showZone && (
|
||||
<>
|
||||
<DropZone className="px-4 py-5" multiple accept=".json,.csv" onFiles={upload}>
|
||||
<DropZone
|
||||
ref={zoneRef}
|
||||
className="px-4 py-5"
|
||||
multiple
|
||||
accept=".json,.csv"
|
||||
disabled={running}
|
||||
onFiles={upload}
|
||||
>
|
||||
{(dragOver) => (
|
||||
<>
|
||||
<UploadCloud className="w-6 h-6 text-muted" />
|
||||
@@ -138,42 +204,49 @@ export default function TakeoutImport() {
|
||||
</>
|
||||
)}
|
||||
|
||||
{running ? (
|
||||
<div
|
||||
className="mt-2 flex items-center gap-2 text-xs text-muted"
|
||||
role="status"
|
||||
data-testid="takeout-running"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
{uploading !== null
|
||||
? t("settings.takeout.uploading", { name: uploading })
|
||||
: t("settings.takeout.running")}
|
||||
<div className="mt-2 flex items-center justify-between gap-3">
|
||||
{/* One live region for the run and its outcome, mounted throughout: what replaces
|
||||
"Processing…" is announced, which a region mounted with its text is not. */}
|
||||
<div role="status" className="min-w-0 text-xs text-muted">
|
||||
{running && current ? (
|
||||
<span className="flex items-center gap-2" data-testid="takeout-running">
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
{uploading !== null
|
||||
? t("settings.takeout.uploading", { name: uploading })
|
||||
: t("settings.takeout.running")}
|
||||
</span>
|
||||
) : (
|
||||
!running &&
|
||||
run &&
|
||||
status.data &&
|
||||
current && (
|
||||
<p data-testid="takeout-last">
|
||||
{run.status === "error"
|
||||
? t("settings.takeout.failed", { when: when(run.finished_at) })
|
||||
: run.status === "interrupted"
|
||||
? t("settings.takeout.interrupted", { when: when(run.started_at) })
|
||||
: [
|
||||
t("settings.takeout.last", { when: when(run.finished_at) }),
|
||||
...countParts(status.data, say),
|
||||
].join(" · ")}
|
||||
</p>
|
||||
)
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
// A failed re-read keeps the previous answer: it is not shown under the line saying the
|
||||
// read failed, as if it were current.
|
||||
run &&
|
||||
status.data &&
|
||||
!status.isError && (
|
||||
<div className="mt-2 flex items-center justify-between gap-3">
|
||||
<p className="text-xs text-muted" data-testid="takeout-last">
|
||||
{run.status === "error"
|
||||
? t("settings.takeout.failed", { when: when(run.finished_at) })
|
||||
: run.status === "interrupted"
|
||||
? t("settings.takeout.interrupted", { when: when(run.started_at) })
|
||||
: [
|
||||
t("settings.takeout.last", { when: when(run.finished_at) }),
|
||||
...countParts(status.data, say),
|
||||
].join(" · ")}
|
||||
</p>
|
||||
{!reopened && (
|
||||
<Button variant="outline" size="sm" onClick={() => setReopened(true)}>
|
||||
{t("settings.takeout.newImport")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
{!running && run && status.data && current && !reopened && (
|
||||
<Button
|
||||
ref={newImportRef}
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
focusNext.current = "zone";
|
||||
setReopened(true);
|
||||
}}
|
||||
>
|
||||
{t("settings.takeout.newImport")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</Section>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// A Takeout import is watched to its end whether or not its section is still on screen, and what it
|
||||
// changed is read afresh then: the feed and its counts, Watch later, and the playlists' sizes.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { QueryClient } from "@tanstack/react-query";
|
||||
import { api, type TakeoutStatus } from "../../lib/api";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { importBusy, watchTakeout } from "./takeoutWatch";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
const st = (s: Partial<TakeoutStatus>): TakeoutStatus => ({
|
||||
queued: false,
|
||||
max_bytes: 1,
|
||||
run: null,
|
||||
...s,
|
||||
});
|
||||
const running = {
|
||||
status: "running",
|
||||
started_at: "S",
|
||||
finished_at: null,
|
||||
result: null,
|
||||
failure: null,
|
||||
};
|
||||
|
||||
describe("the Takeout import's watch", () => {
|
||||
it("a queued import is busy as much as a running one", () => {
|
||||
expect(importBusy(st({ queued: true }))).toBe(true);
|
||||
expect(importBusy(st({ run: { ...running, status: "running" } as TakeoutStatus["run"] }))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(importBusy(st({ run: { ...running, status: "ok" } as TakeoutStatus["run"] }))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(importBusy(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("asks until the import is done, then reads again everything it changed", async () => {
|
||||
vi.useFakeTimers();
|
||||
const qc = new QueryClient();
|
||||
qc.setQueryData(qk.takeout(), st({ queued: true }));
|
||||
const answers = [st({ run: running as TakeoutStatus["run"] }), st({})];
|
||||
const read = vi.spyOn(api, "takeoutStatus").mockImplementation(async () => answers.shift()!);
|
||||
const stale = vi.spyOn(qc, "invalidateQueries");
|
||||
|
||||
const done = watchTakeout(qc);
|
||||
void watchTakeout(qc); // a second call while it watches is the same watch
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
expect(stale).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
await done;
|
||||
|
||||
expect(read).toHaveBeenCalledTimes(2);
|
||||
expect(stale.mock.calls.map(([f]) => f?.queryKey)).toEqual([
|
||||
qk.feed(),
|
||||
qk.feedSummary(),
|
||||
qk.feedYears(),
|
||||
qk.playlist(),
|
||||
qk.playlists(),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { QueryClient } from "@tanstack/react-query";
|
||||
import { api, type TakeoutStatus } from "../../lib/api";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
|
||||
// A Takeout import, watched to its end: its status is asked again until no run is queued or going,
|
||||
// then the lists it changed are read afresh. It lives outside the section on purpose: the user may
|
||||
// leave Settings while the import runs, and the feed they go to must not keep showing as new what
|
||||
// the import has just marked watched.
|
||||
|
||||
const POLL_MS = 1500;
|
||||
|
||||
export const importBusy = (s: TakeoutStatus | undefined): boolean =>
|
||||
!!s && (s.queued || s.run?.status === "running");
|
||||
|
||||
let watching = false;
|
||||
|
||||
const pause = () => new Promise<void>((done) => setTimeout(done, POLL_MS));
|
||||
|
||||
async function read(qc: QueryClient): Promise<TakeoutStatus | undefined> {
|
||||
try {
|
||||
return await qc.fetchQuery({
|
||||
queryKey: qk.takeout(),
|
||||
queryFn: api.takeoutStatus,
|
||||
staleTime: 0,
|
||||
});
|
||||
} catch {
|
||||
// A read that failed is the section's to say (its query holds the error); the watch asks again.
|
||||
return qc.getQueryData<TakeoutStatus>(qk.takeout());
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark stale everything an import may have changed: the feed and its counts, Watch later, and the
|
||||
* playlists that show its size. */
|
||||
export function invalidateImported(qc: QueryClient): void {
|
||||
for (const queryKey of [
|
||||
qk.feed(),
|
||||
qk.feedSummary(),
|
||||
qk.feedYears(),
|
||||
qk.playlist(),
|
||||
qk.playlists(),
|
||||
])
|
||||
void qc.invalidateQueries({ queryKey });
|
||||
}
|
||||
|
||||
/** Watch the user's import until it is done (once: a second call while it watches is that watch),
|
||||
* then mark stale what it changed. */
|
||||
export async function watchTakeout(qc: QueryClient): Promise<void> {
|
||||
if (watching) return;
|
||||
watching = true;
|
||||
let status = qc.getQueryData<TakeoutStatus>(qk.takeout());
|
||||
while (importBusy(status)) {
|
||||
await pause();
|
||||
status = await read(qc);
|
||||
}
|
||||
watching = false;
|
||||
invalidateImported(qc);
|
||||
}
|
||||
Reference in New Issue
Block a user