fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { RefreshCw, UploadCloud } from "lucide-react";
|
||||
@@ -6,19 +6,19 @@ import { Section } from "../../components/ui/form";
|
||||
import { Button } from "../../components/ui/Button";
|
||||
import { DropZone } from "../../components/ui/DropZone";
|
||||
import { StateMessage } from "../../components/ui/QueryState";
|
||||
import { api, type TakeoutStatus } from "../../lib/api";
|
||||
import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api";
|
||||
import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { notify } from "../../lib/notifications";
|
||||
import { notifyActionError } from "../../lib/actionErrors";
|
||||
import { formatDateTime, formatNumber } from "../../lib/format";
|
||||
import { formatBytes, formatDateTime, formatNumber } from "../../lib/format";
|
||||
import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch";
|
||||
|
||||
// Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and
|
||||
// their YouTube Watch later list reach no API, so the export's two files are uploaded here, one
|
||||
// request each; the server tells them apart and runs the import in the background, and its outcome
|
||||
// also reaches the inbox. The import has no steps to count (the whole history is applied in one
|
||||
// go), so while it runs the section says so and asks again until it is done.
|
||||
|
||||
const POLL_MS = 1500;
|
||||
// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`).
|
||||
|
||||
// The outcome counts, in the order they are said: the history's, then Watch later's. A count the
|
||||
// import did not produce is not said at all.
|
||||
@@ -35,6 +35,12 @@ const WATCH_LATER = [
|
||||
["waiting", "laterWaiting"],
|
||||
] as const;
|
||||
|
||||
// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it).
|
||||
type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number];
|
||||
type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0];
|
||||
const everyOutcomeSaid: [Exclude<TakeoutOutcome, Said>] extends [never] ? true : never = true;
|
||||
void everyOutcomeSaid;
|
||||
|
||||
function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] {
|
||||
const parts: string[] = [];
|
||||
for (const [outcome, key] of HISTORY) {
|
||||
@@ -52,58 +58,111 @@ export default function TakeoutImport() {
|
||||
const { t, i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
const qc = useQueryClient();
|
||||
const status = useQuery({
|
||||
queryKey: qk.takeout(),
|
||||
queryFn: api.takeoutStatus,
|
||||
refetchInterval: (q) => (q.state.data?.run?.status === "running" ? POLL_MS : false),
|
||||
});
|
||||
const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus });
|
||||
const [uploading, setUploading] = useState<string | null>(null);
|
||||
// "New import" reopens the drop zone over a finished one; before the first import it is open.
|
||||
const [reopened, setReopened] = useState(false);
|
||||
const run = status.data?.run ?? null;
|
||||
const running = uploading !== null || run?.status === "running";
|
||||
// Queued counts too: until its run row is written, `run` is still the import before it.
|
||||
const busy = importBusy(status.data);
|
||||
const running = uploading !== null || busy;
|
||||
const finishedAt = run?.finished_at ?? null;
|
||||
|
||||
// A run that ended has changed what the feed and Watch later show: those lists are read again
|
||||
// when the user next opens them, not 30 s later from the cache.
|
||||
// A run under way is watched to its end, even if the user leaves Settings meanwhile.
|
||||
useEffect(() => {
|
||||
if (!finishedAt) return;
|
||||
void qc.invalidateQueries({ queryKey: qk.feed() });
|
||||
void qc.invalidateQueries({ queryKey: qk.playlist() });
|
||||
if (busy) void watchTakeout(qc);
|
||||
}, [busy, qc]);
|
||||
// One that ended while nothing here watched it (a reload, another tab) has changed what the feed
|
||||
// and Watch later show too: those lists are read again when next opened, not from the cache.
|
||||
useEffect(() => {
|
||||
if (finishedAt) invalidateImported(qc);
|
||||
}, [finishedAt, qc]);
|
||||
|
||||
// A file dropped beside the zone, or while it is not there, is kept from the browser: it would
|
||||
// open the file in the tab, unloading the app and the upload under way. The zone takes its own.
|
||||
useEffect(() => {
|
||||
const missed = (e: DragEvent) => {
|
||||
if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return;
|
||||
e.preventDefault();
|
||||
if (e.type === "dragover") e.dataTransfer.dropEffect = "none";
|
||||
};
|
||||
window.addEventListener("dragover", missed);
|
||||
window.addEventListener("drop", missed);
|
||||
return () => {
|
||||
window.removeEventListener("dragover", missed);
|
||||
window.removeEventListener("drop", missed);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and
|
||||
// the zone, gone once the import it started is done, hands it to "New import".
|
||||
const zone = useRef<HTMLButtonElement | null>(null);
|
||||
const focusNext = useRef<"zone" | "newImport" | null>(null);
|
||||
const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => {
|
||||
if (!el || focusNext.current !== which) return;
|
||||
focusNext.current = null;
|
||||
// Only focus that was lost with the control that held it: never taken from where the user went.
|
||||
if (document.activeElement === null || document.activeElement === document.body) el.focus();
|
||||
};
|
||||
const zoneRef = (el: HTMLButtonElement | null) => {
|
||||
zone.current = el;
|
||||
focusIfNext("zone", el);
|
||||
};
|
||||
const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el);
|
||||
|
||||
const upload = async (files: File[]) => {
|
||||
for (const file of files) await uploadOne(file);
|
||||
if (document.activeElement === zone.current) focusNext.current = "newImport";
|
||||
let taken = false;
|
||||
for (const file of files) if (await uploadOne(file)) taken = true;
|
||||
// The new run read first, then the upload's line let go: the other way round the previous
|
||||
// import's line showed for a moment, as if this upload had done nothing.
|
||||
await qc.invalidateQueries({ queryKey: qk.takeout() });
|
||||
setUploading(null);
|
||||
setReopened(false);
|
||||
// Nothing taken, nothing imported: the zone stays open for the right file.
|
||||
if (taken) setReopened(false);
|
||||
else focusNext.current = null;
|
||||
};
|
||||
const uploadOne = async (file: File) => {
|
||||
setUploading(file.name);
|
||||
try {
|
||||
const res = await api.takeoutUpload(file);
|
||||
/** Send one file; whether the server took it. */
|
||||
const uploadOne = async (file: File): Promise<boolean> => {
|
||||
const max = status.data?.max_bytes ?? Infinity;
|
||||
if (file.size > max) {
|
||||
// Refused here, not after minutes of upload, and with the limit named.
|
||||
notify({
|
||||
level: "success",
|
||||
message: t("settings.takeout.read", {
|
||||
name: file.name,
|
||||
count: res.videos,
|
||||
n: formatNumber(res.videos, lang),
|
||||
}),
|
||||
level: "error",
|
||||
message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
setUploading(file.name);
|
||||
let res: TakeoutUpload | null = null;
|
||||
try {
|
||||
res = await api.takeoutUpload(file);
|
||||
} catch (e) {
|
||||
// A refusal names its reason (not a Takeout file, nothing in it); the next file still goes.
|
||||
notifyActionError(e, t("errors.writeFailed.takeoutUpload"));
|
||||
}
|
||||
if (res === null) return false;
|
||||
notify({
|
||||
level: "success",
|
||||
message: t("settings.takeout.read", {
|
||||
name: file.name,
|
||||
count: res.videos,
|
||||
n: formatNumber(res.videos, lang),
|
||||
}),
|
||||
});
|
||||
return true;
|
||||
};
|
||||
|
||||
const say = (key: string, n: number) =>
|
||||
t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) });
|
||||
const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : "");
|
||||
// Not while the last import is still being asked for: a zone that flashes before the line that
|
||||
// replaces it would offer a first import to someone who has done one.
|
||||
const showZone = status.isSuccess && !running && (run === null || reopened);
|
||||
// replaces it would offer a first import to someone who has done one. While an import runs the
|
||||
// zone stays, unavailable: the control that started it keeps the focus.
|
||||
const showZone = status.isSuccess && (running || run === null || reopened);
|
||||
// A failed re-read keeps the previous answer: it is not shown under the line saying the read
|
||||
// failed, as if it were current — a run's spinner no more than its outcome.
|
||||
const current = !status.isError || uploading !== null;
|
||||
|
||||
return (
|
||||
<Section title={t("settings.takeout.title")}>
|
||||
@@ -117,7 +176,14 @@ export default function TakeoutImport() {
|
||||
|
||||
{showZone && (
|
||||
<>
|
||||
<DropZone className="px-4 py-5" multiple accept=".json,.csv" onFiles={upload}>
|
||||
<DropZone
|
||||
ref={zoneRef}
|
||||
className="px-4 py-5"
|
||||
multiple
|
||||
accept=".json,.csv"
|
||||
disabled={running}
|
||||
onFiles={upload}
|
||||
>
|
||||
{(dragOver) => (
|
||||
<>
|
||||
<UploadCloud className="w-6 h-6 text-muted" />
|
||||
@@ -138,42 +204,49 @@ export default function TakeoutImport() {
|
||||
</>
|
||||
)}
|
||||
|
||||
{running ? (
|
||||
<div
|
||||
className="mt-2 flex items-center gap-2 text-xs text-muted"
|
||||
role="status"
|
||||
data-testid="takeout-running"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
{uploading !== null
|
||||
? t("settings.takeout.uploading", { name: uploading })
|
||||
: t("settings.takeout.running")}
|
||||
<div className="mt-2 flex items-center justify-between gap-3">
|
||||
{/* One live region for the run and its outcome, mounted throughout: what replaces
|
||||
"Processing…" is announced, which a region mounted with its text is not. */}
|
||||
<div role="status" className="min-w-0 text-xs text-muted">
|
||||
{running && current ? (
|
||||
<span className="flex items-center gap-2" data-testid="takeout-running">
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
{uploading !== null
|
||||
? t("settings.takeout.uploading", { name: uploading })
|
||||
: t("settings.takeout.running")}
|
||||
</span>
|
||||
) : (
|
||||
!running &&
|
||||
run &&
|
||||
status.data &&
|
||||
current && (
|
||||
<p data-testid="takeout-last">
|
||||
{run.status === "error"
|
||||
? t("settings.takeout.failed", { when: when(run.finished_at) })
|
||||
: run.status === "interrupted"
|
||||
? t("settings.takeout.interrupted", { when: when(run.started_at) })
|
||||
: [
|
||||
t("settings.takeout.last", { when: when(run.finished_at) }),
|
||||
...countParts(status.data, say),
|
||||
].join(" · ")}
|
||||
</p>
|
||||
)
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
// A failed re-read keeps the previous answer: it is not shown under the line saying the
|
||||
// read failed, as if it were current.
|
||||
run &&
|
||||
status.data &&
|
||||
!status.isError && (
|
||||
<div className="mt-2 flex items-center justify-between gap-3">
|
||||
<p className="text-xs text-muted" data-testid="takeout-last">
|
||||
{run.status === "error"
|
||||
? t("settings.takeout.failed", { when: when(run.finished_at) })
|
||||
: run.status === "interrupted"
|
||||
? t("settings.takeout.interrupted", { when: when(run.started_at) })
|
||||
: [
|
||||
t("settings.takeout.last", { when: when(run.finished_at) }),
|
||||
...countParts(status.data, say),
|
||||
].join(" · ")}
|
||||
</p>
|
||||
{!reopened && (
|
||||
<Button variant="outline" size="sm" onClick={() => setReopened(true)}>
|
||||
{t("settings.takeout.newImport")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
{!running && run && status.data && current && !reopened && (
|
||||
<Button
|
||||
ref={newImportRef}
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
focusNext.current = "zone";
|
||||
setReopened(true);
|
||||
}}
|
||||
>
|
||||
{t("settings.takeout.newImport")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</Section>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user