Files
siftlode/frontend/src/modules/settings/TakeoutImport.tsx
T
peter 9f580fd8c8 fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
2026-10-23 20:48:23 +02:00

253 lines
10 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { useEffect, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { RefreshCw, UploadCloud } from "lucide-react";
import { Section } from "../../components/ui/form";
import { Button } from "../../components/ui/Button";
import { DropZone } from "../../components/ui/DropZone";
import { StateMessage } from "../../components/ui/QueryState";
import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api";
import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed";
import { qk } from "../../lib/queryKeys";
import { notify } from "../../lib/notifications";
import { notifyActionError } from "../../lib/actionErrors";
import { formatBytes, formatDateTime, formatNumber } from "../../lib/format";
import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch";
// Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and
// their YouTube Watch later list reach no API, so the export's two files are uploaded here, one
// request each; the server tells them apart and runs the import in the background, and its outcome
// also reaches the inbox. The import has no steps to count (the whole history is applied in one
// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`).
// The outcome counts, in the order they are said: the history's, then Watch later's. A count the
// import did not produce is not said at all.
const HISTORY = [
["watched", "watched"],
["waiting", "waiting"],
["short", "short"],
["kept", "kept"],
] as const;
const WATCH_LATER = [
["added", "added"],
["present", "present"],
["missing", "missing"],
["waiting", "laterWaiting"],
] as const;
// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it).
type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number];
type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0];
const everyOutcomeSaid: [Exclude<TakeoutOutcome, Said>] extends [never] ? true : never = true;
void everyOutcomeSaid;
function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] {
const parts: string[] = [];
for (const [outcome, key] of HISTORY) {
const n = s.watched?.[outcome] ?? 0;
if (n) parts.push(say(key, n));
}
for (const [outcome, key] of WATCH_LATER) {
const n = s.watch_later?.[outcome] ?? 0;
if (n) parts.push(say(key, n));
}
return parts;
}
export default function TakeoutImport() {
const { t, i18n } = useTranslation();
const lang = i18n.language;
const qc = useQueryClient();
const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus });
const [uploading, setUploading] = useState<string | null>(null);
// "New import" reopens the drop zone over a finished one; before the first import it is open.
const [reopened, setReopened] = useState(false);
const run = status.data?.run ?? null;
// Queued counts too: until its run row is written, `run` is still the import before it.
const busy = importBusy(status.data);
const running = uploading !== null || busy;
const finishedAt = run?.finished_at ?? null;
// A run under way is watched to its end, even if the user leaves Settings meanwhile.
useEffect(() => {
if (busy) void watchTakeout(qc);
}, [busy, qc]);
// One that ended while nothing here watched it (a reload, another tab) has changed what the feed
// and Watch later show too: those lists are read again when next opened, not from the cache.
useEffect(() => {
if (finishedAt) invalidateImported(qc);
}, [finishedAt, qc]);
// A file dropped beside the zone, or while it is not there, is kept from the browser: it would
// open the file in the tab, unloading the app and the upload under way. The zone takes its own.
useEffect(() => {
const missed = (e: DragEvent) => {
if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return;
e.preventDefault();
if (e.type === "dragover") e.dataTransfer.dropEffect = "none";
};
window.addEventListener("dragover", missed);
window.addEventListener("drop", missed);
return () => {
window.removeEventListener("dragover", missed);
window.removeEventListener("drop", missed);
};
}, []);
// Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and
// the zone, gone once the import it started is done, hands it to "New import".
const zone = useRef<HTMLButtonElement | null>(null);
const focusNext = useRef<"zone" | "newImport" | null>(null);
const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => {
if (!el || focusNext.current !== which) return;
focusNext.current = null;
// Only focus that was lost with the control that held it: never taken from where the user went.
if (document.activeElement === null || document.activeElement === document.body) el.focus();
};
const zoneRef = (el: HTMLButtonElement | null) => {
zone.current = el;
focusIfNext("zone", el);
};
const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el);
const upload = async (files: File[]) => {
if (document.activeElement === zone.current) focusNext.current = "newImport";
let taken = false;
for (const file of files) if (await uploadOne(file)) taken = true;
// The new run read first, then the upload's line let go: the other way round the previous
// import's line showed for a moment, as if this upload had done nothing.
await qc.invalidateQueries({ queryKey: qk.takeout() });
setUploading(null);
// Nothing taken, nothing imported: the zone stays open for the right file.
if (taken) setReopened(false);
else focusNext.current = null;
};
/** Send one file; whether the server took it. */
const uploadOne = async (file: File): Promise<boolean> => {
const max = status.data?.max_bytes ?? Infinity;
if (file.size > max) {
// Refused here, not after minutes of upload, and with the limit named.
notify({
level: "error",
message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }),
});
return false;
}
setUploading(file.name);
let res: TakeoutUpload | null = null;
try {
res = await api.takeoutUpload(file);
} catch (e) {
// A refusal names its reason (not a Takeout file, nothing in it); the next file still goes.
notifyActionError(e, t("errors.writeFailed.takeoutUpload"));
}
if (res === null) return false;
notify({
level: "success",
message: t("settings.takeout.read", {
name: file.name,
count: res.videos,
n: formatNumber(res.videos, lang),
}),
});
return true;
};
const say = (key: string, n: number) =>
t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) });
const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : "");
// Not while the last import is still being asked for: a zone that flashes before the line that
// replaces it would offer a first import to someone who has done one. While an import runs the
// zone stays, unavailable: the control that started it keeps the focus.
const showZone = status.isSuccess && (running || run === null || reopened);
// A failed re-read keeps the previous answer: it is not shown under the line saying the read
// failed, as if it were current — a run's spinner no more than its outcome.
const current = !status.isError || uploading !== null;
return (
<Section title={t("settings.takeout.title")}>
<p className="text-xs text-muted leading-relaxed mb-2">{t("settings.takeout.intro")}</p>
{status.isError && (
<StateMessage compact tone="error" onRetry={() => void status.refetch()}>
{t("settings.takeout.loadFailed")}
</StateMessage>
)}
{showZone && (
<>
<DropZone
ref={zoneRef}
className="px-4 py-5"
multiple
accept=".json,.csv"
disabled={running}
onFiles={upload}
>
{(dragOver) => (
<>
<UploadCloud className="w-6 h-6 text-muted" />
<span className="text-sm text-muted">
{dragOver ? t("settings.takeout.dropActive") : t("settings.takeout.drop")}
</span>
</>
)}
</DropZone>
<details className="mt-2">
<summary className="text-xs text-muted cursor-pointer">
{t("settings.takeout.howTo")}
</summary>
<p className="mt-1 text-xs text-muted leading-relaxed">
{t("settings.takeout.howToBody")}
</p>
</details>
</>
)}
<div className="mt-2 flex items-center justify-between gap-3">
{/* One live region for the run and its outcome, mounted throughout: what replaces
"Processing…" is announced, which a region mounted with its text is not. */}
<div role="status" className="min-w-0 text-xs text-muted">
{running && current ? (
<span className="flex items-center gap-2" data-testid="takeout-running">
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
{uploading !== null
? t("settings.takeout.uploading", { name: uploading })
: t("settings.takeout.running")}
</span>
) : (
!running &&
run &&
status.data &&
current && (
<p data-testid="takeout-last">
{run.status === "error"
? t("settings.takeout.failed", { when: when(run.finished_at) })
: run.status === "interrupted"
? t("settings.takeout.interrupted", { when: when(run.started_at) })
: [
t("settings.takeout.last", { when: when(run.finished_at) }),
...countParts(status.data, say),
].join(" · ")}
</p>
)
)}
</div>
{!running && run && status.data && current && !reopened && (
<Button
ref={newImportRef}
variant="outline"
size="sm"
onClick={() => {
focusNext.current = "zone";
setReopened(true);
}}
>
{t("settings.takeout.newImport")}
</Button>
)}
</div>
</Section>
);
}