From b9eb8458b3cee32647aeb147115ba6c51e8086fa Mon Sep 17 00:00:00 2001
From: npeter83
Date: Sun, 11 Oct 2026 00:45:13 +0200
Subject: [PATCH 01/20] test(e2e): a localhost connect the machine dropped is
tried again on the test side
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
On a Windows host a burst of test traffic can drop a loopback connect now and then (every port,
any program — measured on the dev desktop). The browser resends it and only waits; Node (libuv)
sends a loopback SYN once and fails at ~300 ms with "connect ETIMEDOUT". `e2e/retryConnect.ts` wraps
Playwright's request context so that one error is tried again (0.5 / 1 / 2 / 4 / 8 s, the system's
own retry waits) while every other failure goes straight to the test; global.setup's health check,
login and /api/me and speed.measure's share probe go through it. Unit-tested; e2e 61/61.
---
frontend/e2e/README.md | 7 ++++
frontend/e2e/global.setup.ts | 11 +++++--
frontend/e2e/retryConnect.ts | 34 ++++++++++++++++++++
frontend/e2e/speed.measure.ts | 3 +-
frontend/src/lib/e2eRetryConnect.test.ts | 41 ++++++++++++++++++++++++
5 files changed, 92 insertions(+), 4 deletions(-)
create mode 100644 frontend/e2e/retryConnect.ts
create mode 100644 frontend/src/lib/e2eRetryConnect.test.ts
diff --git a/frontend/e2e/README.md b/frontend/e2e/README.md
index fba13bd3..2fa68d23 100644
--- a/frontend/e2e/README.md
+++ b/frontend/e2e/README.md
@@ -17,6 +17,13 @@ siftlode e2e-seed # (re)seed the test account only, no tests
The localdev stack must be up (`siftlode dev`). `global.setup.ts` reseeds the account and logs in
once, sharing the session via `e2e/.auth/state.json` (git-ignored).
+**A dropped localhost connect** — on a Windows host, a burst of test traffic can drop a loopback
+connect now and then (every port, any program). The browser resends it and only waits (a loader
+outlives an `expect` timeout); Node does not, so a request from the test side fails with
+`connect ETIMEDOUT`. Those requests go through `retryConnect` (`e2e/retryConnect.ts`), which tries
+that one error again and passes every other failure straight on. Don't run two e2e suites on the
+same machine at once.
+
**Browsers** — every flow runs in three projects: `chromium` (Desktop Chrome), `webkit` (Desktop
Safari) and `ipad` (iPad gen 7 landscape: 1080×810, touch, mobile). Install the engines once per
machine with `npx playwright install chromium webkit`; run one project with
diff --git a/frontend/e2e/global.setup.ts b/frontend/e2e/global.setup.ts
index 782ca13e..5c706ecf 100644
--- a/frontend/e2e/global.setup.ts
+++ b/frontend/e2e/global.setup.ts
@@ -3,6 +3,7 @@ import { execFileSync } from "node:child_process";
import { mkdirSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
+import { retryConnect } from "./retryConnect";
/**
* One-time setup for the characterization net (the `setup` project the tests depend on):
@@ -32,8 +33,12 @@ setup("seed + authenticate", async ({ page }) => {
"E2E_PASSWORD is not set — run the suite through `siftlode e2e` (it reads the secret store)",
).not.toBe("");
+ // These requests run in Node (sharing the page's cookie jar), so a localhost connect the machine
+ // dropped is tried again (retryConnect).
+ const api = retryConnect(page.request);
+
// 1. Preflight: fail fast (with a useful message) if the stack isn't running.
- const health = await page.request.get("/healthz");
+ const health = await api.get("/healthz");
expect(
health.ok(),
`localdev stack not reachable at ${BASE_URL} — start it with \`siftlode dev\``,
@@ -66,12 +71,12 @@ setup("seed + authenticate", async ({ page }) => {
// 3. Authenticate. page.request shares the browser context's cookie jar, so the `session`
// cookie it receives is captured by storageState below.
- const login = await page.request.post("/auth/password-login", {
+ const login = await api.post("/auth/password-login", {
data: { email: EMAIL, password: PASSWORD },
});
expect(login.ok(), "password-login failed — is the E2E account seeded?").toBeTruthy();
- const me = await (await page.request.get("/api/me")).json();
+ const me = await (await api.get("/api/me")).json();
expect(me.authenticated, "session not established after login").toBeTruthy();
// 4. Pin the UI locale, suppress the first-login onboarding wizard (a full-screen overlay
diff --git a/frontend/e2e/retryConnect.ts b/frontend/e2e/retryConnect.ts
new file mode 100644
index 00000000..78df1e2c
--- /dev/null
+++ b/frontend/e2e/retryConnect.ts
@@ -0,0 +1,34 @@
+// Playwright's request context (it runs in Node), trying a connection again when it never came about —
+// as every TCP stack does. Measured on the Windows dev desktop (2026-10-10, the work-group-timesheet
+// suite): now and then, in bursts of 1–15 s, a connect to localhost is dropped — every port, IPv4 and
+// IPv6, any program, no Docker needed; set off by any big test run on the machine. Every other client
+// sends its SYN again and gets in a moment later, but Node (libuv) sends a loopback SYN only once on
+// Windows and fails at ~300 ms with "connect ETIMEDOUT". The request never left this side, so trying it
+// again is safe: the waits the system's own retries take (0.5, 1, 2, 4, 8 s). Anything else — a refused
+// port, a reset, an answer — goes straight to the test. (The browser resends its own SYN: a page only
+// waits, and nothing here can or should hide that.)
+const METHODS = new Set(["get", "post", "put", "patch", "delete", "head", "fetch"]);
+const NO_CONNECTION = /\bconnect ETIMEDOUT\b/;
+const WAITS_MS = [500, 1000, 2000, 4000, 8000];
+
+/** `ctx` with its request methods trying a connect that timed out again (see above). */
+export function retryConnect(ctx: T): T {
+ return new Proxy(ctx, {
+ get(target, key) {
+ const v: unknown = Reflect.get(target, key, target);
+ if (typeof v !== "function") return v;
+ if (typeof key !== "string" || !METHODS.has(key)) return v.bind(target);
+ return async (...args: unknown[]) => {
+ for (let i = 0; ; i++) {
+ try {
+ return await v.apply(target, args);
+ } catch (e) {
+ const wait = WAITS_MS[i];
+ if (wait === undefined || !NO_CONNECTION.test(String((e as Error)?.message))) throw e;
+ await new Promise((r) => setTimeout(r, wait));
+ }
+ }
+ };
+ },
+ });
+}
diff --git a/frontend/e2e/speed.measure.ts b/frontend/e2e/speed.measure.ts
index 8a661ca8..139bbd25 100644
--- a/frontend/e2e/speed.measure.ts
+++ b/frontend/e2e/speed.measure.ts
@@ -1,6 +1,7 @@
import { test, type Page } from "@playwright/test";
import { execFileSync } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
+import { retryConnect } from "./retryConnect";
/**
* The speed baseline: numbers, not assertions — read by `scripts/audit_metrics.py --speed`, which
@@ -83,7 +84,7 @@ test("speed baseline", async ({ browser, page: page0 }) => {
// the visitor's tap — the page has no autoplay).
let token: string | null = null;
for (const t of shareTokens()) {
- if ((await page0.request.get(`/api/public/watch/${t}`)).ok()) {
+ if ((await retryConnect(page0.request).get(`/api/public/watch/${t}`)).ok()) {
token = t;
break;
}
diff --git a/frontend/src/lib/e2eRetryConnect.test.ts b/frontend/src/lib/e2eRetryConnect.test.ts
new file mode 100644
index 00000000..d87e3aa3
--- /dev/null
+++ b/frontend/src/lib/e2eRetryConnect.test.ts
@@ -0,0 +1,41 @@
+// The e2e suite's request wrapper (e2e/retryConnect.ts): a connect that never came about is tried
+// again, as every TCP stack would; any other failure, or an answer, goes straight to the test.
+import { describe, expect, it, vi } from "vitest";
+import { retryConnect } from "../../e2e/retryConnect";
+
+const noConnection = () => new Error("apiRequestContext.get: connect ETIMEDOUT 127.0.0.1:8080");
+
+describe("retryConnect", () => {
+ it("tries a connect that timed out again, and hands back the answer that came", async () => {
+ vi.useFakeTimers();
+ const get = vi
+ .fn()
+ .mockRejectedValueOnce(noConnection())
+ .mockRejectedValueOnce(noConnection())
+ .mockResolvedValue("ok");
+ const api = retryConnect({ get });
+ const answer = api.get("/healthz");
+ await vi.runAllTimersAsync();
+ await expect(answer).resolves.toBe("ok");
+ expect(get).toHaveBeenCalledTimes(3);
+ expect(get).toHaveBeenLastCalledWith("/healthz");
+ vi.useRealTimers();
+ });
+
+ it("passes any other failure straight on, untried", async () => {
+ const post = vi.fn().mockRejectedValue(new Error("read ECONNRESET"));
+ await expect(retryConnect({ post }).post("/auth/password-login")).rejects.toThrow("ECONNRESET");
+ expect(post).toHaveBeenCalledTimes(1);
+ });
+
+ it("gives up after its last wait, with the connect error", async () => {
+ vi.useFakeTimers();
+ const get = vi.fn().mockRejectedValue(noConnection());
+ const answer = retryConnect({ get }).get("/api/me");
+ const settled = expect(answer).rejects.toThrow("ETIMEDOUT");
+ await vi.runAllTimersAsync();
+ await settled;
+ expect(get).toHaveBeenCalledTimes(6); // the first try and five more
+ vi.useRealTimers();
+ });
+});
From 07e0990ecf0f8b6e7164cc07e20d054638588caf Mon Sep 17 00:00:00 2001
From: npeter83
Date: Sun, 11 Oct 2026 02:59:51 +0200
Subject: [PATCH 02/20] =?UTF-8?q?design(files):=20the=20one-library=20roun?=
=?UTF-8?q?d=20in=20the=20lab=20=E2=80=94=20aligned=20list=20columns,=20th?=
=?UTF-8?q?e=20tray,=20the=20done=20notice?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The Files page replaces the Downloads page: list rows with one column per value kind and a
right-click menu, an app-wide download tray, a done notice that copies the link. Picked
variants are logged in DESIGN.md.
---
docs/DESIGN.md | 4 +
frontend/src/design-lab/DesignLab.tsx | 11 +
frontend/src/design-lab/filesRound.tsx | 730 +++++++++++++++++++++++++
frontend/src/design-lab/labKit.tsx | 77 ++-
4 files changed, 819 insertions(+), 3 deletions(-)
create mode 100644 frontend/src/design-lab/filesRound.tsx
diff --git a/docs/DESIGN.md b/docs/DESIGN.md
index 53680f95..f3588f18 100644
--- a/docs/DESIGN.md
+++ b/docs/DESIGN.md
@@ -774,3 +774,7 @@ of a full-width panel and nothing on the right.
| 2026-10-10 | Clip mode in the players (E38) | Scissors in the control row + `C`, `[` `]`; the panel in the player with the watched video as preview; overview + zoomed strip on `ui/RangeTrack` (ends are sliders); a toast + the Ready notification after Cut | A ⋯ menu entry; permanent In / Out buttons; a big dialog with its own preview (a second Plex transcode / YouTube iframe) | Owner's picks A, P1, a; a film is hours and a clip seconds, so only a zoomed strip gives a hand the precision. The Downloads editor's cuts moved onto the same track (keyboard-steppable now) |
| 2026-10-10 | Crop in clip mode (E38) | A Crop toggle in the clip panel: a frame over the playing picture (`ui/CropFrame`, the editor's crop too), Free / 16:9 / 9:16 / 1:1, Whole picture; maximised YouTube keeps the panel in the bar's place; the wide side panel shuts while clipping | A frame always on the picture in clip mode (K2); a free shape only | Owner's K1 + a: a crop is rare, so it never covers the picture unasked; a phone or square clip needs a held shape. The iframe hides its picture's size: drawn on the 16:9 box, the cut fits it to the real picture |
| 2026-10-10 | Marks on the player chrome | `--media-mark` (amber) and `--media-danger` (light red), the same in both themes; `media-marks` points the focus ring (`--focus`) and the chosen mark (`--chosen`: a track's tick, a tab's underline; the label stays white) at the amber on the player and every chrome surface; `media-scope` (a form) also the errors at the red | The theme's accent ring, chosen text, warn and danger | On the chrome (#171717) a light theme's accent ring (Slate 2.84:1), warn (2.99:1) and danger (2.77:1) all fall under 3:1; the accent as chosen text under 4.5:1 |
+| 2026-10-11 | Files page (one library) | List rows by default, the media grid a toggle remembered per folder; a sidebar of automatic source folders, own folders, Shared with me and the storage meter; the paste-a-link box in the toolbar | A media grid by default | The library holds videos AND plain files: rows read both, show more per screen and scan by size and date |
+| 2026-10-11 | Library list columns | Every kind of value in its own column (name, source, phone-safe, retention, size, added, actions), one grid with a subgrid per row so they line up; an action that does not apply keeps its slot empty; a right-click menu on every row (the ⋯ opens the same menu) | Badges flowing after the title; icons that shift when one is missing | Owner: values of one kind must fall in one vertical line; a row's every action reachable by right click |
+| 2026-10-11 | Running downloads | An app-wide tray, bottom right, shown only while something runs or failed (reason + retry), foldable to a pill | A rail icon with a count and a popover | Downloading became an action you start anywhere; a running one must be visible without going to look |
+| 2026-10-11 | Download done | The notification says whether it plays on a phone and carries Link másolása + Megnyitás (the folder, the item focused); never an automatic jump | A plain "ready" notice; auto-navigation to the item | The daily path is paste → done → copy the link; a jump would pull the user out of what they were doing |
diff --git a/frontend/src/design-lab/DesignLab.tsx b/frontend/src/design-lab/DesignLab.tsx
index 33a3233b..bf846765 100644
--- a/frontend/src/design-lab/DesignLab.tsx
+++ b/frontend/src/design-lab/DesignLab.tsx
@@ -4,6 +4,7 @@ import { BatchA } from "./batchA";
import { BatchB } from "./batchB";
import { BatchC } from "./batchC";
import { BatchD } from "./batchD";
+import { FilesRound } from "./filesRound";
import { Chip } from "../components/ui/Chip";
import { Slider } from "../components/ui/NativeControls";
import { SURFACE } from "../components/ui/surfaces";
@@ -74,6 +75,9 @@ export default function DesignLab() {
/>
+
+
Open · Files, the one library
+
+ The page that replaces Downloads. The recommended variant has the accent edge.
+
+ );
+}
+
+// ── F1: the list and the grid ────────────────────────────────────────────────────────────────────
+
+// One grid for the whole list, each row a subgrid: every kind of value sits in its own column, so the
+// badges, the sizes and the action icons line up down the page. An action that does not apply keeps
+// its slot empty (a file has no "copy the play link").
+const LIST_COLUMNS =
+ "grid grid-cols-[4rem_minmax(0,1fr)_7rem_6.5rem_7.5rem_4.5rem_4.5rem_auto] items-center gap-x-3";
+const ROW = "col-span-full grid grid-cols-subgrid items-center rounded-lg px-2 py-1.5";
+
+function ListHead() {
+ const th = "text-2xs uppercase tracking-wider text-muted";
+ return (
+
+
+ Név
+ Forrás
+ Telefon
+ Megőrzés
+ Méret
+ Hozzáadva
+
+
+ );
+}
diff --git a/frontend/src/design-lab/labKit.tsx b/frontend/src/design-lab/labKit.tsx
index b5332f59..1bad5c48 100644
--- a/frontend/src/design-lab/labKit.tsx
+++ b/frontend/src/design-lab/labKit.tsx
@@ -1,10 +1,11 @@
import { useSyncExternalStore, type ReactNode } from "react";
import clsx from "clsx";
+import { Badge } from "../components/ui/Badge";
import { SURFACE } from "../components/ui/surfaces";
-// Building blocks of the design lab. Every decision is taken, so the lab is the reference half
-// only: each is the chosen rendering with the rule it became. A new design round brings
-// back the side-by-side / pair (as before) for its open questions. Everything
+// Building blocks of the design lab. A taken decision is a : the chosen rendering with the
+// rule it became. An open design round shows its questions as s with their s side
+// by side, until the owner picks. Everything
// is static markup on purpose: a rendering that only exists here must not need app state.
/** A decision already taken: the chosen rendering only, with the rule it became. The lab's
@@ -36,6 +37,76 @@ export function Reference({
);
}
+/** An open question of a design round: what the app does today, then the variants side by side. */
+export function Decision({
+ id,
+ code,
+ title,
+ question,
+ now,
+ children,
+}: {
+ id: string;
+ code: string;
+ title: string;
+ question: string;
+ /** What the app does today. */
+ now: ReactNode;
+ children: ReactNode;
+}) {
+ return (
+
+
+
+ {code}
+
{title}
+
+
{question}
+
+ Today:
+ {now}
+
+
+
{children}
+
+ );
+}
+
+/** One answer to a Decision; the recommended one carries the accent edge. */
+export function Variant({
+ letter,
+ name,
+ recommended = false,
+ why,
+ children,
+}: {
+ letter: string;
+ name: string;
+ recommended?: boolean;
+ why: string;
+ children: ReactNode;
+}) {
+ return (
+
+
+
+ {letter}
+
+
{name}
+ {recommended && Recommended}
+
+
{children}
+
{why}
+
+ );
+}
+
/** A small caption above a sample inside a variant. */
export function Sample({ label, children }: { label: string; children: ReactNode }) {
return (
From bf99444ded261d822415f470fb7717c6b6180e6d Mon Sep 17 00:00:00 2001
From: npeter83
Date: Sun, 11 Oct 2026 03:30:34 +0200
Subject: [PATCH 03/20] feat(files): every file sits in a library folder, and
keeps its own expiry
The Files library starts here. A user's every download, clip and upload now
lands in a source folder made per origin (YouTube, Plex clips, uploads, one per
other site), decided once in downloads/folders.py at every place a job is made.
How long a file stays moves from the shared asset to the user's job: stamped
from its folder when it is done, never for a file the user keeps. The GC warns
each user once, then takes the expired file out of their library; the cache cap
never evicts a kept file, and an evicted one leaves every holder's library.
The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume
siftlode_files); docs/self-hosting.md says how to carry an old volume over.
Migration 0102 places every existing job and carries each done file's expiry
over from its asset.
safe_abs_path takes a missing path and answers None, which drops one baselined
type error.
---
.env.example | 6 +-
.gitignore | 4 +-
Dockerfile | 6 +-
backend/.basedpyright/baseline.json | 10 -
backend/alembic/versions/0050_asset_poster.py | 2 +-
backend/alembic/versions/0102_file_folders.py | 193 +++++++++++++
backend/app/config.py | 6 +-
backend/app/downloads/backfill.py | 2 +-
backend/app/downloads/edit.py | 8 +-
backend/app/downloads/folders.py | 139 +++++++++
backend/app/downloads/gc.py | 135 +++++----
backend/app/downloads/jobdb.py | 4 +-
backend/app/downloads/og.py | 2 +-
backend/app/downloads/pipeline.py | 53 ++--
backend/app/downloads/service.py | 23 +-
backend/app/downloads/storage.py | 41 +--
backend/app/downloads/upload.py | 13 +-
backend/app/models.py | 59 +++-
backend/app/plex/cache.py | 2 +-
backend/app/plex/stream.py | 2 +-
backend/app/routes/downloads.py | 15 +-
backend/app/routes/public.py | 6 +-
backend/app/worker.py | 2 +-
backend/tests/test_clip.py | 2 +-
.../tests/test_download_failure_catalog.py | 10 +-
backend/tests/test_download_limits.py | 6 +-
backend/tests/test_edit_cap.py | 2 +-
backend/tests/test_failure_lines.py | 16 +-
backend/tests/test_file_folders.py | 263 ++++++++++++++++++
backend/tests/test_migration_0102.py | 73 +++++
backend/tests/test_odd_frame.py | 2 +-
backend/tests/test_phone_safe_contract.py | 2 +-
backend/tests/test_public_unlock_limit.py | 2 +-
backend/tests/test_spa_entries.py | 2 +-
backend/tests/test_storage.py | 2 +-
backend/tests/test_upload.py | 6 +-
backend/tests/test_upload_conversion.py | 2 +-
backend/tests/test_worker_dispatch.py | 2 +-
docker-compose.home.yml.example | 14 +-
docker-compose.localdev.yml | 6 +-
docker-compose.selfhost.yml | 12 +-
docker-compose.yml | 12 +-
docs/self-hosting.md | 20 +-
.../DownloadCenter.confirm.dom.test.tsx | 3 +
.../src/components/LibraryPlayer.dom.test.tsx | 4 +-
.../src/components/downloadRow.dom.test.tsx | 4 +-
frontend/src/lib/api/downloads.ts | 6 +-
install.ps1 | 2 +-
install.sh | 2 +-
49 files changed, 984 insertions(+), 226 deletions(-)
create mode 100644 backend/alembic/versions/0102_file_folders.py
create mode 100644 backend/app/downloads/folders.py
create mode 100644 backend/tests/test_file_folders.py
create mode 100644 backend/tests/test_migration_0102.py
diff --git a/.env.example b/.env.example
index fd17c2d5..95b3ad55 100644
--- a/.env.example
+++ b/.env.example
@@ -76,10 +76,10 @@ SCHEDULER_ENABLED=true
# --- Download center ---
# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
-# Downloaded media defaults to a Docker-managed named volume. Set DOWNLOAD_HOST_PATH to a host
+# Downloaded media defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 .
-# DOWNLOAD_HOST_PATH=/mnt/media/youtube
+# FILES_HOST_PATH=/mnt/media/youtube
# --- Plex integration (optional) ---
# The optional Plex module browses/plays your Plex library, playing the LOCAL physical file
@@ -89,6 +89,6 @@ SCHEDULER_ENABLED=true
# to /plex-media inside the container (the local side of the admin "path map", whose default is
# /data=/plex-media). Leave unset if you don't use Plex.
# PLEX_MEDIA_HOST_PATH=/mnt/media
-# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to DOWNLOAD_ROOT; point it at
+# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to FILES_ROOT; point it at
# fast local storage if your download volume is slow.
# PLEX_HLS_DIR=/var/tmp/plex-hls
diff --git a/.gitignore b/.gitignore
index 04718ff8..af6667dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,9 +29,9 @@ frontend/e2e/.auth/
*.sql.gz
backups/
-# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root).
+# Download center: locally downloaded media (the FILES_ROOT bind mount at the repo root).
# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package.
-/downloads/
+/files/
# OS / editor
.DS_Store
diff --git a/Dockerfile b/Dockerfile
index 2a31b361..b04e23be 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -39,13 +39,13 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app
# Create the app user BEFORE the copies so files land already-owned via COPY --chown (a trailing
-# `chown -R /app` would rewrite every file into a duplicate layer). /downloads must be appuser-owned
+# `chown -R /app` would rewrite every file into a duplicate layer). /files must be appuser-owned
# so a fresh named volume mounted there inherits that ownership (Docker copies the image dir's
# ownership into a new empty volume) — the worker/API can then write without a manual chown. A bind
# mount to a host path instead needs that path writable by this uid (see docs/self-hosting.md).
RUN adduser --disabled-password --gecos "" appuser \
- && mkdir -p /downloads \
- && chown appuser /app /downloads
+ && mkdir -p /files \
+ && chown appuser /app /files
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode.
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json
index cc5f74f1..8f2e3b98 100644
--- a/backend/.basedpyright/baseline.json
+++ b/backend/.basedpyright/baseline.json
@@ -1196,16 +1196,6 @@
}
}
],
- "./app/routes/public.py": [
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 57,
- "endColumn": 71,
- "lineCount": 1
- }
- }
- ],
"./app/routes/scheduler.py": [
{
"code": "reportArgumentType",
diff --git a/backend/alembic/versions/0050_asset_poster.py b/backend/alembic/versions/0050_asset_poster.py
index 7c210b7b..3059ea2b 100644
--- a/backend/alembic/versions/0050_asset_poster.py
+++ b/backend/alembic/versions/0050_asset_poster.py
@@ -6,7 +6,7 @@ Create Date: 2026-07-07
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
-saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
+saves it as a poster sidecar; `poster_path` (relative to files_root) records it so the card /
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
"""
diff --git a/backend/alembic/versions/0102_file_folders.py b/backend/alembic/versions/0102_file_folders.py
new file mode 100644
index 00000000..30b965a6
--- /dev/null
+++ b/backend/alembic/versions/0102_file_folders.py
@@ -0,0 +1,193 @@
+"""The Files library: folders, and a file's retention on the user's job
+
+Revision ID: 0102_file_folders
+Revises: 0101_plex_text_plain
+Create Date: 2026-10-11
+
+Every download, clip and upload of a user now sits in a folder of their Files library
+(`app.downloads.folders`): a source folder made per origin (YouTube, Plex, uploads, one per other
+site) or one of their own. How long a file stays moves from the shared asset to each user's job
+(`download_jobs.expires_at`, stamped from the folder's `retention_days`), so one user keeping a file
+no longer depends on another's copy; the asset's own `expires_at` / `gc_notified` go.
+
+This places every existing job in its source folder and carries each done job's expiry over from
+the asset it holds (an upload keeps none). The source-key rule is copied here, frozen, from
+`folders.source_key_for` as it is today — a migration must not follow the code's later changes.
+"""
+
+from urllib.parse import urlsplit
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0102_file_folders"
+down_revision: str | None = "0101_plex_text_plain"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
+_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
+_DEFAULT_RETENTION_DAYS = 30 # config.download_retention_days on the day this was written
+
+
+def _site_key(url: str) -> str:
+ host = (urlsplit(url).hostname or "").lower().rstrip(".")
+ for door in _FRONT_DOORS:
+ if host.startswith(door) and host.count(".") > 1:
+ host = host[len(door) :]
+ break
+ if host in _YOUTUBE_HOSTS:
+ return "youtube"
+ return f"site:{host or 'unknown'}"
+
+
+def _asset_key(kind: str | None, ref: str | None) -> str:
+ if kind == "youtube":
+ return "youtube"
+ if kind == "url" and ref:
+ return _site_key(ref)
+ return "upload"
+
+
+def _key(job, jobs_by_id, assets) -> str:
+ kind = job.source_kind
+ if kind == "youtube":
+ return "youtube"
+ if kind == "upload":
+ return "upload"
+ if kind == "clip":
+ spec = job.edit_spec or {}
+ return "plex" if spec.get("source") == "plex" else "youtube"
+ if kind == "url":
+ return _site_key(job.source_ref)
+ if kind == "edit":
+ parent = jobs_by_id.get(job.source_job_id)
+ if parent is not None and parent.id != job.id:
+ return _key(parent, jobs_by_id, assets)
+ asset = assets.get(job.source_asset_id)
+ if asset is not None:
+ return _asset_key(asset.source_kind, asset.source_ref)
+ return "upload"
+
+
+def upgrade() -> None:
+ op.create_table(
+ "file_folders",
+ sa.Column("id", sa.Integer(), primary_key=True),
+ sa.Column(
+ "user_id",
+ sa.Integer(),
+ sa.ForeignKey("users.id", ondelete="CASCADE"),
+ nullable=False,
+ ),
+ sa.Column("kind", sa.String(16), nullable=False),
+ sa.Column("source_key", sa.String(260)), # "site:" + a host (<= 253)
+ sa.Column("name", sa.String(255)),
+ sa.Column("retention_days", sa.Integer()),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ sa.UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
+ sa.CheckConstraint(
+ "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
+ " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
+ name="ck_file_folder_kind",
+ ),
+ )
+ op.create_index("ix_file_folders_user_id", "file_folders", ["user_id"])
+ op.create_index(
+ "uq_file_folder_own_name",
+ "file_folders",
+ ["user_id", sa.text("lower(name)")],
+ unique=True,
+ postgresql_where=sa.text("name IS NOT NULL"),
+ )
+
+ op.add_column(
+ "download_jobs",
+ sa.Column(
+ "folder_id",
+ sa.Integer(),
+ sa.ForeignKey("file_folders.id", ondelete="RESTRICT"),
+ ),
+ )
+ op.add_column(
+ "download_jobs",
+ sa.Column("kept", sa.Boolean(), server_default="false", nullable=False),
+ )
+ op.add_column("download_jobs", sa.Column("expires_at", sa.DateTime(timezone=True)))
+ op.add_column(
+ "download_jobs",
+ sa.Column("expiry_warned", sa.Boolean(), server_default="false", nullable=False),
+ )
+ op.create_index("ix_download_jobs_folder_id", "download_jobs", ["folder_id"])
+ op.create_index("ix_download_jobs_expires_at", "download_jobs", ["expires_at"])
+
+ conn = op.get_bind()
+ raw = conn.execute(
+ sa.text("SELECT value FROM system_config WHERE key = 'download_retention_days'")
+ ).scalar()
+ retention = int(raw) if raw not in (None, "") else _DEFAULT_RETENTION_DAYS
+
+ jobs = conn.execute(
+ sa.text(
+ "SELECT j.id, j.user_id, j.source_kind, j.source_ref, j.edit_spec, j.source_job_id,"
+ " j.source_asset_id, j.status, j.asset_id, a.expires_at AS asset_expires_at"
+ " FROM download_jobs j LEFT JOIN media_assets a ON a.id = j.asset_id"
+ )
+ ).fetchall()
+ jobs_by_id = {j.id: j for j in jobs}
+ assets = {
+ a.id: a
+ for a in conn.execute(sa.text("SELECT id, source_kind, source_ref FROM media_assets"))
+ }
+
+ folder_ids: dict[tuple[int, str], int] = {}
+ for job in jobs:
+ key = _key(job, jobs_by_id, assets)
+ slot = (job.user_id, key)
+ if slot not in folder_ids:
+ folder_ids[slot] = conn.execute(
+ sa.text(
+ "INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
+ " VALUES (:u, 'source', :k, :r) RETURNING id"
+ ),
+ {"u": job.user_id, "k": key, "r": None if key == "upload" else retention},
+ ).scalar_one()
+ # A done file keeps the expiry it had (its asset's); an upload had none and keeps none.
+ expires = job.asset_expires_at if job.status == "done" and key != "upload" else None
+ conn.execute(
+ sa.text("UPDATE download_jobs SET folder_id = :f, expires_at = :e WHERE id = :id"),
+ {"f": folder_ids[slot], "e": expires, "id": job.id},
+ )
+
+ op.drop_column("media_assets", "expires_at")
+ op.drop_column("media_assets", "gc_notified")
+
+
+def downgrade() -> None:
+ op.add_column("media_assets", sa.Column("expires_at", sa.DateTime(timezone=True)))
+ op.add_column(
+ "media_assets",
+ sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False),
+ )
+ # The asset goes when its last holder would: the latest of its holders' expiries (a holder that
+ # never expires keeps the asset unexpired).
+ op.execute(
+ "UPDATE media_assets a SET expires_at = sub.e FROM ("
+ " SELECT asset_id, CASE WHEN bool_or(expires_at IS NULL) THEN NULL ELSE max(expires_at) END"
+ " AS e FROM download_jobs WHERE asset_id IS NOT NULL AND status = 'done' GROUP BY asset_id"
+ ") sub WHERE a.id = sub.asset_id AND a.source_kind <> 'upload'"
+ )
+ op.drop_index("ix_download_jobs_expires_at", table_name="download_jobs")
+ op.drop_index("ix_download_jobs_folder_id", table_name="download_jobs")
+ op.drop_column("download_jobs", "expiry_warned")
+ op.drop_column("download_jobs", "expires_at")
+ op.drop_column("download_jobs", "kept")
+ op.drop_column("download_jobs", "folder_id")
+ op.drop_index("uq_file_folder_own_name", table_name="file_folders")
+ op.drop_index("ix_file_folders_user_id", table_name="file_folders")
+ op.drop_table("file_folders")
diff --git a/backend/app/config.py b/backend/app/config.py
index 8edf93aa..4007159e 100644
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -173,7 +173,7 @@ class Settings(BaseSettings):
# --- Download center (yt-dlp) ---
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
- download_root: str = "/downloads"
+ files_root: str = "/files"
# Whether THIS process runs the download worker loop. On for the dedicated worker
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
worker_enabled: bool = False
@@ -345,8 +345,8 @@ class Settings(BaseSettings):
plex_transcode_crf: int = 20
plex_transcode_max_height: int = 1080
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
- # dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
- # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
+ # dir under files_root. On localdev the files_root is a slow Windows bind-mount, so point
+ # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's files_root is fast
# local disk, so the default is fine there.
plex_hls_dir: str = ""
diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py
index b793a770..13d035ab 100644
--- a/backend/app/downloads/backfill.py
+++ b/backend/app/downloads/backfill.py
@@ -89,7 +89,7 @@ def run() -> dict:
if a.poster_path is None:
# Record a self-hosted poster for every download (the reliable og:image source):
# ensure_poster returns the existing .jpg thumbnail sidecar, or cuts a frame.
- rel = editmod.ensure_poster(a, settings.download_root)
+ rel = editmod.ensure_poster(a, settings.files_root)
if rel:
a.poster_path = rel
filled_poster += 1
diff --git a/backend/app/downloads/edit.py b/backend/app/downloads/edit.py
index 21b82983..2ce485b1 100644
--- a/backend/app/downloads/edit.py
+++ b/backend/app/downloads/edit.py
@@ -732,7 +732,7 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
return dest if dest.exists() else None
-def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
+def ensure_poster(asset, files_root: str, timeout_s: int = 30) -> str | None:
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
`.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
@@ -740,7 +740,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
short clip) dodges black intro frames."""
if not asset.rel_path:
return None
- root = Path(download_root)
+ root = Path(files_root)
src = root / asset.rel_path
if not src.exists():
return None
@@ -756,7 +756,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
return rel if dest.exists() else None
-def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
+def ensure_storyboard(asset, files_root: str, timeout_s: int = 90) -> dict | None:
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow,
@@ -768,7 +768,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
if not asset.rel_path:
return None
geom = storyboard_geometry(asset.duration_s)
- root = Path(download_root)
+ root = Path(files_root)
rel = f".storyboards/asset-{asset.id}.jpg"
dest = root / rel
diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py
new file mode 100644
index 00000000..b962bf65
--- /dev/null
+++ b/backend/app/downloads/folders.py
@@ -0,0 +1,139 @@
+"""The folders of the Files library — where each of a user's files sits.
+
+A folder is a row, never a directory: on disk a file stays where the download layout put it (one
+shared asset can be held by several users), so a rename or a move is a column update. Two kinds:
+
+- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
+ uploads, one per other site). Its `source_key` names the origin; the client renders its name
+ from the key, in the reader's language.
+- an OWN folder is the user's, with the name they gave it.
+
+Every file of a user is in exactly one folder. The folder a new file lands in is decided here
+(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once.
+
+Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
+folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
+done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
+kept it.
+"""
+
+from datetime import datetime, timedelta, timezone
+from enum import StrEnum
+from urllib.parse import urlsplit
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app import sysconfig
+from app.models import DownloadJob, FileFolder, MediaAsset
+
+
+class FolderKind(StrEnum):
+ SOURCE = "source"
+ OWN = "own"
+
+
+class SourceKey(StrEnum):
+ """The fixed origins. Any other site is `site:` (see `site_key`)."""
+
+ YOUTUBE = "youtube"
+ PLEX = "plex"
+ UPLOAD = "upload"
+
+
+SITE_PREFIX = "site:"
+
+# A host's leading label that only says "the same site, another front door".
+_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
+_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
+
+
+def site_key(url: str) -> str:
+ """The source key of a page on some site: `site:` without its front door (www., m.),
+ or the YouTube key for a YouTube address that arrived as a plain URL."""
+ host = (urlsplit(url).hostname or "").lower().rstrip(".")
+ for door in _FRONT_DOORS:
+ if host.startswith(door) and host.count(".") > 1:
+ host = host[len(door) :]
+ break
+ if host in _YOUTUBE_HOSTS:
+ return SourceKey.YOUTUBE
+ return f"{SITE_PREFIX}{host or 'unknown'}"
+
+
+def source_key_for(db: Session, job: DownloadJob) -> str:
+ """The origin a job's file came from — the source folder it belongs in.
+
+ youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL →
+ its site; an edit → wherever its parent's file came from (its parent job, or, once that is
+ deleted, the parent's asset)."""
+ kind = job.source_kind
+ if kind == "youtube":
+ return SourceKey.YOUTUBE
+ if kind == "upload":
+ return SourceKey.UPLOAD
+ if kind == "clip":
+ return (
+ SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
+ )
+ if kind == "url":
+ return site_key(job.source_ref)
+ if kind == "edit":
+ parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
+ if parent is not None and parent.id != job.id:
+ return source_key_for(db, parent)
+ asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
+ if asset is not None:
+ if asset.source_kind == "youtube":
+ return SourceKey.YOUTUBE
+ if asset.source_kind == "upload":
+ return SourceKey.UPLOAD
+ if asset.source_kind == "url":
+ return site_key(asset.source_ref)
+ return SourceKey.UPLOAD
+
+
+def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
+ """The user's source folder for `key`, made on first use. A race on the unique (user, key) is
+ settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
+ stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
+ folder = db.execute(stmt).scalar_one_or_none()
+ if folder is not None:
+ return folder
+ retention = (
+ None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
+ )
+ folder = FileFolder(
+ user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
+ )
+ try:
+ with db.begin_nested():
+ db.add(folder)
+ db.flush()
+ # silent-ok: the expected outcome of the race the savepoint exists for — another request made
+ # the same folder first; re-selecting it is the whole handling.
+ except IntegrityError:
+ folder = db.execute(stmt).scalar_one()
+ return folder
+
+
+def place(db: Session, job: DownloadJob) -> None:
+ """Put a new job's file in its source folder. Called wherever a job is created; the job must
+ carry its `user_id`, `source_kind` and source refs already."""
+ job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
+
+
+def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
+ """When a file that finishes now goes: its folder's retention from now, or never (kept, or a
+ folder that keeps files)."""
+ if kept or folder_id is None:
+ return None
+ days = db.execute(select(FileFolder.retention_days).where(FileFolder.id == folder_id)).scalar()
+ return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)
+
+
+def stamp_expiry(db: Session, job: DownloadJob) -> None:
+ """Stamp a job that just became done (in this session) with its expiry."""
+ job.expires_at = done_expiry(db, job.folder_id, job.kept)
+ job.expiry_warned = False
diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py
index 7a9c2afd..6b22c2ea 100644
--- a/backend/app/downloads/gc.py
+++ b/backend/app/downloads/gc.py
@@ -1,12 +1,13 @@
"""Download retention garbage collector (a scheduler job).
Five passes each run:
- 1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
- once (gc_notified flag) so a download isn't silently deleted.
- 2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
- holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
- 3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
- ready assets (never an upload) until back under it.
+ 1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
+ within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
+ 2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
+ released, so the shared file goes with its last holder.
+ 3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
+ assets (never an upload, never one a user kept) until back under it; each holder's file leaves
+ their library, with a notice.
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
passes 1-3 skip them; benign row-count hygiene).
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
@@ -14,7 +15,7 @@ Five passes each run:
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
so an in-flight download is never touched.
-Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
+Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
"""
import time
@@ -26,7 +27,8 @@ from sqlalchemy.orm import Session
from app import logs, sysconfig
from app.config import settings
-from app.downloads import quota, storage
+from app.downloads import quota, service, storage
+from app.downloads.status import JobStatus
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
@@ -37,42 +39,39 @@ _UPLOAD_TREE = ".uploads"
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
-def _holders(db: Session, asset_id: int) -> list[int]:
- return list(
- db.execute(
- select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
- ).scalars()
+def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
+ title = job.display_name or (asset.title if asset else None) or "download"
+ create_notification(
+ db,
+ user_id=job.user_id,
+ type=ntype,
+ title=title, # English fallback; the client renders a translated message from data
+ data={
+ "kind": ntype,
+ "title": title,
+ "video_id": asset.source_ref if asset else None,
+ "job_id": job.id,
+ },
+ commit=False,
)
-def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
- title = asset.title or "download"
- for uid in _holders(db, asset.id):
- create_notification(
- db,
- user_id=uid,
- type=ntype,
- title=title, # English fallback; the client renders a translated message from data
- data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
- commit=False,
- )
+def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
+ """Take a file out of its user's library: tell them, release the job's hold on the asset (the
+ file goes with the last holder), delete the row (its shares and links cascade)."""
+ _notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
+ service.release_asset(db, job)
+ db.delete(job)
-def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
- if asset.rel_path:
- storage.delete_asset_files(settings.download_root, asset.rel_path)
- _notify_holders(db, asset, ntype)
- db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
-
-
-def _sweep_orphan_files(db: Session, download_root: str) -> int:
- """Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
+def _sweep_orphan_files(db: Session, files_root: str) -> int:
+ """Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
row names at all. Skips the other dot-prefixed system trees and only removes files older than
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
NOT owned by a surviving asset."""
- root = Path(download_root)
+ root = Path(files_root)
if not root.is_dir():
return 0
keep: set[Path] = set()
@@ -151,47 +150,52 @@ def run_download_gc(db: Session) -> dict:
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = reaped = 0
- # 1. Pre-expiry warnings.
+ # 1. Pre-expiry warnings — one per file, the first run inside the grace window.
warn_before = now + timedelta(days=grace_days)
soon = (
db.execute(
- select(MediaAsset).where(
- MediaAsset.status == "ready",
- MediaAsset.gc_notified.is_(False),
- MediaAsset.expires_at.is_not(None),
- MediaAsset.expires_at > now,
- MediaAsset.expires_at <= warn_before,
+ select(DownloadJob).where(
+ DownloadJob.status == JobStatus.DONE,
+ DownloadJob.kept.is_(False),
+ DownloadJob.expiry_warned.is_(False),
+ DownloadJob.expires_at.is_not(None),
+ DownloadJob.expires_at > now,
+ DownloadJob.expires_at <= warn_before,
)
)
.scalars()
.all()
)
- for asset in soon:
- _notify_holders(db, asset, "download_expiring")
- asset.gc_notified = True
+ for job in soon:
+ _notify(
+ db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
+ )
+ job.expiry_warned = True
warned += 1
if soon:
db.commit()
- # 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
+ # 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
dead = (
db.execute(
- select(MediaAsset).where(
- MediaAsset.status.in_(("ready", "error")),
- MediaAsset.expires_at.is_not(None),
- MediaAsset.expires_at <= now,
+ select(DownloadJob).where(
+ DownloadJob.status == JobStatus.DONE,
+ DownloadJob.kept.is_(False),
+ DownloadJob.expires_at.is_not(None),
+ DownloadJob.expires_at <= now,
)
)
.scalars()
.all()
)
- for asset in dead:
- _delete_asset(db, asset, "download_expired")
+ for job in dead:
+ _drop_job(db, job, "download_expired")
expired += 1
if dead:
db.commit()
- # 3. LRU eviction over the total cache cap.
+ # 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
+ # fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = (
@@ -201,12 +205,17 @@ def run_download_gc(db: Session) -> dict:
or 0
)
if total > cap:
- # An upload counts toward the total (it is disk like any other) but is never evicted:
- # it has no source to fetch again, and it lives until its owner deletes it.
+ kept_assets = select(DownloadJob.asset_id).where(
+ DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
+ )
candidates = (
db.execute(
select(MediaAsset)
- .where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
+ .where(
+ MediaAsset.status == "ready",
+ MediaAsset.source_kind != "upload",
+ MediaAsset.id.not_in(kept_assets),
+ )
.order_by(MediaAsset.last_access_at.asc().nulls_first())
)
.scalars()
@@ -216,7 +225,17 @@ def run_download_gc(db: Session) -> dict:
if total <= cap:
break
total -= asset.size_bytes or 0
- _delete_asset(db, asset, "download_evicted")
+ holders = (
+ db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id))
+ .scalars()
+ .all()
+ )
+ for job in holders:
+ _notify(db, job, asset, "download_evicted")
+ db.delete(job)
+ if asset.rel_path:
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
+ db.delete(asset)
evicted += 1
db.commit()
@@ -257,7 +276,7 @@ def run_download_gc(db: Session) -> dict:
)
for asset in unheld:
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
- storage.delete_asset_files(settings.download_root, asset.rel_path)
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
reaped += 1
if unheld:
@@ -265,7 +284,7 @@ def run_download_gc(db: Session) -> dict:
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
- swept = _sweep_orphan_files(db, settings.download_root)
+ swept = _sweep_orphan_files(db, settings.files_root)
result = {
"warned": warned,
diff --git a/backend/app/downloads/jobdb.py b/backend/app/downloads/jobdb.py
index a0935749..a8e95b29 100644
--- a/backend/app/downloads/jobdb.py
+++ b/backend/app/downloads/jobdb.py
@@ -68,8 +68,8 @@ def _reset_asset_pending(asset_id: int) -> None:
def _staging_dir(asset_id: int) -> Path:
- return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
+ return Path(settings.files_root) / ".staging" / f"asset-{asset_id}"
def _edit_staging_dir(asset_id: int) -> Path:
- return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
+ return Path(settings.files_root) / ".staging" / f"edit-{asset_id}"
diff --git a/backend/app/downloads/og.py b/backend/app/downloads/og.py
index 2bc7bf47..99cd2154 100644
--- a/backend/app/downloads/og.py
+++ b/backend/app/downloads/og.py
@@ -135,7 +135,7 @@ def _watch_tags(token: str, db: Session) -> str | None:
if image_dims is None and image.endswith("poster.jpg") and asset.poster_path:
# safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root —
# guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page.
- poster_abs = storage.safe_abs_path(settings.download_root, asset.poster_path)
+ poster_abs = storage.safe_abs_path(settings.files_root, asset.poster_path)
image_dims = _jpeg_size(poster_abs) if poster_abs else None
if image_dims:
tags.append(_tag("og:image:width", str(image_dims[0])))
diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py
index 2b937438..3e792513 100644
--- a/backend/app/downloads/pipeline.py
+++ b/backend/app/downloads/pipeline.py
@@ -3,7 +3,7 @@
import re
import shutil
import time
-from datetime import date, datetime, timedelta, timezone
+from datetime import date, datetime, timezone
from pathlib import Path
import yt_dlp
@@ -434,14 +434,12 @@ def _extract_with_fallback(
raise last_exc # unreachable (loop either returns or raises)
-def _download_settings() -> tuple[str, int]:
- """Layout + retention days from the DB config (admin-overridable on the Configuration page),
- falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
+def _download_layout() -> str:
+ """The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
+ back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
+ a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
with SessionLocal() as db:
- return (
- sysconfig.get_str(db, "download_layout"),
- sysconfig.get_int(db, "download_retention_days"),
- )
+ return sysconfig.get_str(db, "download_layout")
def _ensure_phone_safe(
@@ -588,19 +586,18 @@ def _download(
thumbnail_url=info.get("thumbnail"),
)
- layout, retention_days = _download_settings()
+ layout = _download_layout()
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
thumb = _find_thumbnail(staging, produced)
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
- storage.place_file(produced, settings.download_root, rel)
- nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
+ storage.place_file(produced, settings.files_root, rel)
+ nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
- final = storage.abs_path(settings.download_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
size = final.stat().st_size if final.exists() else None
- expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -626,12 +623,11 @@ def _download(
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
# and can't be relied on cross-origin. write_sidecars already wrote .jpg from
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
- asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
+ asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
asset.source_webpage_url = info.get("webpage_url")
asset.nfo_written = nfo_ok
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
- asset.expires_at = expires
job = db.get(DownloadJob, job_id)
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
owner_id = job.user_id if job else None
@@ -840,7 +836,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
if not src_ready or not src_rel:
_fail_edit(job_id, asset_id, "The source download is no longer available.")
return
- src_path = storage.abs_path(settings.download_root, src_rel)
+ src_path = storage.abs_path(settings.files_root, src_rel)
if not src_path.exists():
_fail_edit(job_id, asset_id, "The source file is missing.")
return
@@ -887,13 +883,11 @@ def _process_edit(job_id: int, asset_id: int) -> None:
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
- storage.place_file(dest_staging, settings.download_root, rel)
- final = storage.abs_path(settings.download_root, rel)
+ storage.place_file(dest_staging, settings.files_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
size = final.stat().st_size if final.exists() else None
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
_, _, out_w, out_h, _ = probe.probe_media(final)
- _, retention_days = _download_settings()
- expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -912,7 +906,6 @@ def _process_edit(job_id: int, asset_id: int) -> None:
asset.width, asset.height = src_w, src_h
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
- asset.expires_at = expires
job = db.get(DownloadJob, job_id)
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
owner_id = job.user_id if job else None
@@ -1086,10 +1079,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
ext = out.suffix.lstrip(".").lower() or "mp4"
rel = storage.edit_rel_path(user_id, asset_id, ext)
- storage.place_file(out, settings.download_root, rel)
- final = storage.abs_path(settings.download_root, rel)
+ storage.place_file(out, settings.files_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
vcodec, acodec, width, height, _ = probe.probe_media(final)
- _, retention_days = _download_settings()
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -1108,10 +1100,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
asset.media_kind = mediakind.derive(vcodec, acodec)
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
# item has no public thumbnail at all.
- asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
+ asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
- asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days)
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
placeholder = service.clip_title(None, None)
@@ -1230,7 +1221,7 @@ def _free_unheld_upload(asset_id: int) -> bool:
if asset is None or asset.ref_count != 0:
return False
if asset.rel_path:
- storage.delete_asset_files(settings.download_root, asset.rel_path)
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
db.commit()
return True
@@ -1253,7 +1244,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
duration = asset.duration_s if asset else None
job = db.get(DownloadJob, job_id)
owner_id = job.user_id if job else None
- src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
+ src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
if old_rel is None or src is None or not src.exists():
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
return
@@ -1267,10 +1258,10 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
new_rel = storage.with_ext(old_rel, old_ext, ext)
if safe != src:
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
- storage.place_file(safe, settings.download_root, new_rel)
+ storage.place_file(safe, settings.files_root, new_rel)
if new_rel != old_rel:
- placed = storage.abs_path(settings.download_root, new_rel)
- final = storage.abs_path(settings.download_root, new_rel)
+ placed = storage.abs_path(settings.files_root, new_rel)
+ final = storage.abs_path(settings.files_root, new_rel)
vcodec, acodec, width, height, dur = probe.probe_media(final)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py
index 96770af0..fd91d10f 100644
--- a/backend/app/downloads/service.py
+++ b/backend/app/downloads/service.py
@@ -19,7 +19,7 @@ from app.downloads import adapters
from app.downloads import clip as clipmod
from app.downloads import credentials
from app.downloads import edit as editmod
-from app.downloads import formats, mediakind, quota, storage
+from app.downloads import folders, formats, mediakind, quota, storage
from app.downloads.status import HOLDING, JobKind, JobStatus
from app.models import (
DISPLAY_TEXT_MAX,
@@ -198,12 +198,14 @@ def enqueue(
queue_pos=next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
incr_ref_count(db, asset.id)
# Cache hit: the file already exists — no worker round-trip needed.
if asset.status == "ready":
job.status = JobStatus.DONE
job.progress = 100
+ folders.stamp_expiry(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
@@ -277,11 +279,13 @@ def enqueue_edit(
queue_pos=next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
incr_ref_count(db, asset.id)
if asset.status == "ready": # identical clip already produced for this user → instant
job.status = JobStatus.DONE
job.progress = 100
+ folders.stamp_expiry(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
@@ -364,11 +368,13 @@ def enqueue_clip(
queue_pos=next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
incr_ref_count(db, asset.id)
if asset.status == "ready": # the same range was already cut for this user → instant
job.status = JobStatus.DONE
job.progress = 100
+ folders.stamp_expiry(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
@@ -392,6 +398,17 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
if not fields:
return False
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
+ if fields.get("status") == JobStatus.DONE:
+ # Done is when a file's retention starts: its folder's, unless the user kept it.
+ held = db.execute(
+ select(DownloadJob.folder_id, DownloadJob.kept).where(DownloadJob.id == job_id)
+ ).one_or_none()
+ if held is not None:
+ fields = {
+ **fields,
+ "expires_at": folders.done_expiry(db, held.folder_id, held.kept),
+ "expiry_warned": False,
+ }
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
@@ -436,7 +453,7 @@ def release_asset(db: Session, job: DownloadJob) -> None:
)
if new_count == 0 and freeable:
if asset.rel_path:
- storage.delete_asset_files(settings.download_root, asset.rel_path)
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
@@ -565,4 +582,6 @@ def redownload(db: Session, job: DownloadJob) -> None:
# name would survive the rebuild).
job.display_name = None
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
+ job.expires_at = None # stamped again when the rebuild is done
+ job.expiry_warned = False
db.commit()
diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py
index 4a3ea17e..d2ec540e 100644
--- a/backend/app/downloads/storage.py
+++ b/backend/app/downloads/storage.py
@@ -122,7 +122,7 @@ def display_filename(name: str) -> str:
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str:
- """Path of the media file relative to DOWNLOAD_ROOT (forward slashes).
+ """Path of the media file relative to FILES_ROOT (forward slashes).
`asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref,
format_sig), so the SAME video downloaded in two formats is two distinct assets — but without
@@ -144,7 +144,7 @@ def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | No
def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
- """On-disk path (relative to DOWNLOAD_ROOT) for a phase-2 editor derivative.
+ """On-disk path (relative to FILES_ROOT) for a phase-2 editor derivative.
Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of
the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster
@@ -161,7 +161,7 @@ def _clean_upload_ext(filename: str) -> str:
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
- """On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
+ """On-disk path (relative to FILES_ROOT) for a locally-uploaded file.
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
@@ -250,24 +250,27 @@ def download_filename(display_name: str, container: str | None, path: Path) -> s
return f"{base}.{ext}"
-def abs_path(download_root: str, rel: str) -> Path:
- return Path(download_root) / rel
+def abs_path(files_root: str, rel: str) -> Path:
+ return Path(files_root) / rel
-def safe_abs_path(download_root: str, rel: str) -> Path | None:
- """The resolved absolute path for `rel`, but ONLY if it stays inside DOWNLOAD_ROOT and exists on
- disk — otherwise None. Centralizes the path-traversal + existence guard every file-serving endpoint
- shares, so the containment check lives in exactly one place."""
- root = Path(download_root).resolve()
- path = abs_path(download_root, rel).resolve()
+def safe_abs_path(files_root: str, rel: str | None) -> Path | None:
+ """The resolved absolute path for `rel`, but ONLY if it stays inside FILES_ROOT and exists on
+ disk — otherwise None (no `rel` at all included: an asset without a file has nothing to serve).
+ Centralizes the path-traversal + existence guard every file-serving endpoint shares, so the
+ containment check lives in exactly one place."""
+ if not rel:
+ return None
+ root = Path(files_root).resolve()
+ path = abs_path(files_root, rel).resolve()
if root not in path.parents or not path.exists():
return None
return path
-def place_file(staging_file: Path, download_root: str, rel: str) -> None:
- """Move a completed staging file into its final location under DOWNLOAD_ROOT."""
- dest = abs_path(download_root, rel)
+def place_file(staging_file: Path, files_root: str, rel: str) -> None:
+ """Move a completed staging file into its final location under FILES_ROOT."""
+ dest = abs_path(files_root, rel)
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(staging_file), str(dest))
@@ -301,9 +304,9 @@ def _sidecar(media: Path, ext: str) -> Path:
return media.with_suffix(ext)
-def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
+def write_sidecars(files_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
"""Write `.nfo` + `.jpg` next to the media file. Returns True if written."""
- media = abs_path(download_root, rel)
+ media = abs_path(files_root, rel)
try:
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
if thumb_src and thumb_src.exists():
@@ -319,9 +322,9 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
return False
-def delete_asset_files(download_root: str, rel: str) -> None:
+def delete_asset_files(files_root: str, rel: str) -> None:
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
- media = abs_path(download_root, rel)
+ media = abs_path(files_root, rel)
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
try:
p.unlink()
@@ -335,7 +338,7 @@ def delete_asset_files(download_root: str, rel: str) -> None:
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
)
# Prune empty Season/Channel dirs up to (but not including) the root.
- root = Path(download_root).resolve()
+ root = Path(files_root).resolve()
parent = media.parent
while parent != root and root in parent.parents:
try:
diff --git a/backend/app/downloads/upload.py b/backend/app/downloads/upload.py
index 7612c4d3..c22ec9c6 100644
--- a/backend/app/downloads/upload.py
+++ b/backend/app/downloads/upload.py
@@ -33,7 +33,7 @@ from sqlalchemy.orm import Session
from app.config import settings
from app.downloads import edit as editmod
-from app.downloads import mediakind, playable, probe, quota, service, storage
+from app.downloads import folders, mediakind, playable, probe, quota, service, storage
from app.downloads.status import JobKind, JobStatus
from app.models import DownloadJob, MediaAsset, display_text
@@ -81,14 +81,14 @@ def ingest_upload(
db.flush() # need asset.id for the on-disk path
rel = storage.upload_rel_path(user_id, asset.id, filename)
- storage.place_file(src_path, settings.download_root, rel)
+ storage.place_file(src_path, settings.files_root, rel)
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
asset_id = asset.id
try:
- final = storage.abs_path(settings.download_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
actual_size = final.stat().st_size if final.exists() else size
asset.status = "pending" if convert else "ready"
@@ -108,7 +108,7 @@ def ingest_upload(
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
# server side even for a codec the browser can't play).
- asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
+ asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
job = DownloadJob(
user_id=user_id,
@@ -123,6 +123,9 @@ def ingest_upload(
queue_pos=service.next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
+ if not convert:
+ folders.stamp_expiry(db, job)
service.incr_ref_count(db, asset.id)
db.commit()
except Exception:
@@ -139,7 +142,7 @@ def ingest_upload(
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
# when none was generated (audio/file, or a video whose poster never started).
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
- storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
+ storage.abs_path(settings.files_root, leaked).unlink(missing_ok=True)
raise
db.refresh(job)
return job
diff --git a/backend/app/models.py b/backend/app/models.py
index ec4c780c..45d57e29 100644
--- a/backend/app/models.py
+++ b/backend/app/models.py
@@ -18,6 +18,7 @@ from sqlalchemy import (
Text,
UniqueConstraint,
func,
+ text,
)
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
@@ -826,12 +827,13 @@ class MediaAsset(Base, TimestampMixin):
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
- `rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
+ `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
system-decided and bound to the source id — never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
- uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
- plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
+ uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
+ the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
+ stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
__tablename__ = "media_assets"
__table_args__ = (
@@ -847,10 +849,10 @@ class MediaAsset(Base, TimestampMixin):
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
- rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
+ rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
- # card / watch page / link preview can still show an image. Relative to download_root.
+ # card / watch page / link preview can still show an image. Relative to files_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
@@ -866,8 +868,6 @@ class MediaAsset(Base, TimestampMixin):
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
media_kind: Mapped[str | None] = mapped_column(String(16))
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
- # Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
- gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
@@ -883,7 +883,6 @@ class MediaAsset(Base, TimestampMixin):
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
- expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class DownloadProfile(Base, TimestampMixin):
@@ -907,6 +906,40 @@ class DownloadProfile(Base, TimestampMixin):
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+class FileFolder(Base, TimestampMixin):
+ """A folder of the Files library (`app.downloads.folders`): a row, never a directory.
+
+ A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a
+ user gets a file from that origin; an OWN folder (`name` set) is one the user made. Every file —
+ a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
+ file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
+
+ __tablename__ = "file_folders"
+ __table_args__ = (
+ UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
+ Index(
+ "uq_file_folder_own_name",
+ "user_id",
+ func.lower(text("name")),
+ unique=True,
+ postgresql_where=text("name IS NOT NULL"),
+ ),
+ CheckConstraint(
+ "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
+ " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
+ name="ck_file_folder_kind",
+ ),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
+ kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
+ # "site:" + a host name (DNS caps one at 253 characters).
+ source_key: Mapped[str | None] = mapped_column(String(260))
+ name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
+ retention_days: Mapped[int | None] = mapped_column(Integer)
+
+
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
@@ -974,6 +1007,16 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+ # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
+ # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
+ # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
+ # never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
+ folder_id: Mapped[int | None] = mapped_column(
+ ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
+ )
+ kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
+ expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+ expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadShare(Base, TimestampMixin):
diff --git a/backend/app/plex/cache.py b/backend/app/plex/cache.py
index b1aec046..1836f713 100644
--- a/backend/app/plex/cache.py
+++ b/backend/app/plex/cache.py
@@ -10,4 +10,4 @@ from pathlib import Path
from app.config import settings
-CACHE_DIR = Path(settings.download_root) / ".plex-img-cache"
+CACHE_DIR = Path(settings.files_root) / ".plex-img-cache"
diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py
index 1039f6af..4616bae0 100644
--- a/backend/app/plex/stream.py
+++ b/backend/app/plex/stream.py
@@ -41,7 +41,7 @@ log = logs.get_logger(__name__)
_HLS_ROOT = (
Path(settings.plex_hls_dir)
if settings.plex_hls_dir
- else Path(settings.download_root) / ".plex-hls"
+ else Path(settings.files_root) / ".plex-hls"
)
_SEG_SECONDS = 6
# Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the
diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py
index 3e3af777..ff9acc32 100644
--- a/backend/app/routes/downloads.py
+++ b/backend/app/routes/downloads.py
@@ -145,6 +145,10 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
"edit_spec": job.edit_spec,
"can_download": ready and job.status == JobStatus.DONE,
"expired": job.asset_id is None and job.status == JobStatus.DONE,
+ # The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
+ "folder_id": job.folder_id,
+ "kept": job.kept,
+ "expires_at": iso(job.expires_at),
"asset": None
if asset is None
else {
@@ -163,7 +167,6 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
"media_kind": asset.media_kind,
"uploader_url": asset.uploader_url,
"thumbnail_url": asset.thumbnail_url,
- "expires_at": iso(asset.expires_at),
},
}
@@ -495,7 +498,7 @@ async def spool_upload(
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
# check a connection straight back out.
release_read_connection(db)
- staging = Path(settings.download_root) / ".staging" / "uploads"
+ staging = Path(settings.files_root) / ".staging" / "uploads"
staging.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(staging))
tmp_path = Path(tmp)
@@ -871,7 +874,7 @@ def download_file(
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
- path = storage.safe_abs_path(settings.download_root, asset.rel_path)
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise not_found("file")
@@ -946,7 +949,7 @@ def storyboard_meta(
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
- meta = editmod.ensure_storyboard(asset, settings.download_root)
+ meta = editmod.ensure_storyboard(asset, settings.files_root)
if meta is None:
return {"available": False}
db.commit() # persist storyboard_path if it was just generated
@@ -967,7 +970,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.poster_path:
raise not_found("poster")
- path = storage.safe_abs_path(settings.download_root, asset.poster_path)
+ path = storage.safe_abs_path(settings.files_root, asset.poster_path)
if path is None:
raise not_found("poster")
return FileResponse(path, media_type="image/jpeg")
@@ -981,7 +984,7 @@ def storyboard_image(
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.storyboard_path:
raise not_found("filmstrip")
- path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
+ path = storage.safe_abs_path(settings.files_root, asset.storyboard_path)
if path is None:
raise not_found("filmstrip")
return FileResponse(path, media_type="image/jpeg")
diff --git a/backend/app/routes/public.py b/backend/app/routes/public.py
index 42099a93..ddb6e8da 100644
--- a/backend/app/routes/public.py
+++ b/backend/app/routes/public.py
@@ -73,7 +73,7 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs
raise refused(404, "video_gone")
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
# loads a player for a missing file — meta and file agree.
- path = storage.safe_abs_path(settings.download_root, asset.rel_path)
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise refused(404, "video_gone")
return job, asset
@@ -158,7 +158,7 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
raise refused(403, "share_password_required")
asset = _ready_asset(db, link)
- path = storage.safe_abs_path(settings.download_root, asset.rel_path)
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise refused(404, "video_gone")
@@ -194,7 +194,7 @@ def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)
job, asset = _ready_target(db, link)
if not asset.poster_path:
raise not_found("poster")
- path = storage.safe_abs_path(settings.download_root, asset.poster_path)
+ path = storage.safe_abs_path(settings.files_root, asset.poster_path)
if path is None:
raise not_found("poster")
return FileResponse(path, media_type="image/jpeg")
diff --git a/backend/app/worker.py b/backend/app/worker.py
index dbff8e35..5f16d46b 100644
--- a/backend/app/worker.py
+++ b/backend/app/worker.py
@@ -715,7 +715,7 @@ def main() -> None:
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
threads: dict[int, threading.Thread] = {}
level = _sync_knobs(threads, None)
- log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.download_root)
+ log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.files_root)
while not _stop.wait(_KNOB_SECONDS):
level = _sync_knobs(threads, level)
log.info("Download worker stopping…")
diff --git a/backend/tests/test_clip.py b/backend/tests/test_clip.py
index 8566ca0d..9778e2cc 100644
--- a/backend/tests/test_clip.py
+++ b/backend/tests/test_clip.py
@@ -395,7 +395,7 @@ def test_a_clip_job_reaches_the_clip_entry(monkeypatch):
def worker(db, tmp_path, monkeypatch):
"""The worker path with its external programs replaced: ffmpeg writes the file it was given, the
probe answers what the test says, the phone-safe step and the poster pass through."""
- monkeypatch.setattr(settings, "download_root", str(tmp_path / "root"))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path / "root"))
seen: dict = {}
def ffmpeg(cmd, *_a, **_k):
diff --git a/backend/tests/test_download_failure_catalog.py b/backend/tests/test_download_failure_catalog.py
index ed585e64..10ea902d 100644
--- a/backend/tests/test_download_failure_catalog.py
+++ b/backend/tests/test_download_failure_catalog.py
@@ -80,7 +80,7 @@ def _asset_and_job(db, user, *, sig, asset_status, job_status, error=None):
def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
cookiefile = tmp_path / "cookies.txt"
cookiefile.write_text("# Netscape HTTP Cookie File\n")
@@ -230,7 +230,7 @@ def test_a_download_yt_dlp_gave_no_duration_stores_the_files_duration(db, monkey
# row lost its trim action. Through the real finalize, with a real 3-second file.
u = _user(db, "fb-duration@example.com")
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
@@ -350,7 +350,7 @@ def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeyp
# file name and the share preview all said "Video". The post's text is in the description.
u = _user(db, "fb-title@example.com")
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
@@ -401,7 +401,7 @@ def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path):
# video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg).
u = _user(db, "fb-mp3@example.com")
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
@@ -430,7 +430,7 @@ def test_a_video_download_is_still_made_browser_playable(db, monkeypatch, tmp_pa
a, j = _asset_and_job(
db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running"
)
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py
index d2dfdc32..5ddbb948 100644
--- a/backend/tests/test_download_limits.py
+++ b/backend/tests/test_download_limits.py
@@ -259,7 +259,7 @@ def test_a_refusal_for_one_user_does_not_poison_the_shared_asset(db, monkeypatch
that shared row — correct for an upstream failure, wrong for "YOUR storage is full", which
`worker._process_job` would then hand verbatim to someone with an empty library.
"""
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
over, other = User(email="over@example.com"), User(email="other@example.com")
db.add_all([over, other])
db.flush()
@@ -306,7 +306,7 @@ def test_a_download_over_the_space_left_says_so_and_releases_the_shared_asset(
# yt-dlp refuses a file over `max_filesize` by saying so and returning with no file (pinned in
# test_formats). That is the owner's storage, not a processing fault: their refusal, in words
# true of a cap with space still left, and the shared asset goes back for the next job.
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
u = User(email="nearcap@example.com")
db.add(u)
db.flush()
@@ -352,7 +352,7 @@ def test_a_download_that_outgrows_the_space_left_unannounced_is_refused_and_not_
# yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's
# fragmented formats never meet it (measured on localdev: 718 MB fetched with 1 MB left). The
# file it produced is held to what is left by its real size, as a clip is.
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
u = User(email="unannounced@example.com")
db.add(u)
db.flush()
diff --git a/backend/tests/test_edit_cap.py b/backend/tests/test_edit_cap.py
index 8f78c28d..fbc52efa 100644
--- a/backend/tests/test_edit_cap.py
+++ b/backend/tests/test_edit_cap.py
@@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg n
def test_an_editor_clip_larger_than_what_is_left_is_refused_and_not_placed(
db, make_user, tmp_path, monkeypatch
):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
src = tmp_path / "in.mp4"
subprocess.run(
[
diff --git a/backend/tests/test_failure_lines.py b/backend/tests/test_failure_lines.py
index 431fc8d8..3ade4eee 100644
--- a/backend/tests/test_failure_lines.py
+++ b/backend/tests/test_failure_lines.py
@@ -258,7 +258,7 @@ def test_an_edit_that_crashes_is_said_once(sink, finalized, monkeypatch, tmp_pat
"""The line `_process_edit`'s handler used to log before `_fail_edit` logged it again."""
(tmp_path / "src.mp4").write_bytes(b"x")
monkeypatch.setattr(pipeline, "SessionLocal", _EditDb)
- monkeypatch.setattr(pipeline.settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(pipeline.settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging")
def odd_spec(*a, **k):
@@ -300,7 +300,7 @@ def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink
RuntimeError("dictionary changed size during iteration"),
RecursionError("maximum recursion depth exceeded"),
NotImplementedError(),
- FileNotFoundError(2, "No such file", "/downloads/Some_Channel/Season_2024/A_Private_Title"),
+ FileNotFoundError(2, "No such file", "/files/Some_Channel/Season_2024/A_Private_Title"),
],
ids=lambda e: type(e).__name__,
)
@@ -499,11 +499,9 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
@pytest.mark.parametrize(
"err",
[
- subprocess.TimeoutExpired(
- ["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30
- ),
- FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"),
- subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]),
+ subprocess.TimeoutExpired(["ffmpeg", "-i", "/files/Some_Channel/A_Private_Title.mp4"], 30),
+ FileNotFoundError(2, "No such file", "/files/Some_Channel/A_Private_Title.jpg"),
+ subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/files/Some_Channel/x.mp4"]),
],
ids=lambda e: type(e).__name__,
)
@@ -523,9 +521,7 @@ def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_
src.write_bytes(b"\xff\xd8\xff rest")
def refuse(a, b):
- raise PermissionError(
- 13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg"
- )
+ raise PermissionError(13, "Permission denied", "/files/Some_Channel/A_Private_Title.jpg")
monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
diff --git a/backend/tests/test_file_folders.py b/backend/tests/test_file_folders.py
new file mode 100644
index 00000000..7bd6681e
--- /dev/null
+++ b/backend/tests/test_file_folders.py
@@ -0,0 +1,263 @@
+"""The Files library's folders and a file's retention: which folder a new file lands in, when a done
+file goes, and the GC that takes it — warning once, never a file its user kept, never a file another
+user still holds.
+
+Pure-logic for the source-key rule; DB-backed for the rest.
+"""
+
+import hashlib
+from datetime import datetime, timedelta, timezone
+
+import pytest
+from sqlalchemy import func, select
+
+from app import sysconfig
+from app.config import settings
+from app.downloads import folders, formats, service
+from app.downloads import gc as gcmod
+from app.downloads.status import JobStatus
+from app.models import DownloadJob, FileFolder, MediaAsset, Notification
+
+NOW = datetime.now(timezone.utc)
+
+
+@pytest.mark.parametrize(
+ ("url", "key"),
+ [
+ ("https://www.tiktok.com/@a/video/1", "site:tiktok.com"),
+ ("https://m.facebook.com/groups/x/posts/2", "site:facebook.com"),
+ ("https://vimeo.com/123", "site:vimeo.com"),
+ ("https://youtu.be/abcdefghijk", "youtube"),
+ ("https://music.youtube.com/watch?v=abcdefghijk", "youtube"),
+ ("https://www.youtube.com/watch?v=abcdefghijk", "youtube"),
+ # A front door is only stripped when a real host remains behind it.
+ ("https://m.co/x", "site:m.co"),
+ ("not a url", "site:unknown"),
+ ],
+)
+def test_a_url_lands_in_its_sites_folder(url, key):
+ assert folders.site_key(url) == key
+
+
+@pytest.fixture
+def _root(tmp_path, monkeypatch):
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
+ return tmp_path
+
+
+def _asset(db, kind="youtube", ref="abcdefghijk", status="ready", rel="a.mp4", size=10):
+ asset = MediaAsset(
+ source_kind=kind,
+ source_ref=ref,
+ format_sig=hashlib.sha256(f"{kind}:{ref}".encode()).hexdigest()[:32],
+ status=status,
+ rel_path=rel,
+ size_bytes=size,
+ ref_count=0,
+ title="A title",
+ )
+ db.add(asset)
+ db.flush()
+ return asset
+
+
+def _job(db, user_id, asset, *, kind="youtube", ref=None, status=JobStatus.DONE, **kw):
+ job = DownloadJob(
+ user_id=user_id,
+ asset_id=asset.id if asset else None,
+ source_kind=kind,
+ source_ref=ref or (asset.source_ref if asset else "x"),
+ profile_snapshot={},
+ status=status,
+ **kw,
+ )
+ db.add(job)
+ db.flush()
+ folders.place(db, job)
+ if asset is not None:
+ service.incr_ref_count(db, asset.id)
+ db.commit()
+ return job
+
+
+def _folder(db, job) -> FileFolder:
+ return db.get(FileFolder, job.folder_id)
+
+
+def test_every_kind_of_job_lands_in_its_source_folder(db, make_user):
+ uid = make_user().id
+ yt = _job(db, uid, _asset(db))
+ site = _job(db, uid, _asset(db, "url", "https://www.tiktok.com/@a/video/1"), kind="url")
+ up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
+ plex = _job(
+ db, uid, _asset(db, "clip", "c1"), kind="clip", edit_spec={"source": "plex", "ref": "9"}
+ )
+ ytclip = _job(
+ db, uid, _asset(db, "clip", "c2"), kind="clip", edit_spec={"source": "youtube", "ref": "v"}
+ )
+ # An edit of a Plex clip: only its parent knows it came from Plex (the clip's asset says "clip").
+ edit = _job(
+ db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=plex.id,
+ source_asset_id=plex.asset_id,
+ ) # fmt: skip
+ keys = {j.id: _folder(db, j).source_key for j in (yt, site, up, plex, ytclip, edit)}
+ assert keys == {
+ yt.id: "youtube",
+ site.id: "site:tiktok.com",
+ up.id: "upload",
+ plex.id: "plex",
+ ytclip.id: "youtube",
+ edit.id: "plex", # an edit sits with the file it was cut from
+ }
+ # One folder per origin per user: the two YouTube files share theirs.
+ assert _folder(db, yt).id == _folder(db, ytclip).id
+
+
+def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
+ uid = make_user().id
+ src = _asset(db, "url", "https://vimeo.com/1")
+ edit = DownloadJob(
+ user_id=uid, source_kind="edit", source_ref="e", profile_snapshot={},
+ source_job_id=None, source_asset_id=src.id, status=JobStatus.DONE,
+ ) # fmt: skip
+ assert folders.source_key_for(db, edit) == "site:vimeo.com"
+
+
+def test_folders_are_per_user(db, make_user):
+ a, b = make_user().id, make_user().id
+ asset = _asset(db)
+ ja, jb = _job(db, a, asset), _job(db, b, asset)
+ assert ja.folder_id != jb.folder_id
+ assert (_folder(db, ja).user_id, _folder(db, jb).user_id) == (a, b)
+
+
+def test_a_source_folder_keeps_files_the_retention_days_and_uploads_forever(db, make_user):
+ sysconfig.set_value(db, "download_retention_days", 12)
+ db.commit()
+ uid = make_user().id
+ yt = _job(db, uid, _asset(db))
+ up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
+ assert _folder(db, yt).retention_days == 12
+ assert _folder(db, up).retention_days is None
+
+
+def test_enqueue_places_the_job_and_a_cache_hit_is_stamped_at_once(db, make_user):
+ uid = make_user().id
+ spec = formats.normalize({})
+ first = service.enqueue(db, uid, "youtube", "abcdefghijk", spec)
+ assert _folder(db, first).source_key == "youtube"
+ assert first.expires_at is None # queued: its retention starts when it is done
+ asset = db.get(MediaAsset, first.asset_id)
+ asset.status, asset.rel_path = "ready", "a.mp4"
+ db.commit()
+ other = make_user().id
+ hit = service.enqueue(db, other, "youtube", "abcdefghijk", spec)
+ assert hit.status == JobStatus.DONE
+ days = _folder(db, hit).retention_days
+ assert hit.expires_at is not None
+ assert abs(hit.expires_at - (NOW + timedelta(days=days))) < timedelta(minutes=5)
+
+
+def test_a_finishing_worker_stamps_the_expiry_unless_kept(db, make_user):
+ uid = make_user().id
+ asset = _asset(db)
+ plain = _job(db, uid, asset, status=JobStatus.RUNNING)
+ kept = _job(db, uid, asset, status=JobStatus.RUNNING, kept=True)
+ for job in (plain, kept):
+ assert service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100)
+ db.commit()
+ db.refresh(plain)
+ db.refresh(kept)
+ assert plain.expires_at is not None
+ assert kept.expires_at is None
+
+
+def _gc(db):
+ return gcmod.run_download_gc(db)
+
+
+def _notices(db, uid, kind):
+ return db.execute(
+ select(func.count())
+ .select_from(Notification)
+ .where(Notification.user_id == uid, Notification.type == kind)
+ ).scalar()
+
+
+def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, _root):
+ (_root / "a.mp4").write_bytes(b"x")
+ uid = make_user().id
+ job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
+ _gc(db)
+ _gc(db)
+ assert _notices(db, uid, "download_expiring") == 1
+ job.expires_at = NOW - timedelta(minutes=1)
+ db.commit()
+ assert _gc(db)["expired"] == 1
+ assert db.get(DownloadJob, job.id) is None
+ assert _notices(db, uid, "download_expired") == 1
+ # Its last holder gone, the shared file and its row go too.
+ assert not (_root / "a.mp4").exists()
+ assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
+
+
+def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
+ (_root / "a.mp4").write_bytes(b"x")
+ uid = make_user().id
+ job = _job(db, uid, _asset(db), kept=True, expires_at=NOW - timedelta(days=1))
+ assert _gc(db)["expired"] == 0
+ assert db.get(DownloadJob, job.id) is not None
+
+
+def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
+ (_root / "a.mp4").write_bytes(b"x")
+ a, b = make_user().id, make_user().id
+ asset = _asset(db)
+ gone = _job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
+ stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
+ _gc(db)
+ assert db.get(DownloadJob, gone.id) is None
+ assert db.get(DownloadJob, stays.id) is not None
+ assert (_root / "a.mp4").exists()
+
+
+def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
+ db, make_user, _root
+):
+ sysconfig.set_value(db, "download_total_max_bytes", 15)
+ db.commit()
+ a, b = make_user().id, make_user().id
+ for name in ("old.mp4", "new.mp4"):
+ (_root / name).write_bytes(b"x")
+ old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
+ new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
+ old.last_access_at = NOW - timedelta(days=9)
+ new.last_access_at = NOW - timedelta(days=1)
+ kept = _job(db, a, old, kept=True) # the older file, but kept: the newer one goes instead
+ ja, jb = _job(db, a, new), _job(db, b, new)
+ assert _gc(db)["evicted"] == 1
+ assert db.get(DownloadJob, kept.id) is not None
+ assert db.get(DownloadJob, ja.id) is None and db.get(DownloadJob, jb.id) is None
+ assert _notices(db, a, "download_evicted") == 1
+ assert _notices(db, b, "download_evicted") == 1
+ assert (_root / "old.mp4").exists() and not (_root / "new.mp4").exists()
+
+
+def test_deleting_a_user_takes_their_folders_and_files_with_them(db, make_user):
+ # Both tables cascade from users; the files' folder FK refuses deleting a folder that still holds
+ # files, which must not stop the account's own deletion.
+ user = make_user()
+ _job(db, user.id, _asset(db))
+ db.delete(user)
+ db.commit()
+ assert db.execute(select(func.count()).select_from(FileFolder)).scalar() == 0
+ assert db.execute(select(func.count()).select_from(DownloadJob)).scalar() == 0
+
+
+def test_the_longest_host_has_room_for_its_folder(db, make_user):
+ label = "a" * 63
+ host = ".".join([label, label, label, "b" * 58]) + ".hu" # 253 characters, the DNS maximum
+ assert len(host) == 253
+ uid = make_user().id
+ job = _job(db, uid, _asset(db, "url", f"https://{host}/v"), kind="url")
+ assert _folder(db, job).source_key == f"site:{host}"
diff --git a/backend/tests/test_migration_0102.py b/backend/tests/test_migration_0102.py
new file mode 100644
index 00000000..384bbcb7
--- /dev/null
+++ b/backend/tests/test_migration_0102.py
@@ -0,0 +1,73 @@
+"""Migration 0102's data step, DB-lane: every existing job lands in its source folder, and a done
+file's expiry moves from the shared asset to the user's job.
+
+The schema half is checked by test_migrations.py; a row placed in the wrong folder is invisible there.
+"""
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import text
+
+from app.db import engine
+
+
+def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user):
+ alice, bob = make_user().id, make_user().id
+ cfg = Config("alembic.ini")
+ command.downgrade(cfg, "0101_plex_text_plain")
+ try:
+ with engine.begin() as conn:
+
+ def asset(kind, ref, expires=None):
+ return conn.execute(
+ text(
+ "INSERT INTO media_assets (source_kind, source_ref, format_sig, status,"
+ " ref_count, expires_at) VALUES (:k, :r, :sig, 'ready', 1, :e)"
+ " RETURNING id"
+ ),
+ {"k": kind, "r": ref, "sig": kind + ref, "e": expires},
+ ).scalar_one()
+
+ def job(user, kind, ref, asset_id, status="done", spec=None, parent=None):
+ return conn.execute(
+ text(
+ "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
+ " profile_snapshot, status, edit_spec, source_job_id)"
+ " VALUES (:u, :a, :k, :r, '{}', :s, CAST(:spec AS json), :p) RETURNING id"
+ ),
+ {"u": user, "a": asset_id, "k": kind, "r": ref, "s": status,
+ "spec": spec, "p": parent},
+ ).scalar_one() # fmt: skip
+
+ yt = asset("youtube", "abcdefghijk", "2030-01-02T00:00:00+00:00")
+ tt = asset("url", "https://www.tiktok.com/@a/video/1", "2030-01-03T00:00:00+00:00")
+ up = asset("upload", "u1")
+ cl = asset("clip", "c1", "2030-01-04T00:00:00+00:00")
+ j_yt = job(alice, "youtube", "abcdefghijk", yt)
+ j_yt_bob = job(bob, "youtube", "abcdefghijk", yt, status="queued")
+ j_tt = job(alice, "url", "https://www.tiktok.com/@a/video/1", tt)
+ j_up = job(alice, "upload", "u1", up)
+ j_cl = job(alice, "clip", "c1", cl, spec='{"source": "plex", "ref": "9"}')
+ j_ed = job(alice, "edit", "e1", cl, parent=j_tt)
+ finally:
+ command.upgrade(cfg, "head")
+
+ with engine.connect() as conn:
+ rows = conn.execute(
+ text(
+ "SELECT j.id, f.user_id, f.source_key, f.retention_days, j.expires_at::date"
+ " FROM download_jobs j JOIN file_folders f ON f.id = j.folder_id"
+ )
+ ).all()
+ folders = conn.execute(text("SELECT count(*) FROM file_folders")).scalar()
+ got = {r[0]: (r[1], r[2], r[3], str(r[4]) if r[4] else None) for r in rows}
+ assert got == {
+ j_yt: (alice, "youtube", 30, "2030-01-02"),
+ j_yt_bob: (bob, "youtube", 30, None), # not done yet: no expiry until it is
+ j_tt: (alice, "site:tiktok.com", 30, "2030-01-03"),
+ j_up: (alice, "upload", None, None), # an upload never expires
+ j_cl: (alice, "plex", 30, "2030-01-04"),
+ j_ed: (alice, "site:tiktok.com", 30, "2030-01-04"),
+ }
+ # alice: youtube, tiktok, upload, plex; bob: youtube.
+ assert folders == 5
diff --git a/backend/tests/test_odd_frame.py b/backend/tests/test_odd_frame.py
index 13f13561..78b1cffe 100644
--- a/backend/tests/test_odd_frame.py
+++ b/backend/tests/test_odd_frame.py
@@ -46,7 +46,7 @@ def test_an_odd_sized_4_4_4_video_converts_to_the_even_frame_below_it(tmp_path):
def test_an_odd_crop_makes_a_clip_and_stores_the_frame_it_has(db, make_user, tmp_path, monkeypatch):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
src = tmp_path / "in.mp4"
_ffmpeg(
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2",
diff --git a/backend/tests/test_phone_safe_contract.py b/backend/tests/test_phone_safe_contract.py
index a15e1476..c40220fe 100644
--- a/backend/tests/test_phone_safe_contract.py
+++ b/backend/tests/test_phone_safe_contract.py
@@ -127,7 +127,7 @@ _FILES = {
def media_root(tmp_path, monkeypatch):
root = tmp_path / "media"
root.mkdir()
- monkeypatch.setattr(settings, "download_root", str(root))
+ monkeypatch.setattr(settings, "files_root", str(root))
return root
diff --git a/backend/tests/test_public_unlock_limit.py b/backend/tests/test_public_unlock_limit.py
index d4fc12d6..f5603cbc 100644
--- a/backend/tests/test_public_unlock_limit.py
+++ b/backend/tests/test_public_unlock_limit.py
@@ -19,7 +19,7 @@ _PW = "open sesame"
@pytest.fixture
def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"}
audio = {"codec_type": "audio", "codec_name": "aac"}
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
diff --git a/backend/tests/test_spa_entries.py b/backend/tests/test_spa_entries.py
index 7fab4d24..95837020 100644
--- a/backend/tests/test_spa_entries.py
+++ b/backend/tests/test_spa_entries.py
@@ -27,7 +27,7 @@ def pages(tmp_path, monkeypatch):
@pytest.fixture
def link_token(as_user, make_user, tmp_path, monkeypatch, upload_headers):
- monkeypatch.setattr(settings, "download_root", str(tmp_path / "media"))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path / "media"))
video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"}
audio = {"codec_type": "audio", "codec_name": "aac"}
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py
index 108c26bc..d679e5ff 100644
--- a/backend/tests/test_storage.py
+++ b/backend/tests/test_storage.py
@@ -100,7 +100,7 @@ class TestDownloadFilename:
class TestSafeAbsPath:
- """The path-traversal guard: only paths that resolve INSIDE download_root AND exist are returned."""
+ """The path-traversal guard: only paths that resolve INSIDE files_root AND exist are returned."""
def test_returns_the_path_for_a_file_inside_the_root(self, tmp_path: Path):
(tmp_path / "sub").mkdir()
diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py
index a240e67f..4d551232 100644
--- a/backend/tests/test_upload.py
+++ b/backend/tests/test_upload.py
@@ -5,7 +5,7 @@ generic file (force-download so uploaded HTML/SVG can't execute inline), quota a
size known up front, the per-file cap enforced while streaming, filename sanitization, and the two
guards that keep an upload out of paths that assume a re-fetchable source (edit / redownload).
-DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is
+DB-backed (real Postgres) — skipped when no database is reachable. `settings.files_root` is
pointed at a per-test tmp dir so ingest's on-disk moves are isolated. The ingest/route tests
monkeypatch the shared `probe.probe_json` so they don't shell out to ffprobe; the classifier's OWN
parsing (image → file, cover-art skip, dims, duration) is covered directly by `TestProbeClassifier`
@@ -51,7 +51,7 @@ def test_derive_media_kind():
@pytest.fixture
def _root(tmp_path, monkeypatch):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
@@ -761,7 +761,7 @@ def test_a_refused_upload_leaves_nothing_in_the_staging_directory(
sysconfig.set_value(db, "download_upload_max_bytes", 8)
db.commit()
- staging = Path(settings.download_root) / ".staging" / "uploads"
+ staging = Path(settings.files_root) / ".staging" / "uploads"
c = as_user(make_user())
r = c.post("/api/downloads/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin"))
diff --git a/backend/tests/test_upload_conversion.py b/backend/tests/test_upload_conversion.py
index 8a8407f0..b4d84521 100644
--- a/backend/tests/test_upload_conversion.py
+++ b/backend/tests/test_upload_conversion.py
@@ -29,7 +29,7 @@ from sqlalchemy.orm import Session
@pytest.fixture
def _root(tmp_path, monkeypatch):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
diff --git a/backend/tests/test_worker_dispatch.py b/backend/tests/test_worker_dispatch.py
index 8dc74b96..2593a306 100644
--- a/backend/tests/test_worker_dispatch.py
+++ b/backend/tests/test_worker_dispatch.py
@@ -32,7 +32,7 @@ def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension(
):
# A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an
# audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file.
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
data = {
"streams": [{"codec_type": "audio", "codec_name": "aac"}],
"format": {"format_name": "mov,mp4", "duration": "10"},
diff --git a/docker-compose.home.yml.example b/docker-compose.home.yml.example
index 69c40b84..fc30bfd8 100644
--- a/docker-compose.home.yml.example
+++ b/docker-compose.home.yml.example
@@ -17,7 +17,7 @@
# TOKEN_ENCRYPTION_KEY (Fernet), GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET,
# OAUTH_REDIRECT_URL=https://siftlode.example.com/auth/callback, ALLOWED_EMAILS, ADMIN_EMAILS,
# YOUTUBE_API_KEY. APP_VERSION is exported by the deploy step (selects the image tag).
-# Optional: DOWNLOAD_HOST_PATH (host dir for the download tree) and PLEX_MEDIA_HOST_PATH (host dir
+# Optional: FILES_HOST_PATH (host dir for the download tree) and PLEX_MEDIA_HOST_PATH (host dir
# holding the Plex library, mounted read-only for the optional Plex module).
name: siftlode
@@ -73,7 +73,7 @@ services:
SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files + builds filmstrips (read-only root is fine —
# it only writes to the /downloads mount + tmpfs). The yt-dlp job loop runs in `worker` below.
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "false"
depends_on:
db:
@@ -84,9 +84,9 @@ services:
# (10.x.x.x:8080) and on the trusted home LAN. Never forwarded to the public internet.
- "8080:8000"
volumes:
- # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env
+ # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Plex media, read-only. Only used when the optional Plex module is enabled: set
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the Plex library (the local side
# of plex_path_map, default /plex-media). Left unset → an empty named volume (harmless; the
@@ -124,7 +124,7 @@ services:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-siftlode}
# The API owns the scheduler; the worker must not also run it.
SCHEDULER_ENABLED: "false"
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "true"
depends_on:
db:
@@ -137,7 +137,7 @@ services:
condition: service_started
networks: [internal]
volumes:
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Plex media (read-only) — same mount as the api; used by Plex clip jobs (a later phase).
- ${PLEX_MEDIA_HOST_PATH:-siftlode_plex_media}:/plex-media:ro
mem_limit: 2g
@@ -167,7 +167,7 @@ services:
volumes:
siftlode_pgdata:
- siftlode_downloads:
+ siftlode_files:
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
siftlode_plex_media:
diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml
index 202ca5b8..c2be617e 100644
--- a/docker-compose.localdev.yml
+++ b/docker-compose.localdev.yml
@@ -21,21 +21,21 @@ services:
environment:
# This instance owns its scheduler (its own DB, so no double-write concern).
SCHEDULER_ENABLED: "true"
- # Plex HLS remux scratch on fast container-local storage (the /downloads bind-mount is a slow
+ # Plex HLS remux scratch on fast container-local storage (the /files bind-mount is a slow
# Windows Docker Desktop mount in dev; prod uses fast local disk so it keeps the default).
PLEX_HLS_DIR: /var/tmp/plex-hls
security_opt:
- apparmor:unconfined
volumes:
# Downloaded media lives on the host so you can inspect the generated Plex tree. Gitignored.
- - ./downloads:/downloads
+ - ./files:/files
- plex_media:/plex-media:ro
worker:
security_opt:
- apparmor:unconfined
volumes:
- - ./downloads:/downloads
+ - ./files:/files
- plex_media:/plex-media:ro
bgutil-pot:
diff --git a/docker-compose.selfhost.yml b/docker-compose.selfhost.yml
index d3087d9d..2095000e 100644
--- a/docker-compose.selfhost.yml
+++ b/docker-compose.selfhost.yml
@@ -49,7 +49,7 @@ services:
SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs
# the yt-dlp job loop. Read-only root is fine — it only writes to the /downloads mount + tmpfs.
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "false"
depends_on:
db:
@@ -60,9 +60,9 @@ services:
# front for HTTPS / public exposure — and set OAUTH_REDIRECT_URL to the https URL.
- "${HTTP_PORT:-8080}:8000"
volumes:
- # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env
+ # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the
# admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty
@@ -98,7 +98,7 @@ services:
environment:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-siftlode}
SCHEDULER_ENABLED: "false"
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "true"
depends_on:
db:
@@ -111,7 +111,7 @@ services:
condition: service_started
networks: [internal]
volumes:
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Same read-only Plex library as the API — the worker resolves library paths too.
- ${PLEX_MEDIA_HOST_PATH:-siftlode_plex_media}:/plex-media:ro
security_opt:
@@ -136,7 +136,7 @@ services:
volumes:
siftlode_pgdata:
- siftlode_downloads:
+ siftlode_files:
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
siftlode_plex_media:
diff --git a/docker-compose.yml b/docker-compose.yml
index fe760858..136fe5ca 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -37,7 +37,7 @@ services:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs
# the yt-dlp job loop.
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "false"
depends_on:
db:
@@ -45,9 +45,9 @@ services:
ports:
- "${APP_PORT:-8080}:8000"
volumes:
- # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env
+ # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
- - ${DOWNLOAD_HOST_PATH:-downloads}:/downloads
+ - ${FILES_HOST_PATH:-files}:/files
# Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the
# admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty
@@ -80,7 +80,7 @@ services:
environment:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
SCHEDULER_ENABLED: "false"
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "true"
depends_on:
db:
@@ -91,7 +91,7 @@ services:
bgutil-pot:
condition: service_started
volumes:
- - ${DOWNLOAD_HOST_PATH:-downloads}:/downloads
+ - ${FILES_HOST_PATH:-files}:/files
# Same read-only Plex library as the API — the worker resolves library paths too.
- ${PLEX_MEDIA_HOST_PATH:-plex_media}:/plex-media:ro
logging: *logging
@@ -107,6 +107,6 @@ services:
volumes:
pgdata:
- downloads:
+ files:
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
plex_media:
diff --git a/docs/self-hosting.md b/docs/self-hosting.md
index b6265c73..1cb68b4d 100644
--- a/docs/self-hosting.md
+++ b/docs/self-hosting.md
@@ -105,12 +105,12 @@ them. It runs two extra containers that come up automatically — a `worker` (th
loop) and a small `bgutil-pot` sidecar (mints YouTube tokens so downloads aren't bot-blocked). No
configuration is required; per-user storage quotas are set on the admin **Downloads → System** page.
-By default the media lives in a Docker-managed volume (`siftlode_downloads`). To keep it somewhere
+By default the media lives in a Docker-managed volume (`siftlode_files`). To keep it somewhere
you can reach from other apps — e.g. a folder your **Plex** server indexes — point it at a host
directory by adding this to `.env` and re-running `up -d`:
```bash
-DOWNLOAD_HOST_PATH=/mnt/media/youtube
+FILES_HOST_PATH=/mnt/media/youtube
```
The directory must be writable by the container user (uid `1000`): `sudo chown -R 1000:1000 `.
@@ -164,8 +164,8 @@ then cannot read its own secrets.
token and every stored refresh token are encrypted with it *inside* the database. Restore a dump
against a different key and those values are lost — you would have to clear the affected rows from
`system_config` and re-enter every secret by hand. Keep `.env` beside the dump, and just as private.
-3. **Downloaded media** *(only if you use the Download Center)* — the `siftlode_downloads` volume, or
- whatever `DOWNLOAD_HOST_PATH` points at. Not needed to bring the instance back; the jobs and their
+3. **Downloaded media** *(only if you use the Download Center)* — the `siftlode_files` volume, or
+ whatever `FILES_HOST_PATH` points at. Not needed to bring the instance back; the jobs and their
metadata are in the database, the files themselves are not.
Dump and restore the database with these two commands. They also exist as scripts
@@ -254,6 +254,18 @@ Then `pull` + `up -d` stays on that version until you change the line. Released
changed in them are listed in the app itself, under **Release Notes** in the account menu, and in
[`CHANGELOG.md`](../CHANGELOG.md).
+**The media mount was renamed** (the Files library): `DOWNLOAD_HOST_PATH` is now `FILES_HOST_PATH`,
+mounted at `/files`, and the default volume is `siftlode_files` instead of `siftlode_downloads`. If you
+set a host directory, rename the variable in `.env`. If you used the default volume, copy it over once
+before you start the new version (Docker cannot rename a volume), then remove the old one when the
+library looks right:
+
+```bash
+docker compose -f docker-compose.selfhost.yml down
+docker volume create siftlode_files
+docker run --rm -v siftlode_downloads:/from -v siftlode_files:/to alpine cp -a /from/. /to/
+```
+
**The registry keeps only the newest few releases, not every version the changelog lists.** A pin is
"decide when you take the next hop", not "stay here forever": once you are several releases behind,
your pinned tag can no longer be pulled — which matters the day you rebuild the host, prune your
diff --git a/frontend/src/components/DownloadCenter.confirm.dom.test.tsx b/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
index 30833de2..f3ca2fc5 100644
--- a/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
+++ b/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
@@ -74,6 +74,9 @@ const job = (id: number, status: DownloadJob["status"]): DownloadJob => ({
extra_links: [],
can_download: status === "done",
expired: false,
+ folder_id: null,
+ kept: false,
+ expires_at: null,
asset: null,
});
diff --git a/frontend/src/components/LibraryPlayer.dom.test.tsx b/frontend/src/components/LibraryPlayer.dom.test.tsx
index 1cae2383..17fbc086 100644
--- a/frontend/src/components/LibraryPlayer.dom.test.tsx
+++ b/frontend/src/components/LibraryPlayer.dom.test.tsx
@@ -54,6 +54,9 @@ const baseJob: DownloadJob = {
extra_links: [],
can_download: true,
expired: false,
+ folder_id: null,
+ kept: false,
+ expires_at: null,
asset: null,
};
@@ -73,7 +76,6 @@ const withAsset = (over: Partial>): DownloadJo
container: null,
media_kind: "video",
thumbnail_url: null,
- expires_at: null,
...over,
},
});
diff --git a/frontend/src/components/downloadRow.dom.test.tsx b/frontend/src/components/downloadRow.dom.test.tsx
index 39df6c0c..1acf0243 100644
--- a/frontend/src/components/downloadRow.dom.test.tsx
+++ b/frontend/src/components/downloadRow.dom.test.tsx
@@ -52,6 +52,9 @@ const baseJob: DownloadJob = {
extra_links: [],
can_download: true,
expired: false,
+ folder_id: null,
+ kept: false,
+ expires_at: null,
asset: null,
};
@@ -74,7 +77,6 @@ const withAsset = (
container,
media_kind: "video",
thumbnail_url: null,
- expires_at: null,
...over,
},
});
diff --git a/frontend/src/lib/api/downloads.ts b/frontend/src/lib/api/downloads.ts
index 805bf043..35cb5607 100644
--- a/frontend/src/lib/api/downloads.ts
+++ b/frontend/src/lib/api/downloads.ts
@@ -40,7 +40,6 @@ interface DownloadAsset {
// while still in flight (set at finalize). See backend downloads/mediakind.py.
media_kind: "video" | "audio" | "file" | null;
thumbnail_url: string | null;
- expires_at: string | null;
}
export type DownloadStatus = "queued" | "running" | "paused" | "done" | "error" | "canceled";
@@ -107,6 +106,11 @@ export interface DownloadJob {
edit_spec?: EditSpec | Omit | null;
can_download: boolean;
expired: boolean;
+ // The Files library (backend downloads/folders): its folder, whether the user keeps it past the
+ // folder's retention, and when it goes (null = never).
+ folder_id: number | null;
+ kept: boolean;
+ expires_at: string | null;
asset: DownloadAsset | null;
shared?: boolean;
user_email?: string;
diff --git a/install.ps1 b/install.ps1
index c3069dee..cf160749 100644
--- a/install.ps1
+++ b/install.ps1
@@ -31,7 +31,7 @@ OAUTH_REDIRECT_URL=$url/auth/callback
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead
# of a Docker volume. Must be writable by uid 1000 (chown -R 1000:1000 ).
-# DOWNLOAD_HOST_PATH=/mnt/media/youtube
+# FILES_HOST_PATH=/mnt/media/youtube
"@ | Set-Content -Path .env -Encoding ascii
Write-Host "Wrote .env (secrets generated)."
}
diff --git a/install.sh b/install.sh
index c6ad65be..b275e9de 100755
--- a/install.sh
+++ b/install.sh
@@ -29,7 +29,7 @@ OAUTH_REDIRECT_URL=$URL/auth/callback
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead
# of a Docker volume. Must be writable by uid 1000 (sudo chown -R 1000:1000 ).
-# DOWNLOAD_HOST_PATH=/mnt/media/youtube
+# FILES_HOST_PATH=/mnt/media/youtube
EOF
chmod 600 .env
echo "Wrote .env (secrets generated)."
From 1a1665b8ec6feedab136f41c07ba93283bd78354 Mon Sep 17 00:00:00 2001
From: npeter83
Date: Sun, 11 Oct 2026 04:42:55 +0200
Subject: [PATCH 04/20] =?UTF-8?q?feat(files):=20the=20Files=20library=20AP?=
=?UTF-8?q?I=20=E2=80=94=20folders,=20moving=20and=20keeping=20files,=20up?=
=?UTF-8?q?load=20into=20a=20folder,=20the=20file's=20hash?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
/api/files is the library's home: list a user's done files (by folder, searched, sorted, or the
ones shared with them), rename / move / keep / delete a file, make, rename and delete own folders
(a deleted folder's files go back to their source folders), and set how long any folder keeps a
file — done files shift by the difference, never closer than the GC's warning window.
An upload can name its folder (an own folder or Uploads), checked before the body is read and again
before the file lands; it is hashed while it streams, and any file can be hashed on first ask. The
download says the hash (Repr-Digest) once it is known; a new file at an asset's path forgets it.
Uploads, deletes, keeps, moves and folder changes are audited (ids only). The fileless "expired"
rows of the old retention are deleted by migration 0103: an expired file now leaves the library.
The shared route helpers moved to routes/files.py; the old /api/downloads library routes delegate
to the new ones until the client moves over.
---
backend/.basedpyright/baseline.json | 40 -
backend/alembic/versions/0103_files_sha256.py | 32 +
backend/app/audit.py | 11 +
backend/app/bodylimit.py | 1 +
backend/app/downloads/folders.py | 90 ++-
backend/app/downloads/upload.py | 19 +-
backend/app/main.py | 2 +
backend/app/models.py | 9 +
backend/app/routes/_common.py | 10 +-
backend/app/routes/downloads.py | 384 ++--------
backend/app/routes/files.py | 691 ++++++++++++++++++
backend/tests/conftest.py | 8 +-
backend/tests/test_body_limit.py | 6 +-
backend/tests/test_download_job_locks.py | 13 +-
backend/tests/test_download_limits.py | 2 +-
backend/tests/test_files_api.py | 432 +++++++++++
backend/tests/test_migration_0103.py | 62 ++
backend/tests/test_route_conventions.py | 4 +-
backend/tests/test_upload.py | 17 +-
frontend/src/i18n/backendIds.contract.test.ts | 7 +-
.../src/i18n/locales/en/auditActions.json | 11 +-
frontend/src/i18n/locales/en/errors.json | 10 +-
.../src/i18n/locales/hu/auditActions.json | 11 +-
frontend/src/i18n/locales/hu/errors.json | 10 +-
24 files changed, 1483 insertions(+), 399 deletions(-)
create mode 100644 backend/alembic/versions/0103_files_sha256.py
create mode 100644 backend/app/routes/files.py
create mode 100644 backend/tests/test_files_api.py
create mode 100644 backend/tests/test_migration_0103.py
diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json
index 8f2e3b98..824be795 100644
--- a/backend/.basedpyright/baseline.json
+++ b/backend/.basedpyright/baseline.json
@@ -727,30 +727,6 @@
}
],
"./app/routes/downloads.py": [
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 29,
- "endColumn": 33,
- "lineCount": 1
- }
- },
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 29,
- "endColumn": 33,
- "lineCount": 1
- }
- },
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 40,
- "endColumn": 50,
- "lineCount": 1
- }
- },
{
"code": "reportOptionalMemberAccess",
"range": {
@@ -790,22 +766,6 @@
"endColumn": 54,
"lineCount": 1
}
- },
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 29,
- "endColumn": 33,
- "lineCount": 1
- }
- },
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 40,
- "endColumn": 50,
- "lineCount": 1
- }
}
],
"./app/routes/playlists.py": [
diff --git a/backend/alembic/versions/0103_files_sha256.py b/backend/alembic/versions/0103_files_sha256.py
new file mode 100644
index 00000000..5e3e8629
--- /dev/null
+++ b/backend/alembic/versions/0103_files_sha256.py
@@ -0,0 +1,32 @@
+"""The Files library: a file's SHA-256, and the expired rows retired
+
+Revision ID: 0103_files_sha256
+Revises: 0102_file_folders
+Create Date: 2026-10-11
+
+`media_assets.sha256` holds the file's hash once it is known (an upload is hashed while it streams
+in, any other file on the first ask), so a machine fetching a file can check what it got.
+
+An expired file now leaves its user's library (`downloads.gc`). Before, an expired download stayed
+as a fileless "expired · download again" row (a done job with no asset). Those rows hold nothing on
+disk and the library no longer shows them, so they are deleted here rather than left behind
+unseen; their shares and public links go with them (ON DELETE CASCADE), and those already served
+nothing. The downgrade cannot bring them back.
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0103_files_sha256"
+down_revision: str | None = "0102_file_folders"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+
+def upgrade() -> None:
+ op.add_column("media_assets", sa.Column("sha256", sa.String(64)))
+ op.execute("DELETE FROM download_jobs WHERE status = 'done' AND asset_id IS NULL")
+
+
+def downgrade() -> None:
+ op.drop_column("media_assets", "sha256")
diff --git a/backend/app/audit.py b/backend/app/audit.py
index 14087279..c7180113 100644
--- a/backend/app/audit.py
+++ b/backend/app/audit.py
@@ -58,6 +58,17 @@ class AuditAction:
# A user sent a debug session's issue to the tracker — the one act that takes their log
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
ISSUE_CREATE = "issue_create"
+ # The Files library (`routes.files`): what a user put in, took out, kept or reorganised. Ids
+ # only — a file's or folder's name is the user's own text, not the admin's to read here.
+ FILE_UPLOAD = "file_upload"
+ FILE_DELETE = "file_delete"
+ FILE_KEEP = "file_keep"
+ FILE_UNKEEP = "file_unkeep"
+ FILE_MOVE = "file_move"
+ FILE_FOLDER_CREATE = "file_folder_create"
+ FILE_FOLDER_RENAME = "file_folder_rename"
+ FILE_FOLDER_RETENTION = "file_folder_retention"
+ FILE_FOLDER_DELETE = "file_folder_delete"
def record(
diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py
index befa1d11..134a5458 100644
--- a/backend/app/bodylimit.py
+++ b/backend/app/bodylimit.py
@@ -94,6 +94,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
# sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
+ ("POST", "/api/files/upload"): settings.download_upload_max_bytes,
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
}
diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py
index b962bf65..04eae041 100644
--- a/backend/app/downloads/folders.py
+++ b/backend/app/downloads/folders.py
@@ -9,7 +9,9 @@ shared asset can be held by several users), so a rename or a move is a column up
- an OWN folder is the user's, with the name they gave it.
Every file of a user is in exactly one folder. The folder a new file lands in is decided here
-(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once.
+(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
+moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube
+file in the TikTok folder would be a lie the folder's name tells.
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
@@ -26,6 +28,7 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app import sysconfig
+from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset
@@ -137,3 +140,88 @@ def stamp_expiry(db: Session, job: DownloadJob) -> None:
"""Stamp a job that just became done (in this session) with its expiry."""
job.expires_at = done_expiry(db, job.folder_id, job.kept)
job.expiry_warned = False
+
+
+class FolderError(Exception):
+ """A folder change the rules refuse; `reason` is the refusal the route answers with."""
+
+ def __init__(self, reason: str) -> None:
+ super().__init__(reason)
+ self.reason = reason
+
+
+def upload_target(db: Session, user_id: int, folder_id: int) -> FileFolder:
+ """The folder an upload asked to land in, if it may: one of the user's own folders, or their
+ Uploads folder. A file put in a site's folder would claim an origin it does not have."""
+ folder = db.get(FileFolder, folder_id)
+ if folder is None or folder.user_id != user_id:
+ raise FolderError("file_folder_gone")
+ if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
+ raise FolderError("file_folder_not_its_source")
+ return folder
+
+
+def restamp(db: Session, job: DownloadJob) -> None:
+ """Start a file's clock again from now: it moved to another folder, or its user stopped keeping
+ it. A file not done yet gets its expiry when it finishes (`stamp_expiry`), so it has none now."""
+ done = job.status == JobStatus.DONE
+ job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None
+ job.expiry_warned = False
+
+
+def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
+ """Keep a file until its user deletes it, or hand it back to its folder's retention."""
+ job.kept = kept
+ restamp(db, job)
+
+
+def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
+ """Put a file in `folder` — one of its user's own folders, or the source folder it came from.
+ The caller has checked that `folder` is the job's user's."""
+ if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
+ raise FolderError("file_folder_not_its_source")
+ if folder.id == job.folder_id:
+ return
+ job.folder_id = folder.id
+ restamp(db, job)
+
+
+def set_retention(db: Session, folder: FileFolder, days: int | None) -> None:
+ """Change how long `folder` keeps a file (None = until deleted). The done files already in it
+ move by the difference — each keeps the age it has — but none ends up closer than the GC's
+ warning window, so a shortened retention still warns before it takes anything."""
+ old = folder.retention_days
+ if old == days:
+ return
+ folder.retention_days = days
+ now = datetime.now(timezone.utc)
+ floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
+ jobs = db.execute(
+ select(DownloadJob).where(
+ DownloadJob.folder_id == folder.id,
+ DownloadJob.status == JobStatus.DONE,
+ DownloadJob.kept.is_(False),
+ )
+ ).scalars()
+ for job in jobs:
+ if days is None:
+ job.expires_at = None
+ elif old is None or job.expires_at is None:
+ job.expires_at = max(now + timedelta(days=days), floor)
+ else:
+ job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
+ job.expiry_warned = False
+
+
+def delete_own(db: Session, folder: FileFolder) -> int:
+ """Delete one of a user's own folders. Its files are not deleted: each goes back to the source
+ folder it came from, its clock starting there now. Returns how many moved."""
+ if folder.kind != FolderKind.OWN:
+ raise FolderError("file_folder_not_own")
+ jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all()
+ for job in jobs:
+ job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
+ restamp(db, job)
+ db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
+ db.delete(folder)
+ return len(jobs)
diff --git a/backend/app/downloads/upload.py b/backend/app/downloads/upload.py
index c22ec9c6..7ca0580a 100644
--- a/backend/app/downloads/upload.py
+++ b/backend/app/downloads/upload.py
@@ -31,6 +31,8 @@ from pathlib import Path
from sqlalchemy.orm import Session
+from app import audit
+from app.audit import AuditAction
from app.config import settings
from app.downloads import edit as editmod
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
@@ -53,14 +55,22 @@ def ingest_upload(
src_path: Path,
size: int,
display_name: str | None = None,
+ sha256: str | None = None,
+ folder_id: int | None = None,
) -> DownloadJob:
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return its job —
`done`, or `queued` for the worker's phone-safe conversion (see the module docstring). `src_path`
- is MOVED into place on success (the caller stages it to a temp file first).
+ is MOVED into place on success (the caller stages it to a temp file first). `sha256` is the
+ hash of those bytes, when the caller took it on the way in; `folder_id` the folder it asked for
+ (`folders.upload_target`), else its Uploads folder.
+
+ Raises folders.FolderError when that folder is not one an upload may land in (or is gone).
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
+ if folder_id is not None:
+ folders.upload_target(db, user_id, folder_id)
data = probe.probe_json(src_path)
vcodec, acodec, width, height, duration = probe.facts(data)
@@ -93,6 +103,9 @@ def ingest_upload(
asset.status = "pending" if convert else "ready"
asset.rel_path = rel
+ # The hash names the bytes that arrived: a conversion replaces them, so it is kept only
+ # when the file stays as it came.
+ asset.sha256 = None if convert else sha256
asset.size_bytes = actual_size
asset.container = ext or None
asset.vcodec = vcodec
@@ -124,9 +137,13 @@ def ingest_upload(
)
db.add(job)
folders.place(db, job)
+ if folder_id is not None:
+ job.folder_id = folder_id
if not convert:
folders.stamp_expiry(db, job)
service.incr_ref_count(db, asset.id)
+ db.flush()
+ audit.record(db, user_id, AuditAction.FILE_UPLOAD, target_type="file", target_id=job.id)
db.commit()
except Exception:
db.rollback()
diff --git a/backend/app/main.py b/backend/app/main.py
index e06746b9..33bbf9ca 100644
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -42,6 +42,7 @@ from app.routes import ( # noqa: E402 — logging is configured first
discover as discover_routes,
downloads,
feed,
+ files as files_routes,
health,
me,
notifications,
@@ -316,6 +317,7 @@ app.include_router(saved_views.router)
app.include_router(plex_routes.router)
app.include_router(downloads.router)
app.include_router(downloads.admin_router)
+app.include_router(files_routes.router)
app.include_router(public_routes.router)
app.include_router(admin.router)
app.include_router(config_routes.router)
diff --git a/backend/app/models.py b/backend/app/models.py
index 45d57e29..fd863d71 100644
--- a/backend/app/models.py
+++ b/backend/app/models.py
@@ -883,6 +883,15 @@ class MediaAsset(Base, TimestampMixin):
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ # The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
+ # streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
+ sha256: Mapped[str | None] = mapped_column(String(64))
+
+ @validates("rel_path")
+ def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
+ # A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
+ self.sha256 = None
+ return rel_path
class DownloadProfile(Base, TimestampMixin):
diff --git a/backend/app/routes/_common.py b/backend/app/routes/_common.py
index 51cd22c4..740f3f2d 100644
--- a/backend/app/routes/_common.py
+++ b/backend/app/routes/_common.py
@@ -6,7 +6,7 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c
one existence-leak policy.
"""
-from typing import IO, Any, TypeVar
+from typing import Any, Protocol, TypeVar
from fastapi import HTTPException, Request
from pydantic import BaseModel, ConfigDict, Field
@@ -122,8 +122,14 @@ async def capped_body(request: Request, limit: int) -> bytes:
return bytes(body)
+class ByteSink(Protocol):
+ """Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one."""
+
+ def write(self, data: bytes, /) -> object: ...
+
+
async def capped_stream_to(
- request: Request, out: IO[bytes], limit: int, refusal: HTTPException
+ request: Request, out: ByteSink, limit: int, refusal: HTTPException
) -> int:
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
straight to `out` instead of accumulated, returning the size.
diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py
index ff9acc32..63300b2a 100644
--- a/backend/app/routes/downloads.py
+++ b/backend/app/routes/downloads.py
@@ -5,17 +5,12 @@ frontend (the worker is a separate process); this module never downloads — it
state, serves finished files (range-aware, with the user's custom display name), and shares them.
"""
-import os
import re
-import tempfile
from typing import Literal
-from collections.abc import AsyncIterator, Sequence
-from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
-from pathlib import Path
-from urllib.parse import parse_qs, unquote, urlparse
+from urllib.parse import parse_qs, urlparse
-from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from sqlalchemy import delete, func, select
@@ -27,13 +22,12 @@ from app.collation import collated
from app import audit, sysconfig
from app.auth import admin_user, current_user
from app.config import settings
-from app.db import get_db, release_read_connection
+from app.db import get_db
from app.downloads import clip as clipmod
from app.downloads import credentials as dlcreds
from app.downloads import edit as editmod
from app.downloads import links as linksmod
from app.downloads import quota, service, storage
-from app.downloads import upload as uploadmod
from app.downloads import urlguard
from app.downloads.status import NOT_REFETCHABLE, JobStatus
from app.models import (
@@ -49,15 +43,28 @@ from app.models import (
)
from app.routes._common import (
ProgressIn,
- capped_stream_to,
get_or_404,
kept_position,
not_found,
owned_or_404,
)
+from app.routes.files import (
+ MAX_LIST,
+ SpooledUpload,
+ accessible_job,
+ assets_for,
+ delete_file,
+ own_job_for_update,
+ positions_for,
+ quota_detail,
+ serialize,
+ serialize_job,
+ serve_file,
+ spool_upload,
+ ingest_spooled,
+)
from app.security import hash_password
from app.userscope import is_messageable_user, messageable_clauses
-from app.utils import iso
from app.refusal import refused
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
@@ -65,11 +72,6 @@ admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"]
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
-# Cap the per-user job listings so a heavy library can't return an unbounded result set. The
-# Download Center renders newest-first; 500 is far beyond any real manual-download history (and
-# matches the admin listing's own cap).
-_MAX_LIST = 500
-
# --- source + serialization ----------------------------------------------------------------
@@ -113,118 +115,6 @@ def resolve_source(raw: str) -> tuple[str, str]:
raise refused(400, "invalid_link")
-def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
- ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
- return {
- "id": job.id,
- "status": job.status,
- "progress": job.progress,
- "speed_bps": job.speed_bps,
- "eta_s": job.eta_s,
- "phase": job.phase,
- "error": job.error,
- "error_code": job.error_code,
- "error_detail": job.error_detail,
- "queue_pos": job.queue_pos,
- "created_at": iso(job.created_at),
- "source_kind": job.source_kind,
- "source_ref": job.source_ref,
- "source_url": service.reference_url(job, asset),
- # A locally-generated poster to fall back to when the source had no thumbnail.
- "poster_url": f"/api/downloads/{job.id}/poster.jpg"
- if (asset is not None and asset.poster_path)
- else None,
- "profile_id": job.profile_id,
- "spec": job.profile_snapshot,
- "display_name": job.display_name,
- "display_uploader": job.display_uploader,
- "display_uploader_url": job.display_uploader_url,
- "extra_links": job.extra_links or [],
- "job_kind": job.job_kind,
- "source_job_id": job.source_job_id,
- "edit_spec": job.edit_spec,
- "can_download": ready and job.status == JobStatus.DONE,
- "expired": job.asset_id is None and job.status == JobStatus.DONE,
- # The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
- "folder_id": job.folder_id,
- "kept": job.kept,
- "expires_at": iso(job.expires_at),
- "asset": None
- if asset is None
- else {
- "id": asset.id,
- "status": asset.status,
- "title": asset.title,
- "uploader": asset.uploader,
- "upload_date": iso(asset.upload_date),
- "duration_s": asset.duration_s,
- "size_bytes": asset.size_bytes,
- "width": asset.width,
- "height": asset.height,
- "container": asset.container,
- # "video" | "audio" | "file" — the library gates play/edit on this (media only); a
- # generic uploaded file is download-only. Null while still in flight (set at finalize).
- "media_kind": asset.media_kind,
- "uploader_url": asset.uploader_url,
- "thumbnail_url": asset.thumbnail_url,
- },
- }
-
-
-def _assets_for(db: Session, jobs: list[DownloadJob]) -> dict[int, MediaAsset]:
- ids = {j.asset_id for j in jobs if j.asset_id}
- if not ids:
- return {}
- rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
- return {a.id: a for a in rows}
-
-
-def _positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
- """`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
- ids = [j.id for j in jobs]
- if not ids:
- return {}
- rows = db.execute(
- select(LibraryState.job_id, LibraryState.position_seconds).where(
- LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
- )
- ).all()
- return {job_id: pos for job_id, pos in rows}
-
-
-def _serialize_job(db: Session, job: DownloadJob) -> dict:
- """Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
- every single-job handler shares."""
- asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
- return _serialize(job, asset)
-
-
-def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
- """The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
- or writes the status those decide on (pause). The job is re-read under a row lock, so two
- overlapping requests for one job run one after the other and the second decides on the first's
- committed result — never both releasing the same hold, and never a pause turning a job a cancel
- just released back into a holder; either can free a file another job still holds.
-
- The asset is locked BEFORE the job: the order a finishing worker child takes them in
- (`service.finalize_if_running`), so the two can never deadlock."""
- while True:
- asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
- if asset_id is not None:
- db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
- job = db.execute(
- select(DownloadJob)
- .where(DownloadJob.id == job_id)
- .with_for_update()
- .execution_options(populate_existing=True)
- ).scalar_one_or_none()
- if job is None:
- raise not_found("download")
- if job.asset_id == asset_id:
- return job
- db.rollback() # another request moved the job to another asset meanwhile: lock that one
-
-
# --- profiles ------------------------------------------------------------------------------
@@ -434,91 +324,8 @@ def enqueue_download(
try:
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
except quota.QuotaExceeded as e:
- raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
- return _serialize_job(db, job)
-
-
-# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
-# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
-# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
-# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
-# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
-# declared AFTER the auth one, is the only shape that closes it — the same one
-# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
-# every route with a raised ceiling rather than leaving it to a comment like this one.
-#
-# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
-# Hungarian filename would otherwise be unsendable.
-UPLOAD_FILENAME_HEADER = "x-upload-filename"
-UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
-
-
-@dataclass(frozen=True)
-class SpooledUpload:
- """What the body dependency hands the handler: the bytes already on disk, plus the two names
- that came alongside them."""
-
- path: Path
- size: int
- user_id: int
- filename: str
- display_name: str | None
-
-
-def _header_text(request: Request, name: str) -> str:
- """One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
- caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
- own fallback names the file."""
- raw = request.headers.get(name, "")
- try:
- return unquote(raw, errors="strict").strip()
- except (UnicodeDecodeError, ValueError):
- # silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
- # cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
- # a label; logging it would hand any client a free log line per request.
- return ""
-
-
-async def spool_upload(
- request: Request,
- user: User = Depends(current_user),
- db: Session = Depends(get_db),
-) -> AsyncIterator[SpooledUpload]:
- """Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
-
- `user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
- naming `current_user` here makes the auth check a precondition of reading the body at all — a
- stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
- MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
- empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
- cap = sysconfig.get_int(db, "download_upload_max_bytes")
- user_id = user.id
- # Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
- # for why, and for the precondition it enforces. Everything this dependency still needs is
- # already an int or a header in hand: touching an ORM attribute below would reload `user` and
- # check a connection straight back out.
- release_read_connection(db)
- staging = Path(settings.files_root) / ".staging" / "uploads"
- staging.mkdir(parents=True, exist_ok=True)
- fd, tmp = tempfile.mkstemp(dir=str(staging))
- tmp_path = Path(tmp)
- try:
- with os.fdopen(fd, "wb") as out:
- size = await capped_stream_to(
- request,
- out,
- cap,
- HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
- )
- yield SpooledUpload(
- path=tmp_path,
- size=size,
- user_id=user_id,
- filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
- display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
- )
- finally:
- tmp_path.unlink(missing_ok=True)
+ raise HTTPException(status_code=422, detail=quota_detail(e)) from e
+ return serialize_job(db, job)
@router.post("/upload")
@@ -526,30 +333,9 @@ def upload_download(
upload: SpooledUpload = Depends(spool_upload),
db: Session = Depends(get_db),
) -> dict:
- """Ingest a local file into the caller's library. Any file type is accepted; media
- (video/audio) gets the full function set, a generic file is download-only. The body IS the file
- (see the header note above); `spool_upload` has already authenticated the caller and streamed it
- to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which
- probes + places it and returns its job — `done`, or `queued` for the worker's phone-safe
- conversion (`downloads.upload`).
-
- A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
- plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because
- reading the body is; keeping them in separate callables is what lets each run where it belongs."""
- if upload.size == 0:
- raise HTTPException(status_code=400, detail={"code": "upload_empty"})
- try:
- job = uploadmod.ingest_upload(
- db,
- upload.user_id,
- filename=upload.filename,
- src_path=upload.path,
- size=upload.size,
- display_name=upload.display_name,
- )
- except quota.QuotaExceeded as e:
- raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
- return _serialize_job(db, job)
+ """The Files library's upload (`routes.files.upload_file`) at its old address, until the client
+ moves to the new one."""
+ return serialize_job(db, ingest_spooled(db, upload))
class EnqueueEditIn(BaseModel):
@@ -567,7 +353,7 @@ def enqueue_edit(
The source may be the user's own download OR one shared with them — either way the resulting
clip is a NEW per-user derivative in the editor's own library (counts against their quota); the
source file is only read, never modified, so editing a shared video is safe."""
- source_job = _accessible_job(db, user, body.source_job_id)
+ source_job = accessible_job(db, user, body.source_job_id)
if source_job.status != JobStatus.DONE or source_job.asset_id is None:
raise refused(409, "edit_unfinished")
spec = body.edit_spec
@@ -575,10 +361,10 @@ def enqueue_edit(
try:
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
except quota.QuotaExceeded as e:
- raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
+ raise HTTPException(status_code=422, detail=quota_detail(e)) from e
except service.EditError as e:
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
- return _serialize_job(db, job)
+ return serialize_job(db, job)
class ClipCropIn(BaseModel):
@@ -620,14 +406,8 @@ def enqueue_clip(
detail = {"code": "clip", "reason": e.reason, **e.params}
raise HTTPException(status_code=status, detail=detail) from e
except quota.QuotaExceeded as e:
- raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
- return _serialize_job(db, job)
-
-
-def _quota_detail(e: quota.QuotaExceeded) -> dict:
- """Structured error the trilingual client localizes (reason key + numbers), instead of a
- pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
- return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
+ raise HTTPException(status_code=422, detail=quota_detail(e)) from e
+ return serialize_job(db, job)
@router.get("")
@@ -637,16 +417,16 @@ def list_downloads(user: User = Depends(current_user), db: Session = Depends(get
select(DownloadJob)
.where(DownloadJob.user_id == user.id)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
- .limit(_MAX_LIST)
+ .limit(MAX_LIST)
)
.scalars()
.all()
)
- assets = _assets_for(db, jobs)
- positions = _positions_for(db, user, jobs)
+ assets = assets_for(db, jobs)
+ positions = positions_for(db, user, jobs)
return [
{
- **_serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
+ **serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
"position_seconds": positions.get(j.id, 0),
}
for j in jobs
@@ -705,16 +485,16 @@ def shared_with_me(user: User = Depends(current_user), db: Session = Depends(get
.join(DownloadShare, DownloadShare.job_id == DownloadJob.id)
.where(DownloadShare.to_user_id == user.id)
.order_by(DownloadShare.created_at.desc(), DownloadJob.id.desc())
- .limit(_MAX_LIST)
+ .limit(MAX_LIST)
)
.scalars()
.all()
)
- assets = _assets_for(db, jobs)
- positions = _positions_for(db, user, jobs)
+ assets = assets_for(db, jobs)
+ positions = positions_for(db, user, jobs)
out = []
for j in jobs:
- data = _serialize(j, assets.get(j.asset_id))
+ data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
data["shared"] = True
data["position_seconds"] = positions.get(j.id, 0)
out.append(data)
@@ -778,25 +558,25 @@ def update_download_meta(
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
job.extra_links = cleaned or None
db.commit()
- return _serialize_job(db, job)
+ return serialize_job(db, job)
@router.post("/{job_id}/pause")
def pause_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
- job = _own_job_for_update(db, user, job_id)
+ job = own_job_for_update(db, user, job_id)
service.pause(db, job)
- return _serialize_job(db, job)
+ return serialize_job(db, job)
@router.post("/{job_id}/resume")
def resume_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
- job = _own_job_for_update(db, user, job_id)
+ job = own_job_for_update(db, user, job_id)
service.resume(db, job)
- return _serialize_job(db, job)
+ return serialize_job(db, job)
@router.post("/{job_id}/redownload")
@@ -815,7 +595,7 @@ def redownload_download(
under the current format/naming rules — never silently reuses a shared cached file — and (b)
never deletes a file another job/user still shares: the old asset is only removed if THIS job
was its last holder."""
- job = _own_job_for_update(db, user, job_id)
+ job = own_job_for_update(db, user, job_id)
if job.job_kind in NOT_REFETCHABLE:
# None has a re-fetchable source: an editor clip is derived locally, an upload IS the
# source, an ad-hoc clip is a range of a source the clip itself does not re-fetch.
@@ -823,82 +603,36 @@ def redownload_download(
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
raise refused(409, "still_downloading")
service.redownload(db, job)
- return _serialize_job(db, job)
+ return serialize_job(db, job)
@router.post("/{job_id}/cancel")
def cancel_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
- job = _own_job_for_update(db, user, job_id)
+ job = own_job_for_update(db, user, job_id)
service.cancel(db, job)
- return _serialize_job(db, job)
+ return serialize_job(db, job)
@router.delete("/{job_id}")
def delete_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
- job = _own_job_for_update(db, user, job_id)
- service.delete(db, job)
- return {"deleted": job_id}
+ """The Files library's delete (`routes.files.delete_file`, audited) at its old address, until the
+ client moves to the new one."""
+ delete_file(job_id, user, db)
+ return {"ok": True, "deleted": job_id}
# --- file download (range-aware, custom display name) --------------------------------------
-def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
- job = db.get(DownloadJob, job_id)
- if job is None:
- raise not_found("download")
- if job.user_id == user.id:
- return job
- shared = db.execute(
- select(DownloadShare.id).where(
- DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
- )
- ).first()
- if shared is None:
- raise not_found("download")
- return job
-
-
@router.get("/{job_id}/file")
-def download_file(
- job_id: int,
- request: Request,
- user: User = Depends(current_user),
- db: Session = Depends(get_db),
-):
- job = _accessible_job(db, user, job_id)
- asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
- if asset is None or asset.status != "ready" or not asset.rel_path:
- raise refused(409, "download_not_ready")
- path = storage.safe_abs_path(settings.files_root, asset.rel_path)
- if path is None:
- raise not_found("file")
-
- filename = storage.download_filename(
- job.display_name or asset.title or asset.source_ref, asset.container, path
- )
-
- asset.last_access_at = datetime.now(timezone.utc)
+def download_file(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
+ response = serve_file(db, accessible_job(db, user, job_id))
db.commit()
- # The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
- # (storage.served_media_type), NEVER guessed from the file's own extension. A generic 'file' (and
- # any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
- # returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
- # this origin. Recognised media is served with its real type so the in-app player streams it. Both
- # are range-aware (FileResponse → 206).
- media_type = storage.served_media_type(asset.media_kind, asset.container)
- if media_type is None:
- return FileResponse(
- path,
- filename=filename,
- media_type="application/octet-stream",
- content_disposition_type="attachment",
- )
- return FileResponse(path, filename=filename, media_type=media_type)
+ return response
# --- editor filmstrip (scrub track) --------------------------------------------------------
@@ -913,7 +647,7 @@ def set_library_progress(
) -> dict:
"""Checkpoint the library player's resume point in a download the caller can play (their own, or
one shared with them). A position too early or too close to the end clears it instead."""
- _accessible_job(db, user, job_id)
+ accessible_job(db, user, job_id)
position = kept_position(body)
mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id)
if not position:
@@ -945,7 +679,7 @@ def storyboard_meta(
Best-effort: for a very long/high-res source the sprite may not generate in time — then
`available` is False and the editor shows a plain timeline."""
- job = _accessible_job(db, user, job_id)
+ job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
@@ -966,7 +700,7 @@ def storyboard_meta(
@router.get("/{job_id}/poster.jpg")
def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
"""The locally-generated poster frame for a download whose source had no thumbnail."""
- job = _accessible_job(db, user, job_id)
+ job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.poster_path:
raise not_found("poster")
@@ -980,7 +714,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
def storyboard_image(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
):
- job = _accessible_job(db, user, job_id)
+ job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.storyboard_path:
raise not_found("filmstrip")
@@ -1236,19 +970,19 @@ def admin_list_downloads(
db.execute(
select(DownloadJob)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
- .limit(_MAX_LIST)
+ .limit(MAX_LIST)
)
.scalars()
.all()
)
- assets = _assets_for(db, jobs)
+ assets = assets_for(db, jobs)
emails = {
u.id: u.email
for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars()
}
out = []
for j in jobs:
- data = _serialize(j, assets.get(j.asset_id))
+ data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
data["user_email"] = emails.get(j.user_id)
out.append(data)
return out
diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py
new file mode 100644
index 00000000..c15d9bd7
--- /dev/null
+++ b/backend/app/routes/files.py
@@ -0,0 +1,691 @@
+"""The Files library HTTP API — a user's files, the folders they sit in, and the file itself.
+
+A file in the library is a user's DONE job (`DownloadJob`) over its shared `MediaAsset`: the job is
+the per-user layer (name, folder, kept, expiry, shares and links all hang off it), the asset the
+bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
+
+In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
+the routes are shared with that module — serialization, the ownership locks, the upload spool —
+because a download and a library file are one row at two moments.
+"""
+
+import base64
+import hashlib
+import os
+import tempfile
+from collections.abc import AsyncIterator, Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import BinaryIO, Literal
+from urllib.parse import unquote
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from fastapi.responses import FileResponse
+from pydantic import BaseModel, Field
+from sqlalchemy import func, or_, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app import audit, sysconfig
+from app.audit import AuditAction
+from app.auth import current_user
+from app.collation import collated
+from app.config import settings
+from app.db import get_db, release_read_connection
+from app.downloads import folders, quota, service, storage
+from app.downloads import upload as uploadmod
+from app.downloads.status import JobStatus
+from app.mailtext import lang_for_user
+from app.models import (
+ DISPLAY_TEXT_MAX,
+ DownloadJob,
+ DownloadShare,
+ FileFolder,
+ LibraryState,
+ MediaAsset,
+ User,
+ display_text,
+)
+from app.refusal import refused
+from app.routes._common import capped_stream_to, not_found, owned_or_404
+from app.utils import iso, like_escape
+
+router = APIRouter(prefix="/api/files", tags=["files"])
+
+# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
+# own cap; far beyond any real library).
+MAX_LIST = 500
+# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
+_MAX_RETENTION_DAYS = 3650
+_HASH_CHUNK = 1 << 20
+
+
+# --- shared with routes.downloads -------------------------------------------------------
+
+
+def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
+ ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
+ return {
+ "id": job.id,
+ "status": job.status,
+ "progress": job.progress,
+ "speed_bps": job.speed_bps,
+ "eta_s": job.eta_s,
+ "phase": job.phase,
+ "error": job.error,
+ "error_code": job.error_code,
+ "error_detail": job.error_detail,
+ "queue_pos": job.queue_pos,
+ "created_at": iso(job.created_at),
+ "source_kind": job.source_kind,
+ "source_ref": job.source_ref,
+ "source_url": service.reference_url(job, asset),
+ # A locally-generated poster to fall back to when the source had no thumbnail.
+ "poster_url": f"/api/downloads/{job.id}/poster.jpg"
+ if (asset is not None and asset.poster_path)
+ else None,
+ "profile_id": job.profile_id,
+ "spec": job.profile_snapshot,
+ "display_name": job.display_name,
+ "display_uploader": job.display_uploader,
+ "display_uploader_url": job.display_uploader_url,
+ "extra_links": job.extra_links or [],
+ "job_kind": job.job_kind,
+ "source_job_id": job.source_job_id,
+ "edit_spec": job.edit_spec,
+ "can_download": ready and job.status == JobStatus.DONE,
+ "expired": job.asset_id is None and job.status == JobStatus.DONE,
+ # The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
+ "folder_id": job.folder_id,
+ "kept": job.kept,
+ "expires_at": iso(job.expires_at),
+ "asset": None
+ if asset is None
+ else {
+ "id": asset.id,
+ "status": asset.status,
+ "title": asset.title,
+ "uploader": asset.uploader,
+ "upload_date": iso(asset.upload_date),
+ "duration_s": asset.duration_s,
+ "size_bytes": asset.size_bytes,
+ "width": asset.width,
+ "height": asset.height,
+ "container": asset.container,
+ # "video" | "audio" | "file" — the library gates play/edit on this (media only); a
+ # generic uploaded file is download-only. Null while still in flight (set at finalize).
+ "media_kind": asset.media_kind,
+ "uploader_url": asset.uploader_url,
+ "thumbnail_url": asset.thumbnail_url,
+ },
+ }
+
+
+def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
+ ids = {j.asset_id for j in jobs if j.asset_id}
+ if not ids:
+ return {}
+ rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
+ return {a.id: a for a in rows}
+
+
+def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
+ """`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
+ ids = [j.id for j in jobs]
+ if not ids:
+ return {}
+ rows = db.execute(
+ select(LibraryState.job_id, LibraryState.position_seconds).where(
+ LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
+ )
+ ).all()
+ return {job_id: pos for job_id, pos in rows}
+
+
+def serialize_job(db: Session, job: DownloadJob) -> dict:
+ """Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
+ every single-job handler shares."""
+ asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
+ return serialize(job, asset)
+
+
+def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
+ """The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
+ or writes the status those decide on (pause). The job is re-read under a row lock, so two
+ overlapping requests for one job run one after the other and the second decides on the first's
+ committed result — never both releasing the same hold, and never a pause turning a job a cancel
+ just released back into a holder; either can free a file another job still holds.
+
+ The asset is locked BEFORE the job: the order a finishing worker child takes them in
+ (`service.finalize_if_running`), so the two can never deadlock."""
+ while True:
+ asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
+ if asset_id is not None:
+ db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
+ job = db.execute(
+ select(DownloadJob)
+ .where(DownloadJob.id == job_id)
+ .with_for_update()
+ .execution_options(populate_existing=True)
+ ).scalar_one_or_none()
+ if job is None:
+ raise not_found("download")
+ if job.asset_id == asset_id:
+ return job
+ db.rollback() # another request moved the job to another asset meanwhile: lock that one
+
+
+def quota_detail(e: quota.QuotaExceeded) -> dict:
+ """Structured error the trilingual client localizes (reason key + numbers), instead of a
+ pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
+ return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
+
+
+# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
+# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
+# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
+# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
+# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
+# declared AFTER the auth one, is the only shape that closes it — the same one
+# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
+# every route with a raised ceiling rather than leaving it to a comment like this one.
+#
+# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
+# Hungarian filename would otherwise be unsendable.
+UPLOAD_FILENAME_HEADER = "x-upload-filename"
+UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
+# The folder the file goes in (its id); without it, the user's Uploads folder.
+UPLOAD_FOLDER_HEADER = "x-upload-folder"
+
+
+@dataclass(frozen=True)
+class SpooledUpload:
+ """What the body dependency hands the handler: the bytes already on disk and their hash, plus
+ the names and the folder that came alongside them."""
+
+ path: Path
+ size: int
+ sha256: str
+ user_id: int
+ filename: str
+ display_name: str | None
+ folder_id: int | None
+
+
+def _header_text(request: Request, name: str) -> str:
+ """One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
+ caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
+ own fallback names the file."""
+ raw = request.headers.get(name, "")
+ try:
+ return unquote(raw, errors="strict").strip()
+ except (UnicodeDecodeError, ValueError):
+ # silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
+ # cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
+ # a label; logging it would hand any client a free log line per request.
+ return ""
+
+
+def _upload_folder(db: Session, user_id: int, raw: str) -> int | None:
+ """The folder id an upload asked for (`UPLOAD_FOLDER_HEADER`), checked; None when it asked for
+ none."""
+ if not raw.strip():
+ return None
+ try:
+ return folders.upload_target(db, user_id, int(raw)).id
+ except ValueError as e:
+ raise refused(409, "file_folder_gone") from e
+ except folders.FolderError as e:
+ raise refused(409, e.reason) from e
+
+
+class _HashingSink:
+ """The upload's file, hashing the bytes on their way in: the hash costs no second read."""
+
+ def __init__(self, out: BinaryIO) -> None:
+ self.out = out
+ self.hash = hashlib.sha256()
+
+ def write(self, data: bytes, /) -> int:
+ self.hash.update(data)
+ return self.out.write(data)
+
+
+async def spool_upload(
+ request: Request,
+ user: User = Depends(current_user),
+ db: Session = Depends(get_db),
+) -> AsyncIterator[SpooledUpload]:
+ """Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
+
+ `user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
+ naming `current_user` here makes the auth check a precondition of reading the body at all — a
+ stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
+ MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
+ empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
+ cap = sysconfig.get_int(db, "download_upload_max_bytes")
+ user_id = user.id
+ # A folder the file may not go in is refused before the transfer, not after it.
+ folder_id = _upload_folder(db, user_id, request.headers.get(UPLOAD_FOLDER_HEADER, ""))
+ # Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
+ # for why, and for the precondition it enforces. Everything this dependency still needs is
+ # already an int or a header in hand: touching an ORM attribute below would reload `user` and
+ # check a connection straight back out.
+ release_read_connection(db)
+ staging = Path(settings.files_root) / ".staging" / "uploads"
+ staging.mkdir(parents=True, exist_ok=True)
+ fd, tmp = tempfile.mkstemp(dir=str(staging))
+ tmp_path = Path(tmp)
+ try:
+ with os.fdopen(fd, "wb") as out:
+ sink = _HashingSink(out)
+ size = await capped_stream_to(
+ request,
+ sink,
+ cap,
+ HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
+ )
+ yield SpooledUpload(
+ path=tmp_path,
+ size=size,
+ sha256=sink.hash.hexdigest(),
+ user_id=user_id,
+ filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
+ display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
+ folder_id=folder_id,
+ )
+ finally:
+ tmp_path.unlink(missing_ok=True)
+
+
+def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
+ """A file the caller may read: their own, or one shared with them."""
+ job = db.get(DownloadJob, job_id)
+ if job is None:
+ raise not_found("download")
+ if job.user_id == user.id:
+ return job
+ shared = db.execute(
+ select(DownloadShare.id).where(
+ DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
+ )
+ ).first()
+ if shared is None:
+ raise not_found("download")
+ return job
+
+
+# --- folders -------------------------------------------------------------------------------
+
+
+def _serialize_folder(folder: FileFolder, count: int, size: int) -> dict:
+ return {
+ "id": folder.id,
+ "kind": folder.kind,
+ # A source folder's name is the client's to render from its key, in the reader's language.
+ "source_key": folder.source_key,
+ "name": folder.name,
+ "retention_days": folder.retention_days,
+ "count": count,
+ "size_bytes": size,
+ "created_at": iso(folder.created_at),
+ }
+
+
+def _own_folder_or_404(db: Session, user: User, folder_id: int) -> FileFolder:
+ return owned_or_404(db, FileFolder, folder_id, user, what="folder")
+
+
+@router.get("/folders")
+def list_folders(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
+ """The caller's folders with how many files each holds and their size. A source folder its
+ files have all left stays (count 0): the client decides whether to show it."""
+ done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
+ rows = db.execute(
+ select(
+ FileFolder,
+ func.count(DownloadJob.id),
+ func.coalesce(func.sum(MediaAsset.size_bytes), 0),
+ )
+ .outerjoin(DownloadJob, done)
+ .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
+ .where(FileFolder.user_id == user.id)
+ .group_by(FileFolder.id)
+ .order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
+ ).all()
+ return [_serialize_folder(f, count, int(size)) for f, count, size in rows]
+
+
+class FolderIn(BaseModel):
+ name: str = Field(min_length=1, max_length=DISPLAY_TEXT_MAX)
+
+
+def _folder_name(raw: str) -> str:
+ name = display_text(raw)
+ if not name:
+ raise refused(400, "file_folder_name_required")
+ return name
+
+
+@router.post("/folders")
+def create_folder(
+ body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
+) -> dict:
+ folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
+ try:
+ with db.begin_nested():
+ db.add(folder)
+ db.flush()
+ except IntegrityError as e: # the unique (user, lower(name)): one folder per name
+ raise refused(409, "file_folder_name_taken") from e
+ audit.record(
+ db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
+ )
+ db.commit()
+ return _serialize_folder(folder, 0, 0)
+
+
+class UpdateFolderIn(BaseModel):
+ # Presence (model_fields_set) decides what is touched; retention_days null = keep forever.
+ name: str | None = Field(default=None, min_length=1, max_length=DISPLAY_TEXT_MAX)
+ retention_days: int | None = Field(default=None, ge=1, le=_MAX_RETENTION_DAYS)
+
+
+@router.patch("/folders/{folder_id}")
+def update_folder(
+ folder_id: int,
+ body: UpdateFolderIn,
+ user: User = Depends(current_user),
+ db: Session = Depends(get_db),
+) -> dict:
+ """Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
+ how long any of their folders keeps a file (`folders.set_retention`)."""
+ folder = _own_folder_or_404(db, user, folder_id)
+ sent = body.model_fields_set
+ if "name" in sent:
+ if folder.kind != folders.FolderKind.OWN:
+ raise refused(409, "file_folder_not_own")
+ folder.name = _folder_name(body.name or "")
+ try:
+ with db.begin_nested():
+ db.flush()
+ except IntegrityError as e:
+ raise refused(409, "file_folder_name_taken") from e
+ audit.record(
+ db,
+ user.id,
+ AuditAction.FILE_FOLDER_RENAME,
+ target_type="file_folder",
+ target_id=folder.id,
+ )
+ if "retention_days" in sent and body.retention_days != folder.retention_days:
+ before = folder.retention_days
+ folders.set_retention(db, folder, body.retention_days)
+ audit.record(
+ db,
+ user.id,
+ AuditAction.FILE_FOLDER_RETENTION,
+ target_type="file_folder",
+ target_id=folder.id,
+ before={"retention_days": before},
+ after={"retention_days": body.retention_days},
+ )
+ db.commit()
+ count, size = _folder_totals(db, folder.id)
+ return _serialize_folder(folder, count, size)
+
+
+def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
+ count, size = db.execute(
+ select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
+ .select_from(DownloadJob)
+ .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
+ .where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
+ ).one()
+ return count, int(size)
+
+
+@router.delete("/folders/{folder_id}")
+def delete_folder(
+ folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
+) -> dict:
+ """Delete one of the caller's own folders; its files go back to their source folders."""
+ folder = _own_folder_or_404(db, user, folder_id)
+ try:
+ moved = folders.delete_own(db, folder)
+ except folders.FolderError as e:
+ raise refused(409, e.reason) from e
+ audit.record(
+ db,
+ user.id,
+ AuditAction.FILE_FOLDER_DELETE,
+ target_type="file_folder",
+ target_id=folder_id,
+ after={"files_moved": moved},
+ )
+ db.commit()
+ return {"ok": True, "deleted": folder_id, "files_moved": moved}
+
+
+# --- files ---------------------------------------------------------------------------------
+
+
+@router.get("")
+def list_files(
+ folder_id: int | None = None,
+ q: str = Query("", max_length=200),
+ sort: Literal["added", "name", "size"] = "added",
+ order: Literal["asc", "desc"] = "desc",
+ scope: Literal["mine", "shared"] = "mine",
+ user: User = Depends(current_user),
+ db: Session = Depends(get_db),
+) -> list[dict]:
+ """The caller's done files (`scope=mine`, optionally one folder's), or the ones shared with
+ them (`scope=shared`); searched by name and channel, sorted by when they were added, by name
+ (in the reader's alphabet) or by size."""
+ stmt = (
+ select(DownloadJob)
+ .join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
+ .where(DownloadJob.status == JobStatus.DONE)
+ )
+ if scope == "shared":
+ stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
+ DownloadShare.to_user_id == user.id
+ )
+ else:
+ stmt = stmt.where(DownloadJob.user_id == user.id)
+ if folder_id is not None:
+ stmt = stmt.where(DownloadJob.folder_id == folder_id)
+ name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
+ term = q.strip()
+ if term:
+ like = f"%{like_escape(term)}%"
+ stmt = stmt.where(
+ or_(
+ name.ilike(like),
+ func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
+ )
+ )
+ key = {
+ "added": DownloadJob.created_at,
+ "name": collated(name, lang_for_user(user)),
+ "size": MediaAsset.size_bytes,
+ }[sort]
+ first = key.asc() if order == "asc" else key.desc()
+ tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
+ jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
+ assets = assets_for(db, jobs)
+ positions = positions_for(db, user, jobs)
+ shared = scope == "shared"
+ return [
+ {
+ **serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
+ "position_seconds": positions.get(j.id, 0),
+ "shared": shared,
+ }
+ for j in jobs
+ ]
+
+
+class UpdateFileIn(BaseModel):
+ # Presence (model_fields_set) decides what is touched; a null/"" name falls back to the title.
+ name: str | None = None
+ folder_id: int | None = None
+ kept: bool | None = None
+
+
+@router.patch("/{job_id}")
+def update_file(
+ job_id: int,
+ body: UpdateFileIn,
+ user: User = Depends(current_user),
+ db: Session = Depends(get_db),
+) -> dict:
+ """Rename a file (its display name; the file on disk keeps its own), move it to another folder
+ (`folders.move`), or keep it until deleted / hand it back to its folder's retention."""
+ job = owned_or_404(db, DownloadJob, job_id, user, what="download")
+ sent = body.model_fields_set
+ if "name" in sent:
+ job.display_name = display_text(body.name)
+ if "folder_id" in sent:
+ if body.folder_id is None:
+ raise refused(400, "file_folder_gone")
+ folder = db.get(FileFolder, body.folder_id)
+ if folder is None or folder.user_id != user.id:
+ raise refused(409, "file_folder_gone")
+ before = job.folder_id
+ try:
+ folders.move(db, job, folder)
+ except folders.FolderError as e:
+ raise refused(409, e.reason) from e
+ if job.folder_id != before:
+ audit.record(
+ db,
+ user.id,
+ AuditAction.FILE_MOVE,
+ target_type="file",
+ target_id=job.id,
+ before={"folder_id": before},
+ after={"folder_id": job.folder_id},
+ )
+ if "kept" in sent and body.kept is not None and body.kept != job.kept:
+ folders.set_kept(db, job, body.kept)
+ action = AuditAction.FILE_KEEP if body.kept else AuditAction.FILE_UNKEEP
+ audit.record(db, user.id, action, target_type="file", target_id=job.id)
+ db.commit()
+ return serialize_job(db, job)
+
+
+@router.delete("/{job_id}")
+def delete_file(
+ job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
+) -> dict:
+ """Delete a file from the caller's library; the bytes go with their last holder."""
+ job = own_job_for_update(db, user, job_id)
+ audit.record(db, user.id, AuditAction.FILE_DELETE, target_type="file", target_id=job_id)
+ service.delete(db, job) # commits, the audit row with it
+ return {"ok": True, "deleted": job_id}
+
+
+@router.post("/upload")
+def upload_file(
+ upload: SpooledUpload = Depends(spool_upload),
+ db: Session = Depends(get_db),
+) -> dict:
+ """Put a local file in the caller's library — in the folder `UPLOAD_FOLDER_HEADER` names, else
+ their Uploads folder. Any file type; media gets the player, a generic file is download-only.
+ The body IS the file (see the header note above `UPLOAD_FILENAME_HEADER`); `spool_upload` has
+ authenticated the caller, checked the folder and streamed it to disk already.
+
+ A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
+ plus sync DB writes — so it must NOT run on the event loop."""
+ return serialize_job(db, ingest_spooled(db, upload))
+
+
+def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
+ """The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
+ its refusals answered as HTTP."""
+ if upload.size == 0:
+ raise HTTPException(status_code=400, detail={"code": "upload_empty"})
+ try:
+ job = uploadmod.ingest_upload(
+ db,
+ upload.user_id,
+ filename=upload.filename,
+ src_path=upload.path,
+ size=upload.size,
+ display_name=upload.display_name,
+ sha256=upload.sha256,
+ folder_id=upload.folder_id,
+ )
+ except quota.QuotaExceeded as e:
+ raise HTTPException(status_code=422, detail=quota_detail(e)) from e
+ except folders.FolderError as e: # the folder went while the file was on its way
+ raise refused(409, e.reason) from e
+ return job
+
+
+def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
+ asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
+ if asset is None or asset.status != "ready" or not asset.rel_path:
+ raise refused(409, "download_not_ready")
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
+ if path is None:
+ raise not_found("file")
+ return asset, path
+
+
+def serve_file(db: Session, job: DownloadJob) -> FileResponse:
+ """The file of `job` (the caller may read it), range-aware, named by the user's display name.
+
+ The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
+ (`storage.served_media_type`), NEVER guessed from the file's own extension. A generic 'file' (and
+ any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
+ returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
+ this origin. Recognised media is served with its real type so the in-app player streams it. Both
+ are range-aware (FileResponse → 206). A file whose hash is known says so (`Repr-Digest`, RFC
+ 9530: the digest of the whole file, whichever range is asked for)."""
+ asset, path = _ready_file(db, job)
+ filename = storage.download_filename(
+ job.display_name or asset.title or asset.source_ref, asset.container, path
+ )
+ headers = {}
+ if asset.sha256:
+ digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
+ headers["Repr-Digest"] = f"sha-256=:{digest}:"
+ asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
+ media_type = storage.served_media_type(asset.media_kind, asset.container)
+ if media_type is None:
+ return FileResponse(
+ path,
+ filename=filename,
+ media_type="application/octet-stream",
+ content_disposition_type="attachment",
+ headers=headers,
+ )
+ return FileResponse(path, filename=filename, media_type=media_type, headers=headers)
+
+
+@router.get("/{job_id}/file")
+def get_file(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
+ response = serve_file(db, accessible_job(db, user, job_id))
+ db.commit()
+ return response
+
+
+@router.get("/{job_id}/sha256")
+def get_sha256(
+ job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
+) -> dict:
+ """The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
+ A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool."""
+ asset, path = _ready_file(db, accessible_job(db, user, job_id))
+ if not asset.sha256:
+ h = hashlib.sha256()
+ with path.open("rb") as f:
+ while chunk := f.read(_HASH_CHUNK):
+ h.update(chunk)
+ asset.sha256 = h.hexdigest()
+ db.commit()
+ return {"sha256": asset.sha256}
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index d4d142af..f2e1b127 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -254,20 +254,24 @@ def as_user(db, override_dep, _configured):
@pytest.fixture
def upload_headers():
- """Build the headers `POST /api/downloads/upload` takes (the file is the raw body).
+ """Build the headers the upload routes take (the file is the raw body).
Shared, because two test modules drive that route and a second copy of this is exactly the drift
that lets the wire shape change in one place only. The body IS the file now; the names ride as
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
latin-1 and a Hungarian filename would not survive raw)."""
- def _headers(filename: str, display_name: str | None = None) -> dict[str, str]:
+ def _headers(
+ filename: str, display_name: str | None = None, folder: int | str | None = None
+ ) -> dict[str, str]:
headers = {
"content-type": "application/octet-stream",
"x-upload-filename": quote(filename),
}
if display_name is not None:
headers["x-upload-display-name"] = quote(display_name)
+ if folder is not None:
+ headers["x-upload-folder"] = str(folder)
return headers
return _headers
diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py
index 4d55c4e7..72a66bc9 100644
--- a/backend/tests/test_body_limit.py
+++ b/backend/tests/test_body_limit.py
@@ -221,7 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
CAPPED_BODY_SITES = {
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
- ("downloads.py", "spool_upload", "capped_stream_to", "cap"),
+ ("files.py", "spool_upload", "capped_stream_to", "cap"),
}
@@ -467,7 +467,7 @@ def test_every_body_reading_site_hands_its_pooled_connection_back_first():
assert not missing, (
"a route reads its own request body while still inside its read transaction, pinning a "
f"pooled connection for the whole transfer: {missing}. Call "
- "db.release_read_connection(db) before the await — see routes/downloads.py:spool_upload."
+ "db.release_read_connection(db) before the await — see routes/files.py:spool_upload."
)
@@ -540,7 +540,7 @@ def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, templa
"its dependencies, so FastAPI reads it before the first dependency runs and the raised "
"ceiling is all that stands between an anonymous caller and the whole allowance. Read the "
"body yourself, from a dependency declared AFTER the auth one — see "
- "routes/downloads.py:spool_upload."
+ "routes/files.py:spool_upload."
)
diff --git a/backend/tests/test_download_job_locks.py b/backend/tests/test_download_job_locks.py
index a4f13d6c..e702d0b3 100644
--- a/backend/tests/test_download_job_locks.py
+++ b/backend/tests/test_download_job_locks.py
@@ -5,7 +5,7 @@ unlocked read: two overlapping requests for one job (a double-click, a second ta
gateway timeout) both saw it holding its asset and both released the hold. The count then reaches 0
while another job still holds the asset, and GC or the ready-file delete removes it under that job.
A pause racing a cancel did the same one step later: its unlocked write turned the released job back
-into a holder, which a later delete releases again. `_own_job_for_update` makes the second request
+into a holder, which a later delete releases again. `own_job_for_update` makes the second request
wait and decide on the first one's committed result.
"""
@@ -18,12 +18,8 @@ from sqlalchemy import select
from app.db import SessionLocal
from app.downloads import service
from app.models import DownloadJob, MediaAsset, User
-from app.routes.downloads import (
- _own_job_for_update,
- cancel_download,
- delete_download,
- pause_download,
-)
+from app.routes.downloads import cancel_download, delete_download, pause_download
+from app.routes.files import delete_file, own_job_for_update
_REF = "https://example.com/video"
@@ -59,6 +55,7 @@ def _asset_held_by_two_jobs(db, status="error"):
[
("error", service.cancel, cancel_download, "canceled"),
("error", service.delete, delete_download, None),
+ ("error", service.delete, delete_file, None),
("queued", service.cancel, pause_download, "canceled"),
],
)
@@ -80,7 +77,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
finished.set()
try:
- locked = _own_job_for_update(first, user, job_id)
+ locked = own_job_for_update(first, user, job_id)
other = threading.Thread(target=the_other_request)
other.start()
# The second request waits for the first one's locks instead of deciding on the same row.
diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py
index 5ddbb948..e2d97fe9 100644
--- a/backend/tests/test_download_limits.py
+++ b/backend/tests/test_download_limits.py
@@ -188,7 +188,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
_job(db, u.id, queue_pos=i, status="done")
db.commit()
- monkeypatch.setattr(dl, "_MAX_LIST", 2)
+ monkeypatch.setattr(dl, "MAX_LIST", 2)
rows = dl.list_downloads(user=u, db=db)
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
diff --git a/backend/tests/test_files_api.py b/backend/tests/test_files_api.py
new file mode 100644
index 00000000..dc0955f3
--- /dev/null
+++ b/backend/tests/test_files_api.py
@@ -0,0 +1,432 @@
+"""The Files library API (`routes.files`), DB-lane: folders (list, own create/rename/delete, a
+folder's retention), the file list (done only, by folder, searched, sorted, shared with me), a
+file's rename / move / keep / delete, the upload into a folder, and the file's hash.
+
+Every route is also held to the caller's own rows: another user's folder or file answers like a
+missing one.
+"""
+
+import base64
+import hashlib
+from datetime import datetime, timedelta, timezone
+
+import pytest
+from sqlalchemy import select
+
+from app import sysconfig
+from app.config import settings
+from app.downloads import folders, service
+from app.downloads import upload as uploadmod
+from app.downloads.status import JobStatus
+from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset
+
+NOW = datetime.now(timezone.utc)
+
+
+@pytest.fixture
+def _root(tmp_path, monkeypatch):
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
+ return tmp_path
+
+
+def _file(db, user_id, *, kind="youtube", ref=None, title="A title", size=10, rel=None, **kw):
+ """A done library file: an asset with bytes on disk-path terms, and the user's job over it."""
+ n = db.execute(select(MediaAsset.id)).all()
+ ref = ref or f"vid{len(n):08d}"
+ asset = MediaAsset(
+ source_kind=kind,
+ source_ref=ref,
+ format_sig=f"sig-{ref}",
+ status="ready",
+ rel_path=rel or f"{ref}.mp4",
+ size_bytes=size,
+ ref_count=0,
+ title=title,
+ media_kind="video",
+ container="mp4",
+ )
+ db.add(asset)
+ db.flush()
+ job = DownloadJob(
+ user_id=user_id,
+ asset_id=asset.id,
+ source_kind=kind,
+ source_ref=ref,
+ profile_snapshot={},
+ status=kw.pop("status", JobStatus.DONE),
+ **kw,
+ )
+ db.add(job)
+ db.flush()
+ folders.place(db, job)
+ folders.stamp_expiry(db, job)
+ service.incr_ref_count(db, asset.id)
+ db.commit()
+ return job
+
+
+def _own(db, user_id, name="Saját"):
+ folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name)
+ db.add(folder)
+ db.commit()
+ return folder
+
+
+def _audits(db, action):
+ return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all()
+
+
+# --- folders -------------------------------------------------------------------------------
+
+
+def test_the_folder_list_counts_each_folders_done_files_and_is_the_callers_own(
+ db, make_user, as_user
+):
+ alice, bob = make_user(), make_user()
+ _file(db, alice.id, size=10)
+ _file(db, alice.id, size=5)
+ _file(db, alice.id, kind="url", ref="https://vimeo.com/1", size=7)
+ _file(db, alice.id, status=JobStatus.RUNNING) # not in the library yet
+ _file(db, bob.id)
+ got = {
+ (f["kind"], f["source_key"]): (f["count"], f["size_bytes"])
+ for f in as_user(alice).get("/api/files/folders").json()
+ }
+ assert got == {("source", "youtube"): (2, 15), ("source", "site:vimeo.com"): (1, 7)}
+
+
+def test_an_own_folder_is_made_once_per_name_whatever_its_case(db, make_user, as_user):
+ alice = make_user()
+ c = as_user(alice)
+ made = c.post("/api/files/folders", json={"name": "Nyaralás"})
+ assert made.status_code == 200, made.text
+ assert (made.json()["kind"], made.json()["name"]) == ("own", "Nyaralás")
+ again = c.post("/api/files/folders", json={"name": "NYARALÁS"})
+ assert again.status_code == 409
+ assert again.json()["detail"]["reason"] == "file_folder_name_taken"
+ blank = c.post("/api/files/folders", json={"name": " "})
+ assert blank.json()["detail"]["reason"] == "file_folder_name_required"
+ # Another user may have a folder of the same name.
+ assert as_user(make_user()).post("/api/files/folders", json={"name": "Nyaralás"}).is_success
+ assert len(_audits(db, "file_folder_create")) == 2
+
+
+def test_only_an_own_folder_is_renamed_and_only_by_its_user(db, make_user, as_user):
+ alice = make_user()
+ c = as_user(alice)
+ src = db.get(FileFolder, _file(db, alice.id).folder_id)
+ own = _own(db, alice.id, "Régi")
+ taken = _own(db, alice.id, "Foglalt")
+ ok = c.patch(f"/api/files/folders/{own.id}", json={"name": "Új"})
+ assert ok.json()["name"] == "Új"
+ clash = c.patch(f"/api/files/folders/{own.id}", json={"name": taken.name.lower()})
+ assert clash.json()["detail"]["reason"] == "file_folder_name_taken"
+ no = c.patch(f"/api/files/folders/{src.id}", json={"name": "YouTube 2"})
+ assert no.status_code == 409 and no.json()["detail"]["reason"] == "file_folder_not_own"
+ stranger = as_user(make_user()).patch(f"/api/files/folders/{own.id}", json={"name": "X"})
+ assert stranger.status_code == 404
+ db.refresh(own)
+ assert own.name == "Új"
+
+
+def test_a_shorter_retention_moves_each_files_expiry_but_never_inside_the_warning_window(
+ db, make_user, as_user
+):
+ sysconfig.set_value(db, "download_gc_grace_days", 3)
+ db.commit()
+ alice = make_user()
+ fresh = _file(db, alice.id) # expires in 30 days
+ old = _file(db, alice.id)
+ old.expires_at = NOW + timedelta(days=10) # 20 days old already
+ kept = _file(db, alice.id, kept=True)
+ db.commit()
+ folder_id = fresh.folder_id
+ r = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": 20})
+ assert r.json()["retention_days"] == 20
+ for j in (fresh, old, kept):
+ db.refresh(j)
+ # 30 → 20 days: each loses 10 — the one with 10 left would be due now, so it gets the window.
+ assert abs(fresh.expires_at - (NOW + timedelta(days=20))) < timedelta(minutes=5)
+ assert abs(old.expires_at - (NOW + timedelta(days=3))) < timedelta(minutes=5)
+ assert kept.expires_at is None
+ forever = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": None})
+ assert forever.json()["retention_days"] is None
+ db.refresh(fresh)
+ assert fresh.expires_at is None
+ rows = _audits(db, "file_folder_retention")
+ assert [(a.before, a.after) for a in rows] == [
+ ({"retention_days": 30}, {"retention_days": 20}),
+ ({"retention_days": 20}, {"retention_days": None}),
+ ]
+
+
+def test_a_retention_set_on_a_folder_that_kept_forever_starts_its_files_clock_now(
+ db, make_user, as_user
+):
+ alice = make_user()
+ up = _file(db, alice.id, kind="upload", ref="u1")
+ assert up.expires_at is None
+ r = as_user(alice).patch(f"/api/files/folders/{up.folder_id}", json={"retention_days": 7})
+ assert r.status_code == 200, r.text
+ db.refresh(up)
+ assert abs(up.expires_at - (NOW + timedelta(days=7))) < timedelta(minutes=5)
+
+
+def test_deleting_an_own_folder_sends_its_files_home_and_a_source_folder_cannot_be_deleted(
+ db, make_user, as_user
+):
+ alice = make_user()
+ c = as_user(alice)
+ yt = _file(db, alice.id)
+ site = _file(db, alice.id, kind="url", ref="https://vimeo.com/2")
+ home = {yt.id: yt.folder_id, site.id: site.folder_id}
+ own = _own(db, alice.id)
+ for j in (yt, site):
+ assert c.patch(f"/api/files/{j.id}", json={"folder_id": own.id}).is_success
+ r = c.delete(f"/api/files/folders/{own.id}")
+ assert r.json() == {"ok": True, "deleted": own.id, "files_moved": 2}
+ assert db.get(FileFolder, own.id) is None
+ for j in (yt, site):
+ db.refresh(j)
+ assert j.folder_id == home[j.id]
+ # Back in a source folder, its clock runs again from now.
+ assert abs(j.expires_at - (NOW + timedelta(days=30))) < timedelta(minutes=5)
+ src = c.delete(f"/api/files/folders/{yt.folder_id}")
+ assert src.status_code == 409 and src.json()["detail"]["reason"] == "file_folder_not_own"
+ other = _own(db, alice.id, "Más")
+ assert as_user(make_user()).delete(f"/api/files/folders/{other.id}").status_code == 404
+ assert len(_audits(db, "file_folder_delete")) == 1
+
+
+# --- the file list -------------------------------------------------------------------------
+
+
+def test_the_list_holds_the_callers_done_files_by_folder(db, make_user, as_user):
+ alice, bob = make_user(), make_user()
+ yt = _file(db, alice.id)
+ site = _file(db, alice.id, kind="url", ref="https://vimeo.com/3")
+ _file(db, alice.id, status=JobStatus.QUEUED)
+ _file(db, bob.id)
+ c = as_user(alice)
+ assert {f["id"] for f in c.get("/api/files").json()} == {yt.id, site.id}
+ only = c.get("/api/files", params={"folder_id": site.folder_id}).json()
+ assert [f["id"] for f in only] == [site.id]
+ assert only[0]["folder_id"] == site.folder_id and only[0]["shared"] is False
+ # Bob's folder id names nothing of Alice's.
+ bobs = db.execute(select(FileFolder.id).where(FileFolder.user_id == bob.id)).scalar()
+ assert c.get("/api/files", params={"folder_id": bobs}).json() == []
+
+
+def test_the_list_is_searched_by_name_and_channel_with_wildcards_taken_literally(
+ db, make_user, as_user
+):
+ alice = make_user()
+ hit = _file(db, alice.id, title="Balaton 100% nyár")
+ by_channel = _file(db, alice.id, title="Más", display_uploader="Balatoni csatorna")
+ _file(db, alice.id, title="Balaton 1000 nyár")
+ c = as_user(alice)
+ assert [f["id"] for f in c.get("/api/files", params={"q": "100%"}).json()] == [hit.id]
+ found = {f["id"] for f in c.get("/api/files", params={"q": "balatoni"}).json()}
+ assert found == {by_channel.id}
+
+
+def test_the_list_sorts_by_name_size_or_when_added(db, make_user, as_user):
+ alice = make_user()
+ b = _file(db, alice.id, title="Béka", size=30)
+ a = _file(db, alice.id, title="alma", size=10)
+ z = _file(db, alice.id, title="Zebra", size=20)
+ c = as_user(alice)
+
+ def ids(**params):
+ return [f["id"] for f in c.get("/api/files", params=params).json()]
+
+ assert ids() == [z.id, a.id, b.id] # newest first
+ assert ids(sort="name", order="asc") == [a.id, b.id, z.id] # the reader's alphabet, any case
+ assert ids(sort="size", order="desc") == [b.id, z.id, a.id]
+
+
+def test_shared_with_me_lists_what_others_shared_and_nothing_else(db, make_user, as_user):
+ alice, bob, carol = make_user(), make_user(), make_user()
+ theirs = _file(db, bob.id)
+ _file(db, bob.id) # not shared
+ for_carol = _file(db, bob.id)
+ db.add(DownloadShare(job_id=theirs.id, from_user_id=bob.id, to_user_id=alice.id))
+ db.add(DownloadShare(job_id=for_carol.id, from_user_id=bob.id, to_user_id=carol.id))
+ db.commit()
+ got = as_user(alice).get("/api/files", params={"scope": "shared"}).json()
+ assert [(f["id"], f["shared"]) for f in got] == [(theirs.id, True)]
+
+
+# --- one file ------------------------------------------------------------------------------
+
+
+def test_a_file_is_renamed_kept_and_given_back_to_its_folders_retention(db, make_user, as_user):
+ alice = make_user()
+ job = _file(db, alice.id)
+ c = as_user(alice)
+ assert c.patch(f"/api/files/{job.id}", json={"name": "Új név"}).json()["display_name"] == (
+ "Új név"
+ )
+ kept = c.patch(f"/api/files/{job.id}", json={"kept": True}).json()
+ assert (kept["kept"], kept["expires_at"]) == (True, None)
+ back = c.patch(f"/api/files/{job.id}", json={"kept": False}).json()
+ assert back["kept"] is False and back["expires_at"] is not None
+ # Setting what it already is records nothing.
+ assert c.patch(f"/api/files/{job.id}", json={"kept": False}).status_code == 200
+ assert len(_audits(db, "file_keep")) == 1 and len(_audits(db, "file_unkeep")) == 1
+ assert as_user(make_user()).patch(f"/api/files/{job.id}", json={"kept": True}).status_code == (
+ 404
+ )
+
+
+def test_a_file_moves_to_an_own_folder_or_home_and_nowhere_else(db, make_user, as_user):
+ alice, bob = make_user(), make_user()
+ job = _file(db, alice.id)
+ home = job.folder_id
+ vimeo = db.get(FileFolder, _file(db, alice.id, kind="url", ref="https://vimeo.com/4").folder_id)
+ own = _own(db, alice.id)
+ bobs = _own(db, bob.id, "Bobé")
+ c = as_user(alice)
+ moved = c.patch(f"/api/files/{job.id}", json={"folder_id": own.id}).json()
+ assert (moved["folder_id"], moved["expires_at"]) == (own.id, None) # an own folder keeps it
+ wrong = c.patch(f"/api/files/{job.id}", json={"folder_id": vimeo.id})
+ assert wrong.json()["detail"]["reason"] == "file_folder_not_its_source"
+ foreign = c.patch(f"/api/files/{job.id}", json={"folder_id": bobs.id})
+ assert foreign.json()["detail"]["reason"] == "file_folder_gone"
+ home_again = c.patch(f"/api/files/{job.id}", json={"folder_id": home}).json()
+ assert home_again["folder_id"] == home and home_again["expires_at"] is not None
+ assert [(a.before, a.after) for a in _audits(db, "file_move")] == [
+ ({"folder_id": home}, {"folder_id": own.id}),
+ ({"folder_id": own.id}, {"folder_id": home}),
+ ]
+
+
+def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as_user, _root):
+ alice = make_user()
+ job = _file(db, alice.id)
+ assert as_user(make_user()).delete(f"/api/files/{job.id}").status_code == 404
+ r = as_user(alice).delete(f"/api/files/{job.id}")
+ assert r.json() == {"ok": True, "deleted": job.id}
+ db.expire_all()
+ assert db.get(DownloadJob, job.id) is None
+ [row] = _audits(db, "file_delete")
+ assert (row.actor_id, row.target_id) == (alice.id, str(job.id))
+
+
+# --- upload, the file, its hash ------------------------------------------------------------
+
+
+@pytest.fixture
+def _no_probe(monkeypatch):
+ # A file ffprobe cannot read: a generic file, done at once (no conversion).
+ monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None)
+
+
+def test_an_upload_lands_in_the_folder_it_asked_for_with_its_hash(
+ db, make_user, as_user, _root, _no_probe, upload_headers
+):
+ alice = make_user()
+ own = _own(db, alice.id)
+ body = b"hello files"
+ c = as_user(alice)
+ r = c.post("/api/files/upload", content=body, headers=upload_headers("a.txt", folder=own.id))
+ assert r.status_code == 200, r.text
+ job = db.get(DownloadJob, r.json()["id"])
+ assert job.folder_id == own.id and job.expires_at is None
+ asset = db.get(MediaAsset, job.asset_id)
+ assert asset.sha256 == hashlib.sha256(body).hexdigest()
+ plain = c.post("/api/files/upload", content=b"x", headers=upload_headers("b.txt")).json()
+ assert db.get(FileFolder, plain["folder_id"]).source_key == "upload"
+ assert len(_audits(db, "file_upload")) == 2
+
+
+@pytest.mark.parametrize(
+ ("which", "reason"),
+ [
+ ("site", "file_folder_not_its_source"), # YouTube's folder: no upload came from there
+ ("foreign", "file_folder_gone"),
+ ("garbage", "file_folder_gone"),
+ ],
+)
+def test_an_upload_to_a_folder_it_may_not_use_is_refused_before_its_body_is_read(
+ db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch, which, reason
+):
+ from app.routes import files as files_routes
+
+ alice, bob = make_user(), make_user()
+ folder = {
+ "site": lambda: _file(db, alice.id).folder_id,
+ "foreign": lambda: _own(db, bob.id).id,
+ "garbage": lambda: "12abc",
+ }[which]()
+ streamed = []
+
+ async def spy(*args, **kwargs):
+ streamed.append(True)
+ raise AssertionError("the body was read")
+
+ monkeypatch.setattr(files_routes, "capped_stream_to", spy)
+ before = db.execute(select(DownloadJob.id)).all()
+ r = as_user(alice).post(
+ "/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=folder)
+ )
+ assert (r.status_code, r.json()["detail"]["reason"]) == (409, reason)
+ assert streamed == []
+ assert db.execute(select(DownloadJob.id)).all() == before
+
+
+def test_a_folder_deleted_while_its_upload_streams_refuses_the_upload(
+ db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch
+):
+ from app.routes import files as files_routes
+
+ alice = make_user()
+ own = _own(db, alice.id)
+ real = files_routes.capped_stream_to
+
+ async def deleting_meanwhile(*args, **kwargs):
+ size = await real(*args, **kwargs)
+ db.delete(db.get(FileFolder, own.id))
+ db.commit()
+ return size
+
+ monkeypatch.setattr(files_routes, "capped_stream_to", deleting_meanwhile)
+ r = as_user(alice).post(
+ "/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=own.id)
+ )
+ assert (r.status_code, r.json()["detail"]["reason"]) == (409, "file_folder_gone")
+ assert db.execute(select(DownloadJob.id).where(DownloadJob.user_id == alice.id)).all() == []
+ assert not (_root / ".uploads").exists() # checked before the file was placed
+
+
+def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask(
+ db, make_user, as_user, _root
+):
+ alice, friend = make_user(), make_user()
+ body = b"\x00\x01video bytes"
+ job = _file(db, alice.id, rel="v.mp4")
+ (_root / "v.mp4").write_bytes(body)
+ c = as_user(alice)
+ first = c.get(f"/api/files/{job.id}/file")
+ assert first.status_code == 200 and first.content == body
+ assert "repr-digest" not in first.headers # not hashed yet: nothing claimed
+ want = hashlib.sha256(body).hexdigest()
+ assert c.get(f"/api/files/{job.id}/sha256").json() == {"sha256": want}
+ assert db.get(MediaAsset, job.asset_id).sha256 == want
+ ranged = c.get(f"/api/files/{job.id}/file", headers={"Range": "bytes=0-3"})
+ assert ranged.status_code == 206
+ digest = base64.b64encode(bytes.fromhex(want)).decode()
+ assert ranged.headers["repr-digest"] == f"sha-256=:{digest}:"
+ # A friend it was shared with may read it too; a stranger is told it does not exist.
+ db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id))
+ db.commit()
+ assert as_user(friend).get(f"/api/files/{job.id}/sha256").json() == {"sha256": want}
+ assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404
+
+
+def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user):
+ job = _file(db, make_user().id)
+ asset = db.get(MediaAsset, job.asset_id)
+ asset.sha256 = "ab" * 32
+ asset.rel_path = "converted.mp4"
+ assert asset.sha256 is None
diff --git a/backend/tests/test_migration_0103.py b/backend/tests/test_migration_0103.py
new file mode 100644
index 00000000..0514b45d
--- /dev/null
+++ b/backend/tests/test_migration_0103.py
@@ -0,0 +1,62 @@
+"""Migration 0103's data step, DB-lane: the fileless "expired" rows of the old retention go, every
+other job stays.
+
+An expired file now leaves the library (`downloads.gc`); before, it stayed as a done job with no
+asset. Left in place, those rows would sit unseen in no listing forever.
+"""
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import text
+
+from app.db import engine
+
+
+def test_the_fileless_expired_rows_go_and_every_other_job_stays(db, make_user):
+ uid = make_user().id
+ cfg = Config("alembic.ini")
+ command.downgrade(cfg, "0102_file_folders")
+ try:
+ with engine.begin() as conn:
+ folder = conn.execute(
+ text(
+ "INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
+ " VALUES (:u, 'source', 'youtube', 30) RETURNING id"
+ ),
+ {"u": uid},
+ ).scalar_one()
+ asset = conn.execute(
+ text(
+ "INSERT INTO media_assets (source_kind, source_ref, format_sig, status, ref_count)"
+ " VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 1) RETURNING id"
+ )
+ ).scalar_one()
+
+ def job(status, asset_id):
+ return conn.execute(
+ text(
+ "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
+ " profile_snapshot, status, folder_id) VALUES (:u, :a, 'youtube',"
+ " 'abcdefghijk', '{}', :s, :f) RETURNING id"
+ ),
+ {"u": uid, "a": asset_id, "s": status, "f": folder},
+ ).scalar_one()
+
+ expired = job("done", None)
+ alive = job("done", asset)
+ failed = job("error", None) # a failed job has no file either, and is not expired
+ queued = job("queued", None)
+ finally:
+ command.upgrade(cfg, "head")
+
+ with engine.connect() as conn:
+ left = set(conn.execute(text("SELECT id FROM download_jobs")).scalars())
+ has_hash = conn.execute(
+ text(
+ "SELECT count(*) FROM information_schema.columns"
+ " WHERE table_name = 'media_assets' AND column_name = 'sha256'"
+ )
+ ).scalar()
+ assert left == {alive, failed, queued}
+ assert expired not in left
+ assert has_hash == 1
diff --git a/backend/tests/test_route_conventions.py b/backend/tests/test_route_conventions.py
index 819232b9..4ea0e820 100644
--- a/backend/tests/test_route_conventions.py
+++ b/backend/tests/test_route_conventions.py
@@ -46,7 +46,9 @@ SERIALIZERS = {
"_collection_card",
"_playlist_card",
"_serialize",
- "_serialize_job",
+ "_serialize_folder",
+ "serialize",
+ "serialize_job",
"_serialize_profile",
"_serialize_user",
"_summary",
diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py
index 4d551232..d563033d 100644
--- a/backend/tests/test_upload.py
+++ b/backend/tests/test_upload.py
@@ -584,7 +584,7 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]:
# one value that needs no bound, and a clear is still a write.
# service.enqueue / enqueue_edit / `display_name` is already bounded by its caller; the fallback
# enqueue_clip slices the source's title to the same width.
-# downloads.upload_download passes through `spool_upload`'s value, bounded there.
+# files.ingest_spooled passes through `spool_upload`'s value, bounded there.
# routes enqueue_clip the `display_name=` keyword passes on the name bound above it.
# pipeline._download / _process_clip / worker._finish_from_cache
# slice a fetched or stored title to the column's width.
@@ -614,8 +614,9 @@ DISPLAY_WRITE_SITES = {
"display_text(body.display_name)",
"display_text(body.display_uploader)",
},
- ("routes/downloads.py", "upload_download"): {"upload.display_name"},
- ("routes/downloads.py", "spool_upload"): {
+ ("routes/files.py", "ingest_spooled"): {"upload.display_name"},
+ ("routes/files.py", "update_file"): {"display_text(body.name)"},
+ ("routes/files.py", "spool_upload"): {
"display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER))"
},
}
@@ -694,12 +695,12 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams(
the checkout is its consequence (measured), and the pool counter is shared with whatever else
the test process is holding.
"""
- from app.routes import downloads as downloads_routes
+ from app.routes import files as files_routes
_probe(monkeypatch, (None, None, None, None, None))
seen: dict[str, bool] = {}
- real_stream = downloads_routes.capped_stream_to
- real_release = downloads_routes.release_read_connection
+ real_stream = files_routes.capped_stream_to
+ real_release = files_routes.release_read_connection
def spy_release(session):
# The PRECONDITION, so this test cannot pass vacuously. Asserting only "no transaction
@@ -714,8 +715,8 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams(
seen["while_streaming"] = bool(db.in_transaction())
return await real_stream(request, out, limit, refusal)
- monkeypatch.setattr(downloads_routes, "release_read_connection", spy_release)
- monkeypatch.setattr(downloads_routes, "capped_stream_to", spy_stream)
+ monkeypatch.setattr(files_routes, "release_read_connection", spy_release)
+ monkeypatch.setattr(files_routes, "capped_stream_to", spy_stream)
c = as_user(make_user())
r = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("held.bin"))
assert r.status_code == 200, r.text
diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts
index e28c5a42..96269317 100644
--- a/frontend/src/i18n/backendIds.contract.test.ts
+++ b/frontend/src/i18n/backendIds.contract.test.ts
@@ -141,7 +141,7 @@ const goneWhats = [
];
// Every generic refusal → `errors.refused.`: each `refused(, "")` anywhere in
// the backend (not `_refused`, a router's own family), plus the values of the settings map, which
-// `setting_refused` picks from at runtime.
+// `setting_refused` picks from at runtime, and every `FolderError` reason.
const appSources = (dir = ""): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
withFileTypes: true,
@@ -157,6 +157,11 @@ const refusedReasons = [
...appSources().flatMap((src) =>
[...src.matchAll(/(? m[1]!),
),
+ // A folder change the rules refuse travels as `FolderError("")` and is answered with
+ // `refused(409, e.reason)`, which the pattern above cannot read.
+ ...appSources().flatMap((src) =>
+ [...src.matchAll(/FolderError\("(\w+)"\)/g)].map((m) => m[1]!),
+ ),
...[
...(backend("refusal.py").split("_SETTING_REASONS = {")[1]?.split("}")[0] ?? "").matchAll(
/:\s*"(\w+)"/g,
diff --git a/frontend/src/i18n/locales/en/auditActions.json b/frontend/src/i18n/locales/en/auditActions.json
index 08e722a8..c12f7b0a 100644
--- a/frontend/src/i18n/locales/en/auditActions.json
+++ b/frontend/src/i18n/locales/en/auditActions.json
@@ -22,5 +22,14 @@
"download_credential_delete": "Download cookies deleted",
"invite_delete": "Whitelist entry deleted",
"invite_send": "Invitation sent",
- "issue_create": "Bug report sent"
+ "issue_create": "Bug report sent",
+ "file_upload": "File uploaded",
+ "file_delete": "File deleted",
+ "file_keep": "File kept",
+ "file_unkeep": "File no longer kept",
+ "file_move": "File moved",
+ "file_folder_create": "Folder created",
+ "file_folder_rename": "Folder renamed",
+ "file_folder_retention": "Folder retention changed",
+ "file_folder_delete": "Folder deleted"
}
diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json
index 239658fb..4e23c8ff 100644
--- a/frontend/src/i18n/locales/en/errors.json
+++ b/frontend/src/i18n/locales/en/errors.json
@@ -37,7 +37,8 @@
"poster": "There's no poster for this",
"profile": "This download profile doesn't exist any more",
"season": "This season isn't in the library any more",
- "show": "This show isn't in the library any more"
+ "show": "This show isn't in the library any more",
+ "folder": "That folder no longer exists"
},
"admin": {
"not_allowed": "Their address isn't approved any more. Add it to the whitelist first.",
@@ -219,6 +220,11 @@
"screenshot_storage_full": "You've reached the {{limit}} MB screenshot allowance. Remove images from older sessions, or wait until they expire.",
"issue_sending": "The report is being sent right now. Wait for it to finish.",
"issue_rate_limited": "At most {{limit}} send attempts an hour. Try again later.",
- "preview_rate_limited": "Too many previews in a row. Wait a minute."
+ "preview_rate_limited": "Too many previews in a row. Wait a minute.",
+ "file_folder_name_required": "Give the folder a name",
+ "file_folder_name_taken": "You already have a folder with that name",
+ "file_folder_not_own": "Only a folder you made can be renamed or deleted",
+ "file_folder_gone": "That folder no longer exists",
+ "file_folder_not_its_source": "A file can only go into one of your own folders or back to the folder it came from"
}
}
diff --git a/frontend/src/i18n/locales/hu/auditActions.json b/frontend/src/i18n/locales/hu/auditActions.json
index 4c1acc37..3ef023cc 100644
--- a/frontend/src/i18n/locales/hu/auditActions.json
+++ b/frontend/src/i18n/locales/hu/auditActions.json
@@ -22,5 +22,14 @@
"download_credential_delete": "Letöltési cookie-k törölve",
"invite_delete": "Fehérlista-bejegyzés törölve",
"invite_send": "Meghívó elküldve",
- "issue_create": "Hibajegy elküldve"
+ "issue_create": "Hibajegy elküldve",
+ "file_upload": "Fájl feltöltve",
+ "file_delete": "Fájl törölve",
+ "file_keep": "Fájl megtartva",
+ "file_unkeep": "Fájl megtartása visszavonva",
+ "file_move": "Fájl áthelyezve",
+ "file_folder_create": "Mappa létrehozva",
+ "file_folder_rename": "Mappa átnevezve",
+ "file_folder_retention": "Mappa megőrzési ideje módosítva",
+ "file_folder_delete": "Mappa törölve"
}
diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json
index 0b9125e7..1a054ce7 100644
--- a/frontend/src/i18n/locales/hu/errors.json
+++ b/frontend/src/i18n/locales/hu/errors.json
@@ -37,7 +37,8 @@
"poster": "Ehhez nincs poszter",
"profile": "Ez a letöltési profil már nem létezik",
"season": "Ez az évad már nincs a könyvtárban",
- "show": "Ez a sorozat már nincs a könyvtárban"
+ "show": "Ez a sorozat már nincs a könyvtárban",
+ "folder": "Ez a mappa már nem létezik"
},
"admin": {
"not_allowed": "A címe már nincs jóváhagyva. Előbb vedd fel a fehérlistára.",
@@ -219,6 +220,11 @@
"screenshot_storage_full": "Elérted a képernyőképek {{limit}} MB-os keretét. Törölj régebbi munkamenetekből képeket, vagy várd meg, amíg lejárnak.",
"issue_sending": "A hibajegy küldése épp folyamatban van. Várd meg, amíg befejeződik.",
"issue_rate_limited": "Óránként legfeljebb {{limit}} küldési kísérlet. Próbáld újra később.",
- "preview_rate_limited": "Túl sok előnézet egymás után. Várj egy percet."
+ "preview_rate_limited": "Túl sok előnézet egymás után. Várj egy percet.",
+ "file_folder_name_required": "Adj nevet a mappának",
+ "file_folder_name_taken": "Már van ilyen nevű mappád",
+ "file_folder_not_own": "Csak a saját mappádat nevezheted át vagy törölheted",
+ "file_folder_gone": "Ez a mappa már nem létezik",
+ "file_folder_not_its_source": "Egy fájl csak a saját mappáidba vagy vissza az eredeti mappájába kerülhet"
}
}
From 85aa917b387531353685130160aa9ec05cf5d7fb Mon Sep 17 00:00:00 2001
From: npeter83
Date: Sun, 11 Oct 2026 05:34:25 +0200
Subject: [PATCH 05/20] =?UTF-8?q?feat(files):=20the=20Files=20page=20?=
=?UTF-8?q?=E2=80=94=20the=20library=20in=20one=20aligned=20list,=20its=20?=
=?UTF-8?q?folders=20in=20a=20rail?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Files is the one library: every done file, the ones shared with me, or one
folder's, picked in a side rail (sources, own folders, the storage meter) and
mirrored to ?folder=. One table with every kind of value in its own column;
Copy link, Share and Save in sight, every action on a right click: play,
cut, rename in the row, move to an own folder or back home, keep, details,
re-download, delete.
The Downloads page keeps only the queue until the tray replaces it. The old
/api/downloads library routes are gone: list shared, upload, the details
PATCH, delete, the file and the usage now live under /api/files, and a
listed file says which source it came from. The job's "expired" flag is gone
with the state it described.
---
backend/app/bodylimit.py | 3 +-
backend/app/logs.py | 4 +-
backend/app/models.py | 2 +-
backend/app/routes/downloads.py | 158 +-----
backend/app/routes/files.py | 126 ++++-
backend/tests/test_body_limit.py | 4 +-
backend/tests/test_download_job_locks.py | 3 +-
backend/tests/test_download_transitions.py | 2 +-
backend/tests/test_files_api.py | 76 +++
backend/tests/test_library_progress.py | 26 +-
backend/tests/test_phone_safe_contract.py | 4 +-
backend/tests/test_public_unlock_limit.py | 4 +-
backend/tests/test_spa_entries.py | 2 +-
backend/tests/test_upload.py | 67 ++-
backend/tests/test_upload_conversion.py | 4 +-
frontend/src/App.tsx | 25 +-
.../DownloadCenter.confirm.dom.test.tsx | 37 +-
.../DownloadCenter.menu.dom.test.tsx | 164 +-----
frontend/src/components/DownloadCenter.tsx | 152 +-----
.../src/components/Files.menu.dom.test.tsx | 397 +++++++++++++++
frontend/src/components/Files.tsx | 478 ++++++++++++++++++
frontend/src/components/FilesProvider.tsx | 55 ++
frontend/src/components/FilesRail.tsx | 187 +++++++
frontend/src/components/Header.tsx | 12 +
.../src/components/LibraryPlayer.dom.test.tsx | 22 +-
frontend/src/components/LibraryPlayer.tsx | 27 +-
.../src/components/UploadDialog.dom.test.tsx | 72 ++-
frontend/src/components/UploadDialog.tsx | 23 +-
frontend/src/components/VideoEditor.tsx | 6 +-
.../WriteFailureReporting.dom.test.tsx | 8 +-
frontend/src/components/copyItems.tsx | 7 +-
frontend/src/components/downloadModals.tsx | 22 +-
.../src/components/downloadRow.dom.test.tsx | 1 -
frontend/src/components/downloadRow.tsx | 60 +--
frontend/src/components/fileFolders.tsx | 35 ++
frontend/src/components/filesContext.ts | 47 ++
frontend/src/components/moduleRegistry.tsx | 11 +-
frontend/src/components/navigationInit.ts | 1 +
.../ui/pressableSelectable.contract.test.ts | 1 +
.../src/components/useDownloadActions.tsx | 231 +--------
frontend/src/components/useFileActions.tsx | 303 +++++++++++
frontend/src/i18n/locales/en/downloads.json | 16 +-
frontend/src/i18n/locales/en/errors.json | 8 +-
frontend/src/i18n/locales/en/files.json | 60 +++
frontend/src/i18n/locales/en/keys.json | 1 +
frontend/src/i18n/locales/en/nav.json | 1 +
frontend/src/i18n/locales/hu/downloads.json | 16 +-
frontend/src/i18n/locales/hu/errors.json | 8 +-
frontend/src/i18n/locales/hu/files.json | 60 +++
frontend/src/i18n/locales/hu/keys.json | 1 +
frontend/src/i18n/locales/hu/nav.json | 1 +
frontend/src/lib/api.ts | 3 +
frontend/src/lib/api/core.test.ts | 19 +-
frontend/src/lib/api/downloads.ts | 65 +--
frontend/src/lib/api/facade.test.ts | 1 +
frontend/src/lib/api/files.ts | 123 +++++
frontend/src/lib/downloads.ts | 9 +-
frontend/src/lib/keymap.ts | 1 +
frontend/src/lib/modules.ts | 7 +
frontend/src/lib/pageMeta.test.ts | 1 +
frontend/src/lib/panelLayout.ts | 12 +-
frontend/src/lib/queryKeys.ts | 9 +-
frontend/src/lib/storage.ts | 6 +
frontend/src/lib/urlState.ts | 1 +
frontend/src/main.tsx | 41 +-
65 files changed, 2295 insertions(+), 1044 deletions(-)
create mode 100644 frontend/src/components/Files.menu.dom.test.tsx
create mode 100644 frontend/src/components/Files.tsx
create mode 100644 frontend/src/components/FilesProvider.tsx
create mode 100644 frontend/src/components/FilesRail.tsx
create mode 100644 frontend/src/components/fileFolders.tsx
create mode 100644 frontend/src/components/filesContext.ts
create mode 100644 frontend/src/components/useFileActions.tsx
create mode 100644 frontend/src/i18n/locales/en/files.json
create mode 100644 frontend/src/i18n/locales/hu/files.json
create mode 100644 frontend/src/lib/api/files.ts
diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py
index 134a5458..c0ecab4c 100644
--- a/backend/app/bodylimit.py
+++ b/backend/app/bodylimit.py
@@ -92,8 +92,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
- # sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
- ("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
+ # sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports.
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
}
diff --git a/backend/app/logs.py b/backend/app/logs.py
index ecafc7d2..62ff3609 100644
--- a/backend/app/logs.py
+++ b/backend/app/logs.py
@@ -690,7 +690,9 @@ _POLLED_ROUTES = frozenset(
"/api/me/notifications/unread_count",
"/api/downloads",
"/api/downloads/index",
- "/api/downloads/usage",
+ "/api/files",
+ "/api/files/folders",
+ "/api/files/usage",
"/api/sync/my-status",
"/api/admin/scheduler",
"/api/admin/downloads",
diff --git a/backend/app/models.py b/backend/app/models.py
index fd863d71..e1136a22 100644
--- a/backend/app/models.py
+++ b/backend/app/models.py
@@ -39,7 +39,7 @@ def display_text(raw: str | None) -> str | None:
blank -> None.
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
- `POST /api/downloads/edit`, `POST /api/downloads/upload`, the metadata PATCH's two fields, and
+ `POST /api/downloads/edit`, `POST /api/files/upload`, the library PATCH's two fields, and
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
than refusing: the name is a label on work the user has already done — a file already
transferred, an account already authenticated — and losing its tail is not worth failing the
diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py
index 63300b2a..f50e4162 100644
--- a/backend/app/routes/downloads.py
+++ b/backend/app/routes/downloads.py
@@ -2,7 +2,8 @@
All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the
frontend (the worker is a separate process); this module never downloads — it enqueues, manages job
-state, serves finished files (range-aware, with the user's custom display name), and shares them.
+state, and shares finished files. The library itself — listing, renaming, moving, deleting and
+serving a done file, and uploading one — is `routes.files`.
"""
import re
@@ -50,18 +51,13 @@ from app.routes._common import (
)
from app.routes.files import (
MAX_LIST,
- SpooledUpload,
accessible_job,
assets_for,
- delete_file,
own_job_for_update,
positions_for,
quota_detail,
serialize,
serialize_job,
- serve_file,
- spool_upload,
- ingest_spooled,
)
from app.security import hash_password
from app.userscope import is_messageable_user, messageable_clauses
@@ -328,16 +324,6 @@ def enqueue_download(
return serialize_job(db, job)
-@router.post("/upload")
-def upload_download(
- upload: SpooledUpload = Depends(spool_upload),
- db: Session = Depends(get_db),
-) -> dict:
- """The Files library's upload (`routes.files.upload_file`) at its old address, until the client
- moves to the new one."""
- return serialize_job(db, ingest_spooled(db, upload))
-
-
class EnqueueEditIn(BaseModel):
source_job_id: int = 0
edit_spec: dict
@@ -433,24 +419,6 @@ def list_downloads(user: User = Depends(current_user), db: Session = Depends(get
]
-@router.get("/usage")
-def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
- """The caller's download quota, plus the per-file upload cap.
-
- The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
- `sysconfig` knob about one route — and this is the route layer, where composing the two is the
- job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
- caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
- `bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
- limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
- boundary: the caps stay where they are and still refuse a client that ignores this.
- """
- return {
- **quota.usage(db, user.id),
- "upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
- }
-
-
# Priority when a source has several jobs (e.g. a done one plus a fresh queued one).
_INDEX_RANK: dict[str, int] = {
JobStatus.DONE: 4,
@@ -477,90 +445,6 @@ def my_download_index(user: User = Depends(current_user), db: Session = Depends(
return out
-@router.get("/shared")
-def shared_with_me(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
- jobs = (
- db.execute(
- select(DownloadJob)
- .join(DownloadShare, DownloadShare.job_id == DownloadJob.id)
- .where(DownloadShare.to_user_id == user.id)
- .order_by(DownloadShare.created_at.desc(), DownloadJob.id.desc())
- .limit(MAX_LIST)
- )
- .scalars()
- .all()
- )
- assets = assets_for(db, jobs)
- positions = positions_for(db, user, jobs)
- out = []
- for j in jobs:
- data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
- data["shared"] = True
- data["position_seconds"] = positions.get(j.id, 0)
- out.append(data)
- return out
-
-
-_MAX_EXTRA_LINKS = 8
-
-
-def _clean_url(raw) -> str | None:
- """Accept only a plausible http(s) URL (these get rendered as clickable links + embedded in a
- public watch page); anything else is dropped rather than trusted."""
- if not isinstance(raw, str):
- return None
- url = raw.strip()
- if not url:
- return None
- if not url.startswith(("http://", "https://")):
- return None
- # The third site in this module handing user input to `urlparse`, which raises on a malformed
- # authority. This one already has a "not a plausible URL" answer, so the typo belongs in it.
- try:
- parsed = urlparse(url)
- # silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
- # answer, the caller already has one, and any line here would carry the URL itself.
- except ValueError:
- return None
- if not parsed.netloc:
- return None
- return url[:2048]
-
-
-class UpdateDownloadMetaIn(BaseModel):
- # PATCH display-only overrides; presence (model_fields_set) decides what's touched. A field
- # set to null/"" clears the override (falls back to the on-disk system value).
- display_name: str | None = None
- display_uploader: str | None = None
- display_uploader_url: str | None = None
- extra_links: list | None = None
-
-
-@router.patch("/{job_id}")
-def update_download_meta(
- job_id: int,
- body: UpdateDownloadMetaIn,
- user: User = Depends(current_user),
- db: Session = Depends(get_db),
-) -> dict:
- """Update a download's display metadata: title, channel name + link, and extra reference URLs.
- All are display-only overrides — the on-disk file keeps its system-decided, id-bound name."""
- job = owned_or_404(db, DownloadJob, job_id, user, what="download")
- sent = body.model_fields_set
- if "display_name" in sent:
- job.display_name = display_text(body.display_name)
- if "display_uploader" in sent:
- job.display_uploader = display_text(body.display_uploader)
- if "display_uploader_url" in sent:
- job.display_uploader_url = _clean_url(body.display_uploader_url)
- if "extra_links" in sent:
- raw = body.extra_links or []
- cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
- job.extra_links = cleaned or None
- db.commit()
- return serialize_job(db, job)
-
-
@router.post("/{job_id}/pause")
def pause_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
@@ -615,26 +499,6 @@ def cancel_download(
return serialize_job(db, job)
-@router.delete("/{job_id}")
-def delete_download(
- job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
-) -> dict:
- """The Files library's delete (`routes.files.delete_file`, audited) at its old address, until the
- client moves to the new one."""
- delete_file(job_id, user, db)
- return {"ok": True, "deleted": job_id}
-
-
-# --- file download (range-aware, custom display name) --------------------------------------
-
-
-@router.get("/{job_id}/file")
-def download_file(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
- response = serve_file(db, accessible_job(db, user, job_id))
- db.commit()
- return response
-
-
# --- editor filmstrip (scrub track) --------------------------------------------------------
@@ -807,24 +671,6 @@ def unshare_download(
return {"ok": True}
-@router.delete("/shared/{job_id}")
-def remove_shared_with_me(
- job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
-) -> dict:
- """Remove a download that was shared WITH this user from their 'Shared with me' list. Deletes
- only the recipient's share grant — the owner's job and the physical file are untouched (this is
- a per-user dismissal, not a delete)."""
- row = db.execute(
- select(DownloadShare).where(
- DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
- )
- ).scalar_one_or_none()
- if row is not None:
- db.delete(row)
- db.commit()
- return {"ok": True, "removed": job_id}
-
-
# --- recipients (for the internal "share with a user" picker) ------------------------------
diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py
index c15d9bd7..7c33a86e 100644
--- a/backend/app/routes/files.py
+++ b/backend/app/routes/files.py
@@ -18,7 +18,7 @@ from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import BinaryIO, Literal
-from urllib.parse import unquote
+from urllib.parse import unquote, urlparse
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import FileResponse
@@ -95,7 +95,6 @@ def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
"source_job_id": job.source_job_id,
"edit_spec": job.edit_spec,
"can_download": ready and job.status == JobStatus.DONE,
- "expired": job.asset_id is None and job.status == JobStatus.DONE,
# The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
"folder_id": job.folder_id,
"kept": job.kept,
@@ -468,6 +467,42 @@ def delete_folder(
return {"ok": True, "deleted": folder_id, "files_moved": moved}
+@router.get("/usage")
+def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
+ """The caller's storage quota, plus the per-file upload cap.
+
+ The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
+ `sysconfig` knob about one route — and this is the route layer, where composing the two is the
+ job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
+ caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
+ `bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
+ limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
+ boundary: the caps stay where they are and still refuse a client that ignores this.
+ """
+ return {
+ **quota.usage(db, user.id),
+ "upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
+ }
+
+
+@router.delete("/shared/{job_id}")
+def remove_shared_with_me(
+ job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
+) -> dict:
+ """Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes
+ only the recipient's share grant — the owner's job and the physical file are untouched (this is
+ a per-user dismissal, not a delete)."""
+ row = db.execute(
+ select(DownloadShare).where(
+ DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
+ )
+ ).scalar_one_or_none()
+ if row is not None:
+ db.delete(row)
+ db.commit()
+ return {"ok": True, "removed": job_id}
+
+
# --- files ---------------------------------------------------------------------------------
@@ -489,7 +524,8 @@ def list_files(
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.status == JobStatus.DONE)
)
- if scope == "shared":
+ shared = scope == "shared"
+ if shared:
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
DownloadShare.to_user_id == user.id
)
@@ -507,8 +543,10 @@ def list_files(
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
)
)
+ # A file shared with the caller was added to THEIR list when it was shared, not when its owner
+ # downloaded it.
key = {
- "added": DownloadJob.created_at,
+ "added": DownloadShare.created_at if shared else DownloadJob.created_at,
"name": collated(name, lang_for_user(user)),
"size": MediaAsset.size_bytes,
}[sort]
@@ -517,20 +555,83 @@ def list_files(
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
assets = assets_for(db, jobs)
positions = positions_for(db, user, jobs)
- shared = scope == "shared"
+ shared_at = _shared_at(db, user, jobs) if shared else {}
+ sources = _source_keys(db, jobs)
return [
{
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
"position_seconds": positions.get(j.id, 0),
"shared": shared,
+ "shared_at": iso(shared_at.get(j.id)),
+ "source_key": sources[j.id],
}
for j in jobs
]
+def _source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
+ """Where each file came from (its source folder's key), for the list's Source column and the
+ "back to its source folder" move: a file in a source folder has its folder's key; one moved into
+ an own folder (or someone else's, shared) is traced back by `folders.source_key_for`."""
+ ids = {j.folder_id for j in jobs if j.folder_id is not None}
+ by_id = (
+ {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
+ if ids
+ else {}
+ )
+ out: dict[int, str] = {}
+ for j in jobs:
+ folder = by_id.get(j.folder_id) if j.folder_id is not None else None
+ if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
+ out[j.id] = folder.source_key
+ else:
+ out[j.id] = folders.source_key_for(db, j)
+ return out
+
+
+def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
+ rows = db.execute(
+ select(DownloadShare.job_id, DownloadShare.created_at).where(
+ DownloadShare.to_user_id == user.id,
+ DownloadShare.job_id.in_([j.id for j in jobs]),
+ )
+ ).all()
+ return {job_id: at for job_id, at in rows}
+
+
+_MAX_EXTRA_LINKS = 8
+
+
+def _clean_url(raw) -> str | None:
+ """Accept only a plausible http(s) URL (these get rendered as clickable links + embedded in a
+ public watch page); anything else is dropped rather than trusted."""
+ if not isinstance(raw, str):
+ return None
+ url = raw.strip()
+ if not url:
+ return None
+ if not url.startswith(("http://", "https://")):
+ return None
+ # `urlparse` raises on a malformed authority; this function already has a "not a plausible URL"
+ # answer, so the typo belongs in it.
+ try:
+ parsed = urlparse(url)
+ # silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
+ # answer, the caller already has one, and any line here would carry the URL itself.
+ except ValueError:
+ return None
+ if not parsed.netloc:
+ return None
+ return url[:2048]
+
+
class UpdateFileIn(BaseModel):
- # Presence (model_fields_set) decides what is touched; a null/"" name falls back to the title.
+ # Presence (model_fields_set) decides what is touched; a null/"" text falls back to the system
+ # value (the title, the channel). All display-only: the file on disk keeps its own name.
name: str | None = None
+ uploader: str | None = None
+ uploader_url: str | None = None
+ extra_links: list | None = None
folder_id: int | None = None
kept: bool | None = None
@@ -542,12 +643,21 @@ def update_file(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
- """Rename a file (its display name; the file on disk keeps its own), move it to another folder
- (`folders.move`), or keep it until deleted / hand it back to its folder's retention."""
+ """Rename a file and its channel, set its reference links (all display-only: the file on disk
+ keeps its own name), move it to another folder (`folders.move`), or keep it until deleted /
+ hand it back to its folder's retention."""
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
sent = body.model_fields_set
if "name" in sent:
job.display_name = display_text(body.name)
+ if "uploader" in sent:
+ job.display_uploader = display_text(body.uploader)
+ if "uploader_url" in sent:
+ job.display_uploader_url = _clean_url(body.uploader_url)
+ if "extra_links" in sent:
+ raw = body.extra_links or []
+ cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
+ job.extra_links = cleaned or None
if "folder_id" in sent:
if body.folder_id is None:
raise refused(400, "file_folder_gone")
diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py
index 72a66bc9..dc308d01 100644
--- a/backend/tests/test_body_limit.py
+++ b/backend/tests/test_body_limit.py
@@ -367,7 +367,7 @@ def test_every_body_reading_site_is_known_and_fits_under_its_ceiling():
assert ("POST", "/api/debug-sessions/{sid}/screenshots") in bodylimit.ROUTE_MAX_BODY_BYTES
# The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather
# than being a constant to compare against — that pairing is its own test below.
- assert ("POST", "/api/downloads/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
+ assert ("POST", "/api/files/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
def test_release_read_connection_ends_the_transaction_it_was_called_for(db):
@@ -487,7 +487,7 @@ def test_the_upload_ceiling_and_the_admin_knob_cannot_drift_apart():
`errors.tooLarge` exists and why the dialog checks the file against `upload_max_bytes` first.
"""
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
- edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/downloads/upload")]
+ edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/files/upload")]
assert spec.max == edge
diff --git a/backend/tests/test_download_job_locks.py b/backend/tests/test_download_job_locks.py
index e702d0b3..ada7e477 100644
--- a/backend/tests/test_download_job_locks.py
+++ b/backend/tests/test_download_job_locks.py
@@ -18,7 +18,7 @@ from sqlalchemy import select
from app.db import SessionLocal
from app.downloads import service
from app.models import DownloadJob, MediaAsset, User
-from app.routes.downloads import cancel_download, delete_download, pause_download
+from app.routes.downloads import cancel_download, pause_download
from app.routes.files import delete_file, own_job_for_update
_REF = "https://example.com/video"
@@ -54,7 +54,6 @@ def _asset_held_by_two_jobs(db, status="error"):
("status", "transition", "route", "left"),
[
("error", service.cancel, cancel_download, "canceled"),
- ("error", service.delete, delete_download, None),
("error", service.delete, delete_file, None),
("queued", service.cancel, pause_download, "canceled"),
],
diff --git a/backend/tests/test_download_transitions.py b/backend/tests/test_download_transitions.py
index 67aab12f..212e22d6 100644
--- a/backend/tests/test_download_transitions.py
+++ b/backend/tests/test_download_transitions.py
@@ -77,7 +77,7 @@ def test_resume_resets_a_failed_shared_asset_to_pending(db, as_user):
def test_delete_releases_and_removes_the_job(db, as_user):
u, asset, job = _seed(db, job_status="done", asset_status="ready", ref_count=1, rel_path=None)
- r = as_user(u).request("DELETE", f"/api/downloads/{job.id}")
+ r = as_user(u).request("DELETE", f"/api/files/{job.id}")
assert r.status_code == 200, r.text
db.expire_all()
assert db.get(DownloadJob, job.id) is None
diff --git a/backend/tests/test_files_api.py b/backend/tests/test_files_api.py
index dc0955f3..b0de520a 100644
--- a/backend/tests/test_files_api.py
+++ b/backend/tests/test_files_api.py
@@ -257,6 +257,54 @@ def test_shared_with_me_lists_what_others_shared_and_nothing_else(db, make_user,
assert [(f["id"], f["shared"]) for f in got] == [(theirs.id, True)]
+def test_shared_with_me_is_ordered_by_when_it_was_shared_and_can_be_dismissed(
+ db, make_user, as_user
+):
+ """A friend's old file shared today is new to me: "added" is the share's date there. Dismissing
+ one drops only my grant; the owner's file stays."""
+ alice, bob = make_user(), make_user()
+ old = _file(db, bob.id, created_at=NOW - timedelta(days=30))
+ new = _file(db, bob.id, created_at=NOW - timedelta(days=1))
+ db.add(
+ DownloadShare(
+ job_id=new.id,
+ from_user_id=bob.id,
+ to_user_id=alice.id,
+ created_at=NOW - timedelta(hours=2),
+ )
+ )
+ db.add(
+ DownloadShare(
+ job_id=old.id,
+ from_user_id=bob.id,
+ to_user_id=alice.id,
+ created_at=NOW - timedelta(hours=1),
+ )
+ )
+ db.commit()
+ c = as_user(alice)
+ got = c.get("/api/files", params={"scope": "shared"}).json()
+ assert [f["id"] for f in got] == [old.id, new.id]
+ assert got[0]["shared_at"] is not None and got[0]["shared_at"] > got[1]["shared_at"]
+ r = c.delete(f"/api/files/shared/{old.id}")
+ assert r.json() == {"ok": True, "removed": old.id}
+ assert [f["id"] for f in c.get("/api/files", params={"scope": "shared"}).json()] == [new.id]
+ assert db.get(DownloadJob, old.id) is not None
+
+
+def test_each_listed_file_says_where_it_came_from_wherever_it_sits(db, make_user, as_user):
+ """The Source column: a file's own source folder, also after it moved into an own folder."""
+ alice = make_user()
+ yt = _file(db, alice.id)
+ site = _file(db, alice.id, kind="url", ref="https://www.tiktok.com/@x/video/1")
+ moved = _file(db, alice.id)
+ own = _own(db, alice.id)
+ c = as_user(alice)
+ assert c.patch(f"/api/files/{moved.id}", json={"folder_id": own.id}).status_code == 200
+ got = {f["id"]: f["source_key"] for f in c.get("/api/files").json()}
+ assert got == {yt.id: "youtube", site.id: "site:tiktok.com", moved.id: "youtube"}
+
+
# --- one file ------------------------------------------------------------------------------
@@ -279,6 +327,34 @@ def test_a_file_is_renamed_kept_and_given_back_to_its_folders_retention(db, make
)
+def test_a_files_channel_and_links_are_set_and_only_plausible_links_are_kept(
+ db, make_user, as_user
+):
+ alice = make_user()
+ job = _file(db, alice.id)
+ c = as_user(alice)
+ got = c.patch(
+ f"/api/files/{job.id}",
+ json={
+ "uploader": " Saját csatorna ",
+ "uploader_url": "mailto:someone@example.com",
+ "extra_links": ["https://a.example/x", "ftp://b.example", "", 7, "http://[::1/x"]
+ + [f"https://c.example/{i}" for i in range(10)],
+ },
+ ).json()
+ assert got["display_uploader"] == "Saját csatorna"
+ assert got["display_uploader_url"] is None
+ assert got["extra_links"][0] == "https://a.example/x" and len(got["extra_links"]) == 8
+ cleared = c.patch(f"/api/files/{job.id}", json={"uploader": "", "extra_links": []}).json()
+ assert (cleared["display_uploader"], cleared["extra_links"]) == (None, [])
+
+
+def test_the_usage_carries_the_quota_and_the_upload_cap(db, make_user, as_user):
+ got = as_user(make_user()).get("/api/files/usage").json()
+ assert got["upload_max_bytes"] == sysconfig.get_int(db, "download_upload_max_bytes")
+ assert "footprint_bytes" in got and "max_bytes" in got
+
+
def test_a_file_moves_to_an_own_folder_or_home_and_nowhere_else(db, make_user, as_user):
alice, bob = make_user(), make_user()
job = _file(db, alice.id)
diff --git a/backend/tests/test_library_progress.py b/backend/tests/test_library_progress.py
index 12228e39..3c033f05 100644
--- a/backend/tests/test_library_progress.py
+++ b/backend/tests/test_library_progress.py
@@ -9,13 +9,26 @@ cannot reach, is ever read or written. DB-backed (real Postgres).
import pytest
from sqlalchemy import select
-from app.models import DownloadJob, DownloadShare, LibraryState
+from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
from app.routes._common import ProgressIn, kept_position
def _done_job(db, user) -> DownloadJob:
+ asset = MediaAsset(
+ source_kind="url",
+ source_ref="https://example.com/a.mp4",
+ format_sig=f"sig-{user.id}",
+ status="ready",
+ rel_path=f"a-{user.id}.mp4",
+ ref_count=1,
+ media_kind="video",
+ container="mp4",
+ )
+ db.add(asset)
+ db.flush()
job = DownloadJob(
user_id=user.id,
+ asset_id=asset.id,
source_kind="url",
source_ref="https://example.com/a.mp4",
status="done",
@@ -59,7 +72,7 @@ def test_a_checkpoint_is_listed_back_on_the_item(db, make_user, as_user):
"position_seconds": 754,
}
- [item] = c.get("/api/downloads").json()
+ [item] = c.get("/api/files").json()
assert item["position_seconds"] == 754
# A second checkpoint moves it rather than adding a row.
assert _checkpoint(c, job.id, 900) == 900
@@ -69,7 +82,7 @@ def test_a_checkpoint_is_listed_back_on_the_item(db, make_user, as_user):
def test_an_item_never_played_lists_zero(db, make_user, as_user):
me = make_user()
_done_job(db, me)
- [item] = as_user(me).get("/api/downloads").json()
+ [item] = as_user(me).get("/api/files").json()
assert item["position_seconds"] == 0
@@ -110,8 +123,11 @@ def test_a_recipient_keeps_their_own_place_in_a_shared_item(db, make_user, as_us
assert _checkpoint(as_user(owner), job.id, 100) == 100
assert _checkpoint(as_user(friend), job.id, 2000) == 2000
- assert as_user(owner).get("/api/downloads").json()[0]["position_seconds"] == 100
- assert as_user(friend).get("/api/downloads/shared").json()[0]["position_seconds"] == 2000
+ assert as_user(owner).get("/api/files").json()[0]["position_seconds"] == 100
+ assert (
+ as_user(friend).get("/api/files", params={"scope": "shared"}).json()[0]["position_seconds"]
+ == 2000
+ )
def test_an_item_the_caller_cannot_play_is_a_404_and_writes_nothing(db, make_user, as_user):
diff --git a/backend/tests/test_phone_safe_contract.py b/backend/tests/test_phone_safe_contract.py
index c40220fe..1a0f659c 100644
--- a/backend/tests/test_phone_safe_contract.py
+++ b/backend/tests/test_phone_safe_contract.py
@@ -141,9 +141,7 @@ def test_what_a_link_serves_passes_the_rule(
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", *args, str(src)], check=True)
owner = as_user(make_user())
- r = owner.post(
- "/api/downloads/upload", content=src.read_bytes(), headers=upload_headers(filename)
- )
+ r = owner.post("/api/files/upload", content=src.read_bytes(), headers=upload_headers(filename))
assert r.status_code == 200, r.text
job_id = r.json()["id"]
assert r.json()["status"] == "queued" # every one of these fails the rule as it came
diff --git a/backend/tests/test_public_unlock_limit.py b/backend/tests/test_public_unlock_limit.py
index f5603cbc..ab14fe61 100644
--- a/backend/tests/test_public_unlock_limit.py
+++ b/backend/tests/test_public_unlock_limit.py
@@ -25,9 +25,7 @@ def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers):
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
owner = as_user(make_user())
- job = owner.post(
- "/api/downloads/upload", content=b"fake-video", headers=upload_headers("clip.mp4")
- )
+ job = owner.post("/api/files/upload", content=b"fake-video", headers=upload_headers("clip.mp4"))
assert job.status_code == 200, job.text
link = owner.post(f"/api/downloads/{job.json()['id']}/links", json={"password": _PW})
assert link.status_code == 200, link.text
diff --git a/backend/tests/test_spa_entries.py b/backend/tests/test_spa_entries.py
index 95837020..22e46475 100644
--- a/backend/tests/test_spa_entries.py
+++ b/backend/tests/test_spa_entries.py
@@ -34,7 +34,7 @@ def link_token(as_user, make_user, tmp_path, monkeypatch, upload_headers):
monkeypatch.setattr(probemod, "probe_json", lambda _p: data)
owner = as_user(make_user())
job = owner.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"fake-video",
headers=upload_headers("clip.mp4", display_name="Holiday clip"),
)
diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py
index d563033d..889192d0 100644
--- a/backend/tests/test_upload.py
+++ b/backend/tests/test_upload.py
@@ -212,7 +212,7 @@ def test_an_anonymous_upload_is_refused_before_a_byte_of_it_is_read(client):
multipart the old route would have spooled, not the raw body the new one expects."""
body, pulled = _counting_multipart()
r = client.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=body,
headers={"content-type": f"multipart/form-data; boundary={_BOUNDARY}"},
)
@@ -227,7 +227,7 @@ def test_an_authenticated_upload_still_has_its_body_read(as_user, make_user, _ro
body, pulled = _counting_body()
c = as_user(make_user())
r = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=body,
headers={"content-type": "application/octet-stream", "x-upload-filename": "clip.bin"},
)
@@ -281,7 +281,7 @@ def test_upload_route_generic_forces_download(
_probe(monkeypatch, (None, None, None, None, None))
c = as_user(make_user())
r = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"",
headers=upload_headers("report.html"),
)
@@ -290,7 +290,7 @@ def test_upload_route_generic_forces_download(
assert job["asset"]["media_kind"] == "file"
assert job["can_download"] is True
# The served file must be a neutral force-download attachment — never inline text/html.
- fr = c.get(f"/api/downloads/{job['id']}/file")
+ fr = c.get(f"/api/files/{job['id']}/file")
assert fr.status_code == 200
assert fr.headers["content-type"].startswith("application/octet-stream")
assert "attachment" in fr.headers["content-disposition"].lower()
@@ -307,7 +307,7 @@ def test_upload_route_media_html_extension_not_served_as_html(
_probe(monkeypatch, ("h264", "aac", 320, 240, 5)) # ffprobe says it's video…
c = as_user(make_user())
r = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"",
headers=upload_headers("evil.html"),
)
@@ -324,7 +324,7 @@ def test_upload_route_media_html_extension_not_served_as_html(
db.commit()
# Authenticated endpoint: never text/html.
- fr = c.get(f"/api/downloads/{job['id']}/file")
+ fr = c.get(f"/api/files/{job['id']}/file")
assert fr.status_code == 200
assert fr.headers["content-type"].startswith("application/octet-stream")
assert "attachment" in fr.headers["content-disposition"].lower()
@@ -348,7 +348,7 @@ def test_public_watch_file_forces_download_for_generic_file(
_probe(monkeypatch, (None, None, None, None, None))
c = as_user(make_user())
job = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"",
headers=upload_headers("notes.html"),
).json()
@@ -369,7 +369,7 @@ def test_public_watch_file_serves_media_inline_with_real_type(
_probe(monkeypatch, ("h264", "aac", 640, 360, 12))
c = as_user(make_user())
job = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"fake-mp4-bytes",
headers=upload_headers("clip.mp4"),
).json()
@@ -389,7 +389,7 @@ def test_upload_route_enforces_per_file_cap(
db.commit()
c = as_user(make_user())
r = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"way over the cap",
headers=upload_headers("big.bin"),
)
@@ -412,14 +412,10 @@ def test_the_cap_admits_a_body_of_EXACTLY_the_cap_and_refuses_one_byte_more(
db.commit()
c = as_user(make_user())
- at_the_cap = c.post(
- "/api/downloads/upload", content=payload, headers=upload_headers("exact.bin")
- )
+ at_the_cap = c.post("/api/files/upload", content=payload, headers=upload_headers("exact.bin"))
assert at_the_cap.status_code == 200, at_the_cap.text
- over = c.post(
- "/api/downloads/upload", content=payload + b"x", headers=upload_headers("over.bin")
- )
+ over = c.post("/api/files/upload", content=payload + b"x", headers=upload_headers("over.bin"))
assert over.status_code == 422
assert over.json()["detail"] == {"code": "upload_too_large", "limit": len(payload)}
@@ -437,9 +433,7 @@ def test_a_non_ascii_filename_survives_the_header_round_trip(
"""
_probe(monkeypatch, (None, None, None, None, None))
c = as_user(make_user())
- r = c.post(
- "/api/downloads/upload", content=b"data", headers=upload_headers("nyaralás ősszel.bin")
- )
+ r = c.post("/api/files/upload", content=b"data", headers=upload_headers("nyaralás ősszel.bin"))
assert r.status_code == 200, r.text
asset = db.execute(select(MediaAsset).order_by(MediaAsset.id.desc())).scalars().first()
assert asset is not None
@@ -467,9 +461,7 @@ def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one(
c = as_user(make_user())
# 1. the upload route — the name arrives in a request header.
- up = c.post(
- "/api/downloads/upload", content=b"video", headers=upload_headers("a.mp4", long_name)
- )
+ up = c.post("/api/files/upload", content=b"video", headers=upload_headers("a.mp4", long_name))
assert up.status_code == 200, up.text
assert up.json()["display_name"] == bound
@@ -494,10 +486,10 @@ def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one(
assert edit.status_code == 200, edit.text
assert edit.json()["display_name"] == bound
- # 4 + 5. the metadata PATCH's two fields.
+ # 4 + 5. the library PATCH's two fields.
patched = c.patch(
- f"/api/downloads/{up.json()['id']}",
- json={"display_name": long_name, "display_uploader": long_name},
+ f"/api/files/{up.json()['id']}",
+ json={"name": long_name, "uploader": long_name},
)
assert patched.status_code == 200, patched.text
assert patched.json()["display_name"] == bound
@@ -610,12 +602,11 @@ DISPLAY_WRITE_SITES = {
("routes/downloads.py", "enqueue_download"): {"display_text(body.display_name)"},
("routes/downloads.py", "enqueue_edit"): {"display_text(body.display_name)"},
("routes/downloads.py", "enqueue_clip"): {"display_text(body.display_name)", "display_name"},
- ("routes/downloads.py", "update_download_meta"): {
- "display_text(body.display_name)",
- "display_text(body.display_uploader)",
- },
("routes/files.py", "ingest_spooled"): {"upload.display_name"},
- ("routes/files.py", "update_file"): {"display_text(body.name)"},
+ ("routes/files.py", "update_file"): {
+ "display_text(body.name)",
+ "display_text(body.uploader)",
+ },
("routes/files.py", "spool_upload"): {
"display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER))"
},
@@ -666,7 +657,7 @@ def test_the_display_name_header_reaches_the_job_and_is_bounded(
c = as_user(make_user())
ok = c.post(
- "/api/downloads/upload",
+ "/api/files/upload",
content=b"data",
headers=upload_headers("clip.bin", "Nyári vakáció"),
)
@@ -675,7 +666,7 @@ def test_the_display_name_header_reaches_the_job_and_is_bounded(
long_name = "n" * 300
bounded = c.post(
- "/api/downloads/upload", content=b"data2", headers=upload_headers("clip2.bin", long_name)
+ "/api/files/upload", content=b"data2", headers=upload_headers("clip2.bin", long_name)
)
assert bounded.status_code == 200, bounded.text
assert bounded.json()["display_name"] == "n" * DISPLAY_TEXT_MAX
@@ -718,7 +709,7 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams(
monkeypatch.setattr(files_routes, "release_read_connection", spy_release)
monkeypatch.setattr(files_routes, "capped_stream_to", spy_stream)
c = as_user(make_user())
- r = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("held.bin"))
+ r = c.post("/api/files/upload", content=b"data", headers=upload_headers("held.bin"))
assert r.status_code == 200, r.text
assert seen["before"] is True, (
"nothing had opened a transaction before the release, so this test proves nothing — the "
@@ -740,14 +731,14 @@ def test_the_usage_route_reports_the_per_file_upload_cap(as_user, make_user, db)
from app import sysconfig
c = as_user(make_user())
- r = c.get("/api/downloads/usage")
+ r = c.get("/api/files/usage")
assert r.status_code == 200, r.text
assert r.json()["upload_max_bytes"] == sysconfig.get_int(db, "download_upload_max_bytes")
# And it is the KNOB, not a constant: a lowered override must show through.
sysconfig.set_value(db, "download_upload_max_bytes", 4096)
db.commit()
- assert c.get("/api/downloads/usage").json()["upload_max_bytes"] == 4096
+ assert c.get("/api/files/usage").json()["upload_max_bytes"] == 4096
def test_a_refused_upload_leaves_nothing_in_the_staging_directory(
@@ -765,7 +756,7 @@ def test_a_refused_upload_leaves_nothing_in_the_staging_directory(
staging = Path(settings.files_root) / ".staging" / "uploads"
c = as_user(make_user())
- r = c.post("/api/downloads/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin"))
+ r = c.post("/api/files/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin"))
assert r.status_code == 422
assert staging.exists(), "the dependency did not get as far as creating the staging dir"
assert list(staging.iterdir()) == [], "a refused upload left its partial file behind"
@@ -787,7 +778,7 @@ def test_upload_cap_cannot_be_disabled_with_zero(db):
def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch, upload_headers):
c = as_user(make_user())
- r = c.post("/api/downloads/upload", content=b"", headers=upload_headers("empty.bin"))
+ r = c.post("/api/files/upload", content=b"", headers=upload_headers("empty.bin"))
assert r.status_code == 400
assert r.json()["detail"]["code"] == "upload_empty"
@@ -795,7 +786,7 @@ def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch,
def test_redownload_rejects_an_upload(as_user, make_user, _root, monkeypatch, upload_headers):
_probe(monkeypatch, (None, None, None, None, None))
c = as_user(make_user())
- job = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("a.bin")).json()
+ job = c.post("/api/files/upload", content=b"data", headers=upload_headers("a.bin")).json()
r = c.post(f"/api/downloads/{job['id']}/redownload")
assert r.status_code == 409
@@ -805,7 +796,7 @@ def test_edit_rejects_a_generic_upload_source(
):
_probe(monkeypatch, (None, None, None, None, None))
c = as_user(make_user())
- job = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("a.bin")).json()
+ job = c.post("/api/files/upload", content=b"data", headers=upload_headers("a.bin")).json()
r = c.post(
"/api/downloads/edit",
json={"source_job_id": job["id"], "edit_spec": {"trim": {"start_s": 0, "end_s": 1}}},
diff --git a/backend/tests/test_upload_conversion.py b/backend/tests/test_upload_conversion.py
index b4d84521..969c33c0 100644
--- a/backend/tests/test_upload_conversion.py
+++ b/backend/tests/test_upload_conversion.py
@@ -549,7 +549,7 @@ def test_a_link_says_preparing_only_while_its_video_is_being_made(
_probe(monkeypatch, vcodec="h264")
c = as_user(make_user())
job = c.post(
- "/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
+ "/api/files/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
).json()
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
assert c.get(f"/api/public/watch/{link['token']}").status_code == 200
@@ -570,7 +570,7 @@ def test_a_link_to_a_suspended_owners_waiting_rebuild_says_it_is_gone(
owner = make_user()
c = as_user(owner)
job = c.post(
- "/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
+ "/api/files/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4")
).json()
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
db.get(MediaAsset, job["asset"]["id"]).status = "pending"
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 5c59153e..c3ac441e 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -87,6 +87,7 @@ export default function App() {
feed: seedLayout("feed", prefs),
plex: seedLayout("plex", prefs),
playlists: seedLayout("playlists", prefs),
+ files: seedLayout("files", prefs),
};
});
const [aboutOpen, setAboutOpen] = useState(false);
@@ -119,6 +120,17 @@ export default function App() {
localKey: LS.playlistsCollapsed,
parse: Boolean,
});
+ // The Files rail is the library's folder list — how you move around it, not a filter you set once —
+ // so it starts pinned open (the owner's mockup), and has its own flag.
+ const [filesCollapsed, setFilesCollapsed] = usePref("filesCollapsed", false, {
+ localKey: LS.filesCollapsed,
+ parse: Boolean,
+ });
+ const railCollapse = {
+ filter: [filterCollapsed, setFilterCollapsed],
+ playlists: [playlistsCollapsed, setPlaylistsCollapsed],
+ files: [filesCollapsed, setFilesCollapsed],
+ } as const;
// (The feed filters + page/channel now LIVE in the address bar — FeedFiltersProvider and
// NavigationProvider read them on load and keep them mirrored — so there's no query to strip here.)
@@ -378,18 +390,17 @@ export default function App() {
/>
{/* The active page's side rail (feed filters / plex filters / playlists list), if it has one.
A floating collapsible column beside the nav rail; hidden while a channel page overlays the
- content. feed + plex share the filter-rail collapse flag, playlists has its own. */}
+ content. feed + plex share the filter-rail collapse flag, playlists and files have their own. */}
{railDef && (
setPanelLayout(railDef.panelId, l)}
- collapsed={railDef.collapse === "playlists" ? playlistsCollapsed : filterCollapsed}
- onToggleCollapse={() =>
- railDef.collapse === "playlists"
- ? setPlaylistsCollapsed(!playlistsCollapsed)
- : setFilterCollapsed(!filterCollapsed)
- }
+ collapsed={railCollapse[railDef.collapse][0]}
+ onToggleCollapse={() => {
+ const [collapsed, setCollapsed] = railCollapse[railDef.collapse];
+ setCollapsed(!collapsed);
+ }}
/>
)}
diff --git a/frontend/src/components/DownloadCenter.confirm.dom.test.tsx b/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
index f3ca2fc5..d216f11f 100644
--- a/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
+++ b/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
@@ -1,8 +1,9 @@
// @vitest-environment jsdom
//
// The Download Center's confirm dialogs name their ACTION on the confirm button: the shared
-// `confirmThen` once labelled it with the dialog's question — "Re-download this file?" on the button,
-// and the cancel dialog's own "Cancel" beside "Cancel". i18n is key-passthrough, so labels read as keys.
+// `confirmThen` once labelled it with the dialog's question — the cancel dialog's own "Cancel" beside
+// "Cancel". A finished file's confirms are the Files page's (Files.menu.dom.test.tsx). i18n is
+// key-passthrough, so labels read as keys.
import { afterEach, describe, expect, it, vi } from "vitest";
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
@@ -73,7 +74,6 @@ const job = (id: number, status: DownloadJob["status"]): DownloadJob => ({
display_uploader_url: null,
extra_links: [],
can_download: status === "done",
- expired: false,
folder_id: null,
kept: false,
expires_at: null,
@@ -105,7 +105,7 @@ async function pick(row: string, label: string) {
describe("DownloadCenter confirm dialogs", () => {
it("label the confirm button with the action, not the dialog's question", async () => {
- mocks.jobs = [job(1, "running"), job(2, "done")];
+ mocks.jobs = [job(1, "running")];
render(
@@ -115,18 +115,9 @@ describe("DownloadCenter confirm dialogs", () => {
// The destructive and rare row buttons live in the row's "…".
await pick("Clip 1", "downloads.actions.cancel");
await waitFor(() => expect(mocks.confirm).toHaveBeenCalledTimes(1));
- // The tab's label shares its button with a count badge.
- fireEvent.click(screen.getByRole("tab", { name: /downloads\.tabs\.done/ }));
- await screen.findByText("Clip 2");
- await pick("Clip 2", "downloads.actions.redownload");
- await waitFor(() => expect(mocks.confirm).toHaveBeenCalledTimes(2));
- await pick("Clip 2", "downloads.actions.delete");
- await waitFor(() => expect(mocks.confirm).toHaveBeenCalledTimes(3));
expect(mocks.confirm.mock.calls.map(([o]) => [o.title, o.confirmLabel])).toEqual([
["downloads.confirm.cancelTitle", "downloads.confirm.cancelAction"],
- ["downloads.confirm.redownloadTitle", "downloads.confirm.redownloadAction"],
- ["downloads.confirm.deleteTitle", "downloads.actions.delete"],
]);
});
@@ -163,24 +154,4 @@ describe("DownloadCenter confirm dialogs", () => {
"downloads.confirm.cancelEditAction",
]);
});
-
- it.each([
- ["upload", "downloads.confirm.deleteUploadTitle", "downloads.confirm.deleteUploadBody"],
- ["edit", "downloads.confirm.deleteEditTitle", "downloads.confirm.deleteEditBody"],
- ["download", "downloads.confirm.deleteTitle", "downloads.confirm.deleteBody"],
- ] as const)("deleting a finished %s says what happens to its file", async (kind, title, body) => {
- // Only a download's file may stay cached for other users; an upload's or a clip's is theirs alone.
- mocks.jobs = [{ ...job(5, "done"), job_kind: kind }];
- render(
-
-
- ,
- );
- fireEvent.click(screen.getByRole("tab", { name: /downloads\.tabs\.done/ }));
- await screen.findByText("Clip 5");
- await pick("Clip 5", "downloads.actions.delete");
- await waitFor(() => expect(mocks.confirm).toHaveBeenCalledTimes(1));
- const [opts] = mocks.confirm.mock.calls[0]!;
- expect([opts.title, opts.message]).toEqual([title, body]);
- });
});
diff --git a/frontend/src/components/DownloadCenter.menu.dom.test.tsx b/frontend/src/components/DownloadCenter.menu.dom.test.tsx
index 59b762d8..a02fe0d5 100644
--- a/frontend/src/components/DownloadCenter.menu.dom.test.tsx
+++ b/frontend/src/components/DownloadCenter.menu.dom.test.tsx
@@ -1,10 +1,9 @@
// @vitest-environment jsdom
//
-// The Download Center: a download row keeps its primary action (play; pause / resume /
-// retry in the queue) and the save link in sight, and everything else — the editors, sharing,
-// re-download, the copies, cancel / delete / remove — is its ONE menu, opened by the row's "…" (it
-// shows on hover or focus) or a right-click on the row. A link inside the row keeps the browser's own
-// menu. i18n is key-passthrough, so labels read as keys.
+// The Download Center's queue: a row keeps its primary action (pause / resume / retry) in sight, and
+// the copies and cancel are its ONE menu, opened by the row's "…" (it shows on hover or focus) or a
+// right-click on the row. A finished download's row is the Files page's (Files.menu.dom.test.tsx).
+// i18n is key-passthrough, so labels read as keys.
import { afterEach, describe, expect, it, vi } from "vitest";
import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
@@ -68,7 +67,6 @@ const job = (
display_uploader_url: null,
extra_links: [],
can_download: status === "done",
- expired: false,
job_kind: "download",
asset: { media_kind: "video", duration_s: 60 },
...extra,
@@ -209,160 +207,6 @@ describe("DownloadCenter · the queue row", () => {
});
});
-describe("DownloadCenter · the library row", () => {
- it("shows play, share, save and its '…'; the editors, managing shares and delete are in the menu", async () => {
- mocks.jobs = [job(2, "done", { source_url: "https://youtu.be/x" })];
- mount("done");
- await screen.findByText("Clip 2");
- expect(rowButtons("Clip 2")).toEqual([
- "downloads.actions.play",
- "downloads.actions.share",
- "downloads.actions.download",
- "downloads.rowMenu",
- ]);
- await openMenu(() => fireEvent.contextMenu(rowOf("Clip 2")));
- expect(menuRows()).toEqual([
- "downloads.actions.play",
- "downloads.actions.share",
- "downloads.actions.download",
- "downloads.actions.edit",
- "downloads.actions.editDetails",
- "downloads.actions.redownload",
- "downloads.actions.shareManage",
- "common.copy.title",
- "common.copy.link",
- "downloads.actions.delete",
- ]);
- });
-
- it("a started item continues — its bar shows how far — and its menu can start it over", async () => {
- mocks.jobs = [job(2, "done", { position_seconds: 30 })];
- mount("done");
- await screen.findByText("Clip 2");
- expect(rowButtons("Clip 2")[0]).toBe("card.continueTitle");
- // The media cards' bar along the picture's bottom edge: 30 s of the 60 s item.
- expect(rowOf("Clip 2").querySelector(".bg-accent")!.style.width).toBe("50%");
- await openMenu(() => fireEvent.contextMenu(rowOf("Clip 2")));
- expect(menuRows().slice(0, 2)).toEqual(["card.continue", "card.restartTitle"]);
-
- // Starting over opens the player at 0:00, not at the kept point.
- fireEvent.click(
- screen
- .getAllByRole("menuitem", { hidden: true })
- .find((b) => b.textContent === "card.restartTitle")!,
- );
- const video = await waitFor(() => {
- const v = document.querySelector("video");
- expect(v).not.toBeNull();
- return v!;
- });
- let t = 0;
- Object.defineProperties(video, {
- currentTime: { get: () => t, set: (v: number) => void (t = v), configurable: true },
- duration: { get: () => 60, configurable: true },
- readyState: { get: () => 1, configurable: true },
- });
- video.dispatchEvent(new Event("loadedmetadata"));
- expect(video.currentTime).toBe(0);
- });
-
- it("an item never played offers Play only, with no bar", async () => {
- mocks.jobs = [job(2, "done", { position_seconds: 0 })];
- mount("done");
- await screen.findByText("Clip 2");
- expect(rowOf("Clip 2").querySelector(".bg-accent")).toBeNull();
- await openMenu(() => fireEvent.contextMenu(rowOf("Clip 2")));
- expect(menuRows()).not.toContain("card.restartTitle");
- });
-
- it("copies the share link from the row's own Share button", async () => {
- mocks.jobs = [job(2, "done")];
- vi.spyOn(api, "downloadLinks").mockResolvedValue([] as never);
- const create = vi
- .spyOn(api, "createDownloadLink")
- .mockResolvedValue({ id: 1, url: "/s/tok" } as never);
- mount("done");
- await screen.findByText("Clip 2");
- fireEvent.click(
- rowOf("Clip 2").querySelector('button[aria-label="downloads.actions.share"]')!,
- );
- await waitFor(() => expect(create).toHaveBeenCalledWith(2, {}));
- expect(screen.queryAllByRole("menuitem", { hidden: true })).toHaveLength(0);
- });
-
- it("saves the file through the same link as the row's save button", async () => {
- mocks.jobs = [job(2, "done")];
- const followed: string[] = [];
- vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(function (
- this: HTMLAnchorElement,
- ) {
- followed.push(this.href);
- });
- mount("done");
- await screen.findByText("Clip 2");
- await openMenu(() => fireEvent.click(moreOf("Clip 2")));
- fireEvent.click(
- screen
- .getAllByRole("menuitem", { hidden: true })
- .find((b) => b.textContent === "downloads.actions.download")!,
- );
- const button = rowOf("Clip 2").querySelector(
- 'a[aria-label="downloads.actions.download"]',
- )!;
- expect(followed).toEqual([button.href]);
- });
-
- it("leaves out what a file or a clip cannot do, and no rule is left behind", async () => {
- mocks.jobs = [job(3, "done", { job_kind: "upload", asset: { media_kind: "file" } } as never)];
- mount("done");
- await screen.findByText("Clip 3");
- await openMenu(() => fireEvent.click(moreOf("Clip 3")));
- expect(menuRows()).toEqual([
- "downloads.actions.share",
- "downloads.actions.download",
- "downloads.actions.shareManage",
- "common.copy.title",
- "downloads.actions.delete",
- ]);
- });
-
- it("gives a link inside the row the browser's own menu", async () => {
- mocks.jobs = [job(2, "done", { source_url: "https://youtu.be/x" })];
- mount("done");
- const link = await screen.findByText("youtu.be/x");
- const ev = new MouseEvent("contextmenu", { bubbles: true, cancelable: true });
- act(() => void link.dispatchEvent(ev));
- expect(ev.defaultPrevented).toBe(false);
- expect(screen.queryAllByRole("menuitem", { hidden: true })).toHaveLength(0);
- });
-});
-
-describe("DownloadCenter · the shared row", () => {
- it("holds play, save, edit, the copies and Remove from my list", async () => {
- mocks.jobs = [];
- vi.spyOn(api, "downloadsShared").mockResolvedValue([
- job(4, "done", { can_download: true }),
- ] as never);
- const remove = vi.spyOn(api, "removeSharedDownload").mockResolvedValue({} as never);
- mount("shared");
- await screen.findByText("Clip 4");
- await openMenu(() => fireEvent.click(moreOf("Clip 4")));
- expect(menuRows()).toEqual([
- "downloads.actions.play",
- "downloads.actions.download",
- "downloads.actions.edit",
- "common.copy.title",
- "downloads.actions.removeShared",
- ]);
- fireEvent.click(
- screen
- .getAllByRole("menuitem", { hidden: true })
- .find((b) => b.textContent === "downloads.actions.removeShared")!,
- );
- await waitFor(() => expect(remove).toHaveBeenCalledWith(4));
- });
-});
-
describe("menuGroups", () => {
const row = (id: string) => ({ id, label: id, onSelect: () => {} });
it("rules between non-empty groups only", () => {
diff --git a/frontend/src/components/DownloadCenter.tsx b/frontend/src/components/DownloadCenter.tsx
index a6f0bea4..79777477 100644
--- a/frontend/src/components/DownloadCenter.tsx
+++ b/frontend/src/components/DownloadCenter.tsx
@@ -1,47 +1,29 @@
import { lazy, Suspense, useState } from "react";
import { qk } from "../lib/queryKeys";
import { useTranslation } from "react-i18next";
-import { useQuery } from "@tanstack/react-query";
-import {
- ClipboardPaste,
- Download,
- Pause,
- Play,
- PlayCircle,
- Plus,
- RefreshCw,
- Settings2,
- Share2,
- UploadCloud,
-} from "lucide-react";
+import { ClipboardPaste, Pause, Play, Plus, RefreshCw, Settings2, UploadCloud } from "lucide-react";
import Tabs from "./ui/Tabs";
import { useTabPanel } from "./ui/tabPanel";
import { PageToolbar } from "./ui/PageShell";
import { LoadingState, StateMessage } from "./ui/QueryState";
-// Lazy: these dialogs/editors open on demand, so they stay out of the Downloads page chunk until
-// used (the video editor in particular is heavy — filmstrip + scrubber).
+// Lazy: these dialogs open on demand, so they stay out of the Downloads page chunk until used.
const DownloadDialog = lazy(() => import("./DownloadDialog"));
const UploadDialog = lazy(() => import("./UploadDialog"));
const ProfileEditor = lazy(() => import("./ProfileEditor"));
-const VideoEditor = lazy(() => import("./VideoEditor"));
-const ShareDialog = lazy(() => import("./ShareDialog"));
-const LibraryPlayer = lazy(() => import("./LibraryPlayer"));
import { useLiveQuery } from "../lib/useLiveQuery";
import { useMe } from "../lib/useMe";
import { api, type DownloadJob } from "../lib/api";
-import { libraryRowActions, sharedRowActions } from "../lib/downloadActions";
import { Button } from "./ui/Button";
import { IconButton } from "./ui/IconButton";
-import { DownloadRow, IconBtn, UsageBar } from "./downloadRow";
-import { MetaEditModal, ErrorModal } from "./downloadModals";
+import { DownloadRow, IconBtn } from "./downloadRow";
+import { ErrorModal } from "./downloadModals";
import { AdminSystem } from "./downloadAdmin";
import NavGrid from "./ui/NavGrid";
import { Input } from "./ui/Input";
-import { IconLink } from "./ui/LinkButton";
import { useContextMenu } from "./ui/useContextMenu";
import { ContextMenu } from "./ui/ContextMenu";
import { MoreButton } from "./ui/MoreButton";
-import { useDownloadActions, type DownloadTab } from "./useDownloadActions";
+import { useDownloadActions } from "./useDownloadActions";
const jobKey = (job: DownloadJob) => job.id;
@@ -55,6 +37,9 @@ async function clipboardText(): Promise {
}
}
+// The download queue: what is downloading, waiting or failed, the link box and the upload, the
+// download profiles, and (admin) the instance's downloads. A finished download is a library file and
+// lives in Files.
export default function DownloadCenter() {
const me = useMe();
const { t } = useTranslation();
@@ -66,12 +51,6 @@ export default function DownloadCenter() {
const [addSource, setAddSource] = useState(null);
const [upload, setUpload] = useState(false);
const [manage, setManage] = useState(false);
- const [metaJob, setMetaJob] = useState(null);
- const [shareJob, setShareJob] = useState(null);
- // What the player opens, and whether over the kept point ("Play from the beginning").
- const [playing, setPlaying] = useState<{ job: DownloadJob; fromStart: boolean } | null>(null);
- const setPlayJob = (job: DownloadJob, fromStart = false) => setPlaying({ job, fromStart });
- const [editJob, setEditJob] = useState(null);
const [errorJob, setErrorJob] = useState(null);
// Poll fast (2s) while work is actually in flight; back off to 15s once the queue is settled
@@ -81,29 +60,15 @@ export default function DownloadCenter() {
intervalMs: (data) =>
data?.some((j) => j.status === "queued" || j.status === "running") ? 2000 : 15000,
});
- const sharedQ = useQuery({
- queryKey: qk.downloadsShared(),
- queryFn: api.downloadsShared,
- enabled: tab === "shared",
- });
const jobs = jobsQ.data ?? [];
const queue = jobs.filter((j) => ["queued", "running", "paused", "error"].includes(j.status));
- const library = jobs.filter((j) => j.status === "done");
// --- The rows' ONE menu -------------------------------------------------------------
// Keyed by the job's id: the queue polls every 2 s while a job runs, replacing the object under an
- // open menu, and its "…" must still close it. The tab is the one on screen.
+ // open menu, and its "…" must still close it.
const rowMenu = useContextMenu({ keyOf: jobKey });
- const actions = useDownloadActions({
- onPlay: setPlayJob,
- onEdit: setEditJob,
- onEditDetails: setMetaJob,
- onManageShares: setShareJob,
- });
- const menuTarget = rowMenu.target;
- const listTab: DownloadTab | null =
- tab === "queue" || tab === "done" || tab === "shared" ? tab : null;
- const menuItems = menuTarget && listTab ? actions.items(menuTarget, listTab) : [];
+ const actions = useDownloadActions();
+ const menuItems = rowMenu.target && tab === "queue" ? actions.items(rowMenu.target) : [];
// The row's menu triggers and its "…".
const rowMenuProps = (job: DownloadJob) => {
const mine = rowMenu.open && rowMenu.target?.id === job.id;
@@ -133,8 +98,6 @@ export default function DownloadCenter() {
const tabs = [
{ id: "queue", label: t("downloads.tabs.queue"), badge: queue.length },
- { id: "done", label: t("downloads.tabs.done"), badge: library.length },
- { id: "shared", label: t("downloads.tabs.shared") },
...(me.role === "admin" ? [{ id: "system", label: t("downloads.tabs.system") }] : []),
];
// Guard against the active tab going away under us (e.g. admin on "system" gets demoted mid-session)
@@ -144,12 +107,6 @@ export default function DownloadCenter() {
// account), so a render-phase write is allowed, and it converges — "queue" is always in the list.
if (!tabs.some((tb) => tb.id === tab)) setTab("queue");
- const saveBtn = (job: DownloadJob) => (
-
-
-
- );
-
return (
<>
{/* The tabs are the page's band (B4); the tab content scrolls under them. No visible title: the
@@ -268,83 +225,6 @@ export default function DownloadCenter() {
>
)}
- {tab === "done" && (
- <>
-
-
- {library.map((job, i) => (
-
- {libraryRowActions(job).play && (
- setPlayJob(job)}
- title={
- job.position_seconds ? t("card.continueTitle") : t("downloads.actions.play")
- }
- >
-
-
- )}
- {/* Sharing the link is the library's main errand: in sight, ahead of save. */}
- actions.share(job)} title={t("downloads.actions.share")}>
-
-
- {saveBtn(job)}
-
- ))}
- {library.length === 0 &&
- (jobsQ.isError && !jobsQ.data ? (
- jobsQ.refetch()}>
- {t("common.loadError")}
-
- ) : jobsQ.isLoading ? (
-
- ) : (
-