diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c53783..40c02ee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.107.0 — 2026-10-15 + +### New + +- API tokens: Settings › API tokens gives one machine of yours a key to the Files folders you pick — or, in one click, to a Hub folder for swapping files with it. Signing out your other sessions or changing your password revokes every token. +- Share by link: a link can now open a whole folder of yours — its page lists what is in it and plays a video above the list. + +### Fixed + +- Uploads: a finished upload is announced once, and shows in Files at once. + ## 0.106.2 — 2026-10-14 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 7d7a807b..b9353518 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ how it gets built. Concretely: ## Architecture shape (don't fight it) `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, -`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`, +`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`, `security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own diff --git a/VERSION b/VERSION index f18da326..a9a7f3fe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.106.2 +0.107.0 diff --git a/backend/alembic/versions/0106_api_tokens.py b/backend/alembic/versions/0106_api_tokens.py new file mode 100644 index 00000000..6ea0e68b --- /dev/null +++ b/backend/alembic/versions/0106_api_tokens.py @@ -0,0 +1,110 @@ +"""Machine access to the Files library: API tokens bound to folders, and the Hub folder + +Revision ID: 0106_api_tokens +Revises: 0105_job_source_key +Create Date: 2026-10-14 + +`api_tokens` is a user's token for a machine (a script, the hub): only its SHA-256 is kept, plus the +first characters to recognise it by, and whether its user was told it is about to run out. `api_token_grants` says what it may do in which folder — list, +read, write, delete — so a token reaches nothing it was not given, folder by folder. + +A third folder kind, `hub`: the one folder a user exchanges files with their hub machine through. +Like a source folder its name is the client's to render; like an own folder a file may be moved in. +One per user. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0106_api_tokens" +down_revision: str | None = "0105_job_source_key" +branch_labels: str | None = None +depends_on: str | None = None + +_KIND_OLD = ( + "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" + " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)" +) +_KIND_NEW = _KIND_OLD + " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)" + + +def upgrade() -> None: + op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check") + op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_NEW) + op.create_index( + "uq_file_folder_hub", + "file_folders", + ["user_id"], + unique=True, + postgresql_where=sa.text("kind = 'hub'"), + ) + op.create_table( + "api_tokens", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("name", sa.String(80), nullable=False), + sa.Column("prefix", sa.String(16), nullable=False), + sa.Column("token_hash", sa.String(64), nullable=False, unique=True), + sa.Column("expires_at", sa.DateTime(timezone=True)), + sa.Column("last_used_at", sa.DateTime(timezone=True)), + sa.Column("expiry_warned", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + ) + op.create_table( + "api_token_grants", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "token_id", + sa.Integer(), + sa.ForeignKey("api_tokens.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "folder_id", + sa.Integer(), + sa.ForeignKey("file_folders.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("can_list", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("can_read", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("can_write", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("can_delete", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.UniqueConstraint("token_id", "folder_id", name="uq_api_token_grant"), + sa.CheckConstraint( + "can_list OR can_read OR can_write OR can_delete", name="ck_api_token_grant_some" + ), + ) + + +def downgrade() -> None: + op.drop_table("api_token_grants") + op.drop_table("api_tokens") + op.drop_index("uq_file_folder_hub", table_name="file_folders") + # A Hub folder may hold files (their foreign key RESTRICTs a delete): it becomes an own folder, + # named "Hub " — or "Hub (2)", … when its user already has a folder of that name, + # which `uq_file_folder_own_name` (per user, case-blind) would refuse. + conn = op.get_bind() + hubs = conn.execute(sa.text("SELECT id, user_id FROM file_folders WHERE kind = 'hub'")).all() + for folder_id, user_id in hubs: + name, n = f"Hub {folder_id}", 1 + while conn.execute( + sa.text("SELECT 1 FROM file_folders WHERE user_id = :u AND lower(name) = lower(:n)"), + {"u": user_id, "n": name}, + ).first(): + n += 1 + name = f"Hub {folder_id} ({n})" + conn.execute( + sa.text("UPDATE file_folders SET kind = 'own', name = :n WHERE id = :i"), + {"n": name, "i": folder_id}, + ) + op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check") + op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_OLD) diff --git a/backend/alembic/versions/0107_link_token_at_rest.py b/backend/alembic/versions/0107_link_token_at_rest.py new file mode 100644 index 00000000..30554da0 --- /dev/null +++ b/backend/alembic/versions/0107_link_token_at_rest.py @@ -0,0 +1,66 @@ +"""Share-link tokens no longer stored in the clear + +Revision ID: 0107_link_token_at_rest +Revises: 0106_api_tokens +Create Date: 2026-10-15 + +A `/watch` link's token is the credential itself, and it sat in `download_links.token` as written, +so a copy of the database was every shared file. Now a link is found by the token's SHA-256 +(`token_hash`), and the token is kept only Fernet-encrypted (`token_enc`, TOKEN_ENCRYPTION_KEY) so its +owner can copy the same link again. Without a key the copy is not kept: the link still opens, its +address just cannot be shown again (`app/downloads/links.py`). + +Every existing link is converted here, so the addresses already sent keep working. The downgrade +writes the tokens back from their encrypted copies; a link with none left (no key, or a key since +changed) gets a fresh random token, which is the same as revoking it. +""" + +import hashlib +import secrets + +import sqlalchemy as sa +from alembic import op + +from app import security + +revision: str = "0107_link_token_at_rest" +down_revision: str | None = "0106_api_tokens" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column("download_links", sa.Column("token_hash", sa.String(64))) + op.add_column("download_links", sa.Column("token_enc", sa.Text())) + conn = op.get_bind() + sealing = security.can_encrypt() + for link_id, token in conn.execute(sa.text("SELECT id, token FROM download_links")).fetchall(): + conn.execute( + sa.text("UPDATE download_links SET token_hash = :h, token_enc = :e WHERE id = :id"), + { + "id": link_id, + "h": hashlib.sha256(token.encode()).hexdigest(), + "e": security.encrypt(token) if sealing else None, + }, + ) + op.alter_column("download_links", "token_hash", nullable=False) + op.create_unique_constraint("uq_download_link_token_hash", "download_links", ["token_hash"]) + op.drop_constraint("uq_download_link_token", "download_links", type_="unique") + op.drop_column("download_links", "token") + + +def downgrade() -> None: + op.add_column("download_links", sa.Column("token", sa.String(64))) + conn = op.get_bind() + rows = conn.execute(sa.text("SELECT id, token_enc FROM download_links")).fetchall() + for link_id, enc in rows: + token = security.decrypt_optional(enc) or secrets.token_urlsafe(24) + conn.execute( + sa.text("UPDATE download_links SET token = :t WHERE id = :id"), + {"id": link_id, "t": token}, + ) + op.alter_column("download_links", "token", nullable=False) + op.create_unique_constraint("uq_download_link_token", "download_links", ["token"]) + op.drop_constraint("uq_download_link_token_hash", "download_links", type_="unique") + op.drop_column("download_links", "token_enc") + op.drop_column("download_links", "token_hash") diff --git a/backend/alembic/versions/0108_folder_links.py b/backend/alembic/versions/0108_folder_links.py new file mode 100644 index 00000000..b018470a --- /dev/null +++ b/backend/alembic/versions/0108_folder_links.py @@ -0,0 +1,40 @@ +"""A share link may open a folder + +Revision ID: 0108_folder_links +Revises: 0107_link_token_at_rest +Create Date: 2026-10-15 + +A `/watch` link opened one file. Now it may open one of its owner's own folders instead: the link's +page lists what is in the folder when it is opened. `folder_id` sits beside `job_id`, exactly one of +the two set; a folder's links go with the folder. + +The downgrade has nowhere to keep a folder link, so it deletes them: they stop opening. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0108_folder_links" +down_revision: str | None = "0107_link_token_at_rest" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column( + "download_links", + sa.Column("folder_id", sa.Integer(), sa.ForeignKey("file_folders.id", ondelete="CASCADE")), + ) + op.create_index("ix_download_links_folder_id", "download_links", ["folder_id"]) + op.alter_column("download_links", "job_id", nullable=True) + op.create_check_constraint( + "ck_download_link_target", "download_links", "(job_id IS NULL) <> (folder_id IS NULL)" + ) + + +def downgrade() -> None: + op.execute("DELETE FROM download_links WHERE folder_id IS NOT NULL") + op.drop_constraint("ck_download_link_target", "download_links", type_="check") + op.alter_column("download_links", "job_id", nullable=False) + op.drop_index("ix_download_links_folder_id", table_name="download_links") + op.drop_column("download_links", "folder_id") diff --git a/backend/app/README.md b/backend/app/README.md index b8cd114c..6ff33ab1 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -20,7 +20,8 @@ Three layers, by *who* runs the code — not by feature: run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline; `issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker. -3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`, +3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth` (with `apitokens`, a machine's + folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`, `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the diff --git a/backend/app/apitokens.py b/backend/app/apitokens.py new file mode 100644 index 00000000..8ee6f4e2 --- /dev/null +++ b/backend/app/apitokens.py @@ -0,0 +1,273 @@ +"""Machine access to the Files library: API tokens, and what each one may do where. + +A user makes a token for a machine of theirs — their hub, a script — and it reaches their library as +`Authorization: Bearer sfl_…`. Only on `/api/files`: the routes there take `files_caller` instead of +`current_user`, and nothing else does (a test holds the route table to that), so a token can never +manage the account, the tokens, or anything outside the library. And only in the folders its grants +name, with the rights they give: list, read, write, delete. Every route asks the one question, +`FilesCaller.may(folder_id, right)`; a signed-in user may everything in their own folders. + +The token is `sfl_` + 32 random bytes in base62 + a CRC32 of them: a mistyped or cut-off token is +refused without a database read, and a secret scanner can tell one by its shape. Only its SHA-256 is +kept (`security.hash_token`) — the user sees it once, when it is made — with its first characters to +tell tokens apart by. It is looked up by that hash on every request: no signed claims, so a deleted +token stops working at once. A token about to run out warns its user once, a week before +(`warn_expiring`), so a machine does not lose its key unannounced. +""" + +import secrets +import zlib +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from enum import StrEnum + +from fastapi import Depends, Request +from sqlalchemy import delete, or_, select, update +from sqlalchemy.orm import Session + +from app import audit, debugsession, logs +from app.audit import AuditAction +from app.auth import current_user +from app.db import get_db +from app.models import ApiToken, ApiTokenGrant, User +from app.notifications import create_notification +from app.ratelimit import RateLimiter, client_key +from app.refusal import refused +from app.security import hash_token +from app.utils import iso + +log = logs.get_logger(__name__) + +PREFIX = "sfl_" +_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +_BODY_BYTES = 32 +_BODY_LEN = 43 # 62**43 > 2**256: every 32-byte value fits +_CHECK_LEN = 6 # 62**6 > 2**32: every CRC32 fits +TOKEN_LEN = len(PREFIX) + _BODY_LEN + _CHECK_LEN +# What a token is shown by once it is made: enough to tell a user's tokens apart, far too little to +# narrow a guess (8 of its 43 random characters). +SHOWN_LEN = len(PREFIX) + 8 + +# A machine that runs away (a loop in a script) is held back per token; a caller trying tokens is +# held back per client, counting only the refusals. +_calls = RateLimiter(600, 60) +_bad = RateLimiter(20, 300) +# How stale `last_used_at` may get before a request writes it again: a busy script costs one write a +# minute, not one per request. +_TOUCH_EVERY = timedelta(minutes=1) +# How long before a token runs out its user is told (`warn_expiring`); the token list marks it +# "expires soon" for the same span. +EXPIRY_WARNING = timedelta(days=7) + + +class Right(StrEnum): + LIST = "list" + READ = "read" + WRITE = "write" + DELETE = "delete" + + +# Each right's column on a grant. A dict over the enum, so a new right without a column fails here. +GRANT_COLUMNS = { + Right.LIST: "can_list", + Right.READ: "can_read", + Right.WRITE: "can_write", + Right.DELETE: "can_delete", +} + + +def _b62(value: int, width: int) -> str: + out = [] + for _ in range(width): + value, digit = divmod(value, 62) + out.append(_ALPHABET[digit]) + return "".join(reversed(out)) + + +def _check(body: str) -> str: + return _b62(zlib.crc32(body.encode()), _CHECK_LEN) + + +def mint() -> str: + """A new token, in full. The caller stores its hash and shows it to the user, once.""" + body = _b62(int.from_bytes(secrets.token_bytes(_BODY_BYTES)), _BODY_LEN) + return PREFIX + body + _check(body) + + +def well_formed(raw: str) -> bool: + """Whether `raw` has a token's shape and its checksum adds up.""" + if len(raw) != TOKEN_LEN or not raw.startswith(PREFIX): + return False + body, check = raw[len(PREFIX) : -_CHECK_LEN], raw[-_CHECK_LEN:] + # The check too: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it. + if any(c not in _ALPHABET for c in body + check): + return False + return secrets.compare_digest(_check(body), check) + + +def grant_rights(grant: ApiTokenGrant) -> frozenset[Right]: + return frozenset(r for r, col in GRANT_COLUMNS.items() if getattr(grant, col)) + + +@dataclass(frozen=True) +class FilesCaller: + """Who is asking the Files library, and where they may do what. `grants` is None for a signed-in + user (everything in their own folders); for a token, folder id → its rights there.""" + + user: User + token_id: int | None = None + token_prefix: str | None = None + grants: Mapping[int, frozenset[Right]] | None = None + + def may(self, folder_id: int | None, right: Right) -> bool: + if self.grants is None: + return True + return folder_id is not None and right in self.grants.get(folder_id, frozenset()) + + def folders_with(self, right: Right) -> set[int] | None: + """The folders `right` is granted in; None = any of the user's own.""" + if self.grants is None: + return None + return {f for f, rights in self.grants.items() if right in rights} + + @property + def via(self) -> str | None: + """How an audit row says a token did it, None when the user did.""" + return None if self.token_prefix is None else f"api token {self.token_prefix}…" + + +def bearer_token(request: Request) -> str | None: + """The token of an `Authorization: Bearer` header; None when the request carries none (another + scheme counts as none — it is not ours to judge).""" + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "bearer": + return None + return value.strip() + + +def session_caller(request: Request, db: Session = Depends(get_db)) -> User | None: + """The signed-in user, or None when the request brings a token instead: a token wins over a + cookie, so a machine's request is judged by its grants alone, whatever else it carries.""" + if bearer_token(request) is not None: + return None + return current_user(request, db) + + +def files_caller( + request: Request, + db: Session = Depends(get_db), + user: User | None = Depends(session_caller), +) -> FilesCaller: + """The `/api/files` caller: a signed-in user, or a valid token's user with its grants.""" + if user is not None: + return FilesCaller(user=user) + return _token_caller(request, db, bearer_token(request) or "") + + +def _refuse_token(key: str, reason: str) -> Exception: + _bad.allow(key) + log.info( + "a request brought an API token that does not hold", + extra=logs.event("apitoken.refused", reason=reason), + ) + return refused(401, "not_authenticated") + + +def _token_caller(request: Request, db: Session, raw: str) -> FilesCaller: + key = client_key(request) + if _bad.exhausted(key): + raise refused(429, "api_token_rate_limited") + if not well_formed(raw): + raise _refuse_token(key, "malformed") + token = db.execute( + select(ApiToken).where(ApiToken.token_hash == hash_token(raw)) + ).scalar_one_or_none() + if token is None: + raise _refuse_token(key, "unknown") + now = datetime.now(timezone.utc) + if token.expires_at is not None and token.expires_at <= now: + raise _refuse_token(key, "expired") + user = db.get(User, token.user_id) + if user is None or not user.is_active or user.is_suspended: + raise _refuse_token(key, "account") + if not _calls.allow(str(token.id)): + raise refused(429, "api_token_rate_limited") + grants = { + g.folder_id: grant_rights(g) + for g in db.execute(select(ApiTokenGrant).where(ApiTokenGrant.token_id == token.id)) + .scalars() + .all() + } + caller = FilesCaller(user=user, token_id=token.id, token_prefix=token.prefix, grants=grants) + db.execute( + update(ApiToken) + .where( + ApiToken.id == token.id, + or_(ApiToken.last_used_at.is_(None), ApiToken.last_used_at < now - _TOUCH_EVERY), + ) + .values(last_used_at=now) + .execution_options(synchronize_session=False) + ) + # Committed here, ahead of the route: a read-only route never commits, and a write route must + # not carry this along into a rollback of its own. + db.commit() + logs.set_user(user.id, *debugsession.level_for(user, None)) + request.state.user_id = user.id + return caller + + +def revoke_all(db: Session, user_id: int) -> int: + """Revoke every token of `user_id`, each with its audit row (NOT committed). A token is a + credential like a session, so whatever ends every session of an account — a password reset or + change, "log out everywhere", a revoked untrusted password — ends its tokens too: one minted + from a stolen session must not outlive the owner's answer to that theft. Returns how many.""" + gone = ( + db.execute(delete(ApiToken).where(ApiToken.user_id == user_id).returning(ApiToken.id)) + .scalars() + .all() + ) + for token_id in gone: + audit.record( + db, + user_id, + AuditAction.API_TOKEN_REVOKE, + target_type="api_token", + target_id=token_id, + summary="revoked with every session of the account", + ) + return len(gone) + + +def expiring(token: ApiToken, now: datetime) -> bool: + """Whether `token` runs out within the warning span (and has not yet).""" + return token.expires_at is not None and now < token.expires_at <= now + EXPIRY_WARNING + + +def warn_expiring(db: Session) -> int: + """Tell each user whose token runs out within `EXPIRY_WARNING` — once per token: the mark and the + notice commit together, and a token made again (a new row) warns afresh. Returns how many.""" + now = datetime.now(timezone.utc) + hits = db.execute( + update(ApiToken) + .where( + ApiToken.expiry_warned.is_(False), + ApiToken.expires_at.is_not(None), + ApiToken.expires_at > now, + ApiToken.expires_at <= now + EXPIRY_WARNING, + ) + .values(expiry_warned=True) + .returning(ApiToken.id, ApiToken.user_id, ApiToken.name, ApiToken.expires_at) + .execution_options(synchronize_session=False) + ).all() + for token_id, user_id, name, expires_at in hits: + create_notification( + db, + user_id, + "api_token_expiring", + title=f"API token {name} expires soon", # English fallback; the client names it from data + data={"token_id": token_id, "name": name, "expires_at": iso(expires_at)}, + commit=False, + ) + db.commit() + return len(hits) diff --git a/backend/app/audit.py b/backend/app/audit.py index c7180113..d014d313 100644 --- a/backend/app/audit.py +++ b/backend/app/audit.py @@ -69,6 +69,16 @@ class AuditAction: FILE_FOLDER_RENAME = "file_folder_rename" FILE_FOLDER_RETENTION = "file_folder_retention" FILE_FOLDER_DELETE = "file_folder_delete" + # A machine's token to the library (`routes.tokens`): made (its folders and rights) or revoked. + # What a token then did is the file rows above, their summary naming it. + API_TOKEN_CREATE = "api_token_create" + API_TOKEN_REVOKE = "api_token_revoke" + # A public `/watch` link to a file or a folder (`routes.downloads`): made, its settings changed, + # revoked. + # Never its token or password — only whether it has one. + DOWNLOAD_LINK_CREATE = "download_link_create" + DOWNLOAD_LINK_UPDATE = "download_link_update" + DOWNLOAD_LINK_REVOKE = "download_link_revoke" def record( diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py index 6908f06c..50f2d98e 100644 --- a/backend/app/downloads/folders.py +++ b/backend/app/downloads/folders.py @@ -1,20 +1,23 @@ """The folders of the Files library — where each of a user's files sits. A folder is a row, never a directory: on disk a file stays where the download layout put it (one -shared asset can be held by several users), so a rename or a move is a column update. Two kinds: +shared asset can be held by several users), so a rename or a move is a column update. Three kinds: - a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips, uploads, one per other site). Its `source_key` names the origin; the client renders its name from the key, in the reader's language. - an OWN folder is the user's, with the name they gave it. +- the HUB folder (one per user, made when they first give a machine a token for it, `hub_folder`) is + where they swap files with that machine. Its name, like a source folder's, is the client's; a file + may be moved in like into an own folder, and it keeps a file until it is deleted. Every file of a user is in exactly one folder. The folder a new file lands in is decided here (`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file -moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube -file in the TikTok folder would be a lie the folder's name tells. +moves only into one of the user's own folders, their Hub folder, or back to its own source folder +(`move`): a YouTube file in the TikTok folder would be a lie the folder's name tells. Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the -folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A +folder was made; the user can change it), Uploads, own folders and the Hub keep it until it is deleted. A done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user kept it — and its `ready_at`, the moment it finished (`stamp_done`). """ @@ -35,6 +38,7 @@ from app.models import DownloadJob, FileFolder, MediaAsset class FolderKind(StrEnum): SOURCE = "source" OWN = "own" + HUB = "hub" class SourceKey(StrEnum): @@ -123,6 +127,26 @@ def source_folder(db: Session, user_id: int, key: str) -> FileFolder: return folder +def hub_folder(db: Session, user_id: int) -> FileFolder: + """The user's Hub folder, made on first use; a race on its one-per-user index is settled like + `source_folder`'s.""" + stmt = select(FileFolder).where( + FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB + ) + folder = db.execute(stmt).scalar_one_or_none() + if folder is not None: + return folder + folder = FileFolder(user_id=user_id, kind=FolderKind.HUB) + try: + with db.begin_nested(): + db.add(folder) + db.flush() + # silent-ok: another request made the Hub folder first; re-selecting it is the whole handling. + except IntegrityError: + folder = db.execute(stmt).scalar_one() + return folder + + def place(db: Session, job: DownloadJob) -> None: """Put a new job's file in its source folder, and record that origin on it. Called wherever a job is created; the job must carry its `user_id`, `source_kind` and source refs already.""" @@ -178,8 +202,9 @@ def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder: - """The folder an upload asked to land in, if it may: one of the user's own folders, or their - Uploads folder. A file put in a site's folder would claim an origin it does not have.""" + """The folder an upload asked to land in, if it may: one of the user's own folders, their Hub + folder, or their Uploads folder. A file put in a site's folder would claim an origin it does not + have.""" folder = own_folder(db, user_id, folder_id, hold=hold) if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD: raise FolderError("file_folder_not_its_source") @@ -201,7 +226,8 @@ def set_kept(db: Session, job: DownloadJob, kept: bool) -> None: def move(db: Session, job: DownloadJob, folder: FileFolder) -> None: - """Put a file in `folder` — one of its user's own folders, or the source folder it came from. + """Put a file in `folder` — one of its user's own folders, their Hub folder, or the source folder + it came from. The caller has checked that `folder` is the job's user's.""" if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job): raise FolderError("file_folder_not_its_source") diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py index 32d9a83a..50f6bb42 100644 --- a/backend/app/downloads/gc.py +++ b/backend/app/downloads/gc.py @@ -1,6 +1,6 @@ """Download retention garbage collector (a scheduler job). -Five passes each run: +Six passes each run: 1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls within the grace window, and that its user did not keep, warns its user once (`expiry_warned`). 2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold @@ -14,6 +14,8 @@ Five passes each run: re-download renames an asset, e.g. old accented channel dirs, or an upload's stranded file). It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour, so an in-flight download is never touched. + 6. Token expiry warning — a machine's token for the library about to run out warns its user once + (`apitokens.warn_expiring`): the library's other clock. Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota. """ @@ -25,7 +27,7 @@ from pathlib import Path from sqlalchemy import func, or_, select, update from sqlalchemy.orm import Session -from app import logs, sysconfig +from app import apitokens, logs, sysconfig from app.config import settings from app.downloads import quota, service, storage from app.downloads.status import JobStatus @@ -366,12 +368,16 @@ def run_download_gc(db: Session) -> dict: # posters). Runs last so the keep-set reflects the assets that survived passes 2-4. swept = _sweep_orphan_files(db, settings.files_root) + # 6. A machine's token about to run out. + tokens_warned = apitokens.warn_expiring(db) + result = { "warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept, + "tokens_warned": tokens_warned, } log.info("download_gc %s", result) return result diff --git a/backend/app/downloads/links.py b/backend/app/downloads/links.py index 0f023774..80a0ac77 100644 --- a/backend/app/downloads/links.py +++ b/backend/app/downloads/links.py @@ -1,10 +1,15 @@ """Public "share by link" helpers for the download center. -A DownloadLink is a capability URL: the unguessable `token` grants anyone read access to one -download's file on the login-free `/watch/{token}` page. Password-protected links additionally -require a short-lived signed **grant** on the file request — because a `