fix(files): ending every session ends every API token, and the share links hold their folder

- A password reset or change, "log out everywhere" and a revoked untrusted password revoke all of the
  account's API tokens too (audited): one minted from a stolen session no longer outlives the answer.
- A bearer token or link grant with a non-ASCII character is refused (401/403), not a 500.
- Making a token's grants or a folder link holds the folder, so a racing delete cannot fail the
  foreign key; a link's view count is one atomic UPDATE.
- A folder link's card counts exactly the files its page lists (one `links.folder_files`).
- A file with no extension is served without a trailing dot.
- 0106's downgrade names a Hub folder so it never collides with one of the user's folders.
- The speed baseline reads share tokens through the app (the database keeps them hashed/encrypted).
- Tests: the token scope on the real session stack, the folder poster, the Hub refused for a link,
  0106's downgrade data step, 0107 without a key; docs/comments follow (three folder kinds, folder
  links, key change, the cleanup job's token warnings).
This commit is contained in:
2026-10-15 04:58:16 +02:00
parent 86712b6b0e
commit 827df7494c
23 changed files with 399 additions and 96 deletions
+17 -2
View File
@@ -89,7 +89,22 @@ def downgrade() -> None:
op.drop_table("api_token_grants") op.drop_table("api_token_grants")
op.drop_table("api_tokens") op.drop_table("api_tokens")
op.drop_index("uq_file_folder_hub", table_name="file_folders") op.drop_index("uq_file_folder_hub", table_name="file_folders")
# A Hub folder may hold files (their foreign key RESTRICTs a delete): it becomes an own folder. # A Hub folder may hold files (their foreign key RESTRICTs a delete): it becomes an own folder,
op.execute("UPDATE file_folders SET kind = 'own', name = 'Hub ' || id WHERE kind = 'hub'") # named "Hub <id>" — or "Hub <id> (2)", … when its user already has a folder of that name,
# which `uq_file_folder_own_name` (per user, case-blind) would refuse.
conn = op.get_bind()
hubs = conn.execute(sa.text("SELECT id, user_id FROM file_folders WHERE kind = 'hub'")).all()
for folder_id, user_id in hubs:
name, n = f"Hub {folder_id}", 1
while conn.execute(
sa.text("SELECT 1 FROM file_folders WHERE user_id = :u AND lower(name) = lower(:n)"),
{"u": user_id, "n": name},
).first():
n += 1
name = f"Hub {folder_id} ({n})"
conn.execute(
sa.text("UPDATE file_folders SET kind = 'own', name = :n WHERE id = :i"),
{"n": name, "i": folder_id},
)
op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check") op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check")
op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_OLD) op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_OLD)
+27 -3
View File
@@ -23,10 +23,11 @@ from datetime import datetime, timedelta, timezone
from enum import StrEnum from enum import StrEnum
from fastapi import Depends, Request from fastapi import Depends, Request
from sqlalchemy import or_, select, update from sqlalchemy import delete, or_, select, update
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import debugsession, logs from app import audit, debugsession, logs
from app.audit import AuditAction
from app.auth import current_user from app.auth import current_user
from app.db import get_db from app.db import get_db
from app.models import ApiToken, ApiTokenGrant, User from app.models import ApiToken, ApiTokenGrant, User
@@ -99,7 +100,8 @@ def well_formed(raw: str) -> bool:
if len(raw) != TOKEN_LEN or not raw.startswith(PREFIX): if len(raw) != TOKEN_LEN or not raw.startswith(PREFIX):
return False return False
body, check = raw[len(PREFIX) : -_CHECK_LEN], raw[-_CHECK_LEN:] body, check = raw[len(PREFIX) : -_CHECK_LEN], raw[-_CHECK_LEN:]
if any(c not in _ALPHABET for c in body): # The check too: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it.
if any(c not in _ALPHABET for c in body + check):
return False return False
return secrets.compare_digest(_check(body), check) return secrets.compare_digest(_check(body), check)
@@ -215,6 +217,28 @@ def _token_caller(request: Request, db: Session, raw: str) -> FilesCaller:
return caller return caller
def revoke_all(db: Session, user_id: int) -> int:
"""Revoke every token of `user_id`, each with its audit row (NOT committed). A token is a
credential like a session, so whatever ends every session of an account — a password reset or
change, "log out everywhere", a revoked untrusted password — ends its tokens too: one minted
from a stolen session must not outlive the owner's answer to that theft. Returns how many."""
gone = (
db.execute(delete(ApiToken).where(ApiToken.user_id == user_id).returning(ApiToken.id))
.scalars()
.all()
)
for token_id in gone:
audit.record(
db,
user_id,
AuditAction.API_TOKEN_REVOKE,
target_type="api_token",
target_id=token_id,
summary="revoked with every session of the account",
)
return len(gone)
def expiring(token: ApiToken, now: datetime) -> bool: def expiring(token: ApiToken, now: datetime) -> bool:
"""Whether `token` runs out within the warning span (and has not yet).""" """Whether `token` runs out within the warning span (and has not yet)."""
return token.expires_at is not None and now < token.expires_at <= now + EXPIRY_WARNING return token.expires_at is not None and now < token.expires_at <= now + EXPIRY_WARNING
+2 -1
View File
@@ -73,7 +73,8 @@ class AuditAction:
# What a token then did is the file rows above, their summary naming it. # What a token then did is the file rows above, their summary naming it.
API_TOKEN_CREATE = "api_token_create" API_TOKEN_CREATE = "api_token_create"
API_TOKEN_REVOKE = "api_token_revoke" API_TOKEN_REVOKE = "api_token_revoke"
# A public `/watch` link to a file (`routes.downloads`): made, its settings changed, revoked. # A public `/watch` link to a file or a folder (`routes.downloads`): made, its settings changed,
# revoked.
# Never its token or password — only whether it has one. # Never its token or password — only whether it has one.
DOWNLOAD_LINK_CREATE = "download_link_create" DOWNLOAD_LINK_CREATE = "download_link_create"
DOWNLOAD_LINK_UPDATE = "download_link_update" DOWNLOAD_LINK_UPDATE = "download_link_update"
+9 -7
View File
@@ -1,7 +1,7 @@
"""The folders of the Files library — where each of a user's files sits. """The folders of the Files library — where each of a user's files sits.
A folder is a row, never a directory: on disk a file stays where the download layout put it (one A folder is a row, never a directory: on disk a file stays where the download layout put it (one
shared asset can be held by several users), so a rename or a move is a column update. Two kinds: shared asset can be held by several users), so a rename or a move is a column update. Three kinds:
- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips, - a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
uploads, one per other site). Its `source_key` names the origin; the client renders its name uploads, one per other site). Its `source_key` names the origin; the client renders its name
@@ -13,11 +13,11 @@ shared asset can be held by several users), so a rename or a move is a column up
Every file of a user is in exactly one folder. The folder a new file lands in is decided here Every file of a user is in exactly one folder. The folder a new file lands in is decided here
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file (`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube moves only into one of the user's own folders, their Hub folder, or back to its own source folder
file in the TikTok folder would be a lie the folder's name tells. (`move`): a YouTube file in the TikTok folder would be a lie the folder's name tells.
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A folder was made; the user can change it), Uploads, own folders and the Hub keep it until it is deleted. A
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
kept it — and its `ready_at`, the moment it finished (`stamp_done`). kept it — and its `ready_at`, the moment it finished (`stamp_done`).
""" """
@@ -202,8 +202,9 @@ def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True)
def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder: def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
"""The folder an upload asked to land in, if it may: one of the user's own folders, or their """The folder an upload asked to land in, if it may: one of the user's own folders, their Hub
Uploads folder. A file put in a site's folder would claim an origin it does not have.""" folder, or their Uploads folder. A file put in a site's folder would claim an origin it does not
have."""
folder = own_folder(db, user_id, folder_id, hold=hold) folder = own_folder(db, user_id, folder_id, hold=hold)
if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD: if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
raise FolderError("file_folder_not_its_source") raise FolderError("file_folder_not_its_source")
@@ -225,7 +226,8 @@ def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
def move(db: Session, job: DownloadJob, folder: FileFolder) -> None: def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
"""Put a file in `folder` — one of its user's own folders, or the source folder it came from. """Put a file in `folder` — one of its user's own folders, their Hub folder, or the source folder
it came from.
The caller has checked that `folder` is the job's user's.""" The caller has checked that `folder` is the job's user's."""
if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job): if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
raise FolderError("file_folder_not_its_source") raise FolderError("file_folder_not_its_source")
+39 -3
View File
@@ -1,7 +1,8 @@
"""Public "share by link" helpers for the download center. """Public "share by link" helpers for the download center.
A DownloadLink is a capability URL: the unguessable token grants anyone read access to one A DownloadLink is a capability URL: the unguessable token grants anyone read access to one
download's file on the login-free `/watch/{token}` page. So the token is never stored as written: download's file, or to the files in one of its owner's own folders (`folder_files`), on the
login-free `/watch/{token}` page. So the token is never stored as written:
the link is found by its SHA-256, and the owner's copy is Fernet-encrypted — a database copy alone the link is found by its SHA-256, and the owner's copy is Fernet-encrypted — a database copy alone
opens nothing. Without that copy (no TOKEN_ENCRYPTION_KEY, or a key since changed) the link still opens nothing. Without that copy (no TOKEN_ENCRYPTION_KEY, or a key since changed) the link still
opens; only its address cannot be shown again, and the owner makes a new link to share. opens; only its address cannot be shown again, and the owner makes a new link to share.
@@ -21,9 +22,14 @@ from sqlalchemy.orm import Session
from app import security from app import security
from app.config import settings from app.config import settings
from app.models import DownloadLink from app.downloads import storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks _GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
# MAX_LIST`; a test holds the two equal — this layer may not import that one).
FOLDER_MAX = 500
def new_token() -> str: def new_token() -> str:
@@ -48,6 +54,34 @@ def recover(link: DownloadLink) -> str | None:
return security.decrypt_optional(link.token_enc) return security.decrypt_optional(link.token_enc)
def in_folder(folder: FileFolder) -> tuple:
"""Which jobs a link to `folder` reaches: the done files in it NOW, the folder owner's. A file
moved out stops opening through the link at its next request; one moved in starts."""
return (
DownloadJob.folder_id == folder.id,
DownloadJob.user_id == folder.user_id,
DownloadJob.status == JobStatus.DONE,
)
def folder_files(db: Session, folder: FileFolder) -> list[tuple[DownloadJob, MediaAsset]]:
"""What a link to `folder` shows — its page lists exactly these, and its link card counts them:
each file `in_folder` whose asset is ready and whose bytes are on disk (so the list never offers
a dead row), newest first, at most `FOLDER_MAX`."""
rows = db.execute(
select(DownloadJob, MediaAsset)
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(*in_folder(folder), MediaAsset.status == "ready")
.order_by(DownloadJob.ready_at.desc().nulls_last(), DownloadJob.id.desc())
.limit(FOLDER_MAX)
).all()
return [
(job, asset)
for job, asset in rows
if storage.safe_abs_path(settings.files_root, asset.rel_path) is not None
]
def is_expired(link: DownloadLink) -> bool: def is_expired(link: DownloadLink) -> bool:
return link.expires_at is not None and link.expires_at <= datetime.now(timezone.utc) return link.expires_at is not None and link.expires_at <= datetime.now(timezone.utc)
@@ -81,7 +115,9 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
return False return False
if exp < datetime.now(timezone.utc).timestamp(): if exp < datetime.now(timezone.utc).timestamp():
return False return False
return hmac.compare_digest(sig, _sign(f"{token}.{exp}.{password_version}")) # As bytes: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it, and
# `sig` is whatever the URL carried.
return hmac.compare_digest(sig.encode(), _sign(f"{token}.{exp}.{password_version}").encode())
# --- serialization ------------------------------------------------------------------------- # --- serialization -------------------------------------------------------------------------
+1 -9
View File
@@ -11,13 +11,11 @@ import html
import re import re
from pathlib import Path from pathlib import Path
from sqlalchemy import func, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.config import settings from app.config import settings
from app.downloads import links as linksmod from app.downloads import links as linksmod
from app.downloads import storage from app.downloads import storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL) _OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
@@ -157,13 +155,7 @@ def _folder_tags(db: Session, link: DownloadLink, url: str) -> str | None:
folder = db.get(FileFolder, link.folder_id) folder = db.get(FileFolder, link.folder_id)
if folder is None or not folder.name: if folder is None or not folder.name:
return None return None
count = db.execute( count = len(linksmod.folder_files(db, folder)) # the page's own list, so the two agree
select(func.count(DownloadJob.id)).where(
DownloadJob.folder_id == folder.id,
DownloadJob.user_id == folder.user_id,
DownloadJob.status == JobStatus.DONE,
)
).scalar_one()
title = folder.name.strip() title = folder.name.strip()
about = f"{count} file{'' if count == 1 else 's'} · Shared via Siftlode" about = f"{count} file{'' if count == 1 else 's'} · Shared via Siftlode"
return "\n ".join( return "\n ".join(
+5 -1
View File
@@ -294,9 +294,13 @@ def served_media_type(media_kind: str | None, container: str | None) -> str | No
def download_filename(display_name: str, container: str | None, path: Path) -> str: def download_filename(display_name: str, container: str | None, path: Path) -> str:
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single """The Content-Disposition filename for a served download: the cleaned display name carrying a single
correct extension — the asset container, else the file's own suffix — without doubling it when the correct extension — the asset container, else the file's own suffix — without doubling it when the
name already ends in that extension. Shared by the private download and the public watch endpoints.""" name already ends in that extension. A file with no extension at all is served without one —
never as `name.` (Windows drops a trailing dot, so the saved file would not be the one named).
Shared by the private download and the public watch endpoints."""
ext = container or path.suffix.lstrip(".") ext = container or path.suffix.lstrip(".")
base = display_filename(display_name) base = display_filename(display_name)
if not ext:
return base
if base.lower().endswith(f".{ext.lower()}"): if base.lower().endswith(f".{ext.lower()}"):
base = base[: -(len(ext) + 1)] base = base[: -(len(ext) + 1)]
return f"{base}.{ext}" return f"{base}.{ext}"
+17 -13
View File
@@ -16,6 +16,7 @@ from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import apitokens
from app import email as email_mod from app import email as email_mod
from app import logs from app import logs
from app import mailtext from app import mailtext
@@ -446,7 +447,7 @@ async def callback(
# SEC-10: drop any password nobody proved the mailbox for, now that Google attests # SEC-10: drop any password nobody proved the mailbox for, now that Google attests
# ownership (bumps the epoch to revoke any session it minted). Runs before the # ownership (bumps the epoch to revoke any session it minted). Runs before the
# activation below, which is what it reads. # activation below, which is what it reads.
revoke_untrusted_preset_password(user) revoke_untrusted_preset_password(db, user)
# Google verified the email and is_allowed granted access, so finish activating a # Google verified the email and is_allowed granted access, so finish activating a
# previously pending password registration (otherwise current_user would bounce it). # previously pending password registration (otherwise current_user would bounce it).
user.email_verified = True user.email_verified = True
@@ -486,7 +487,7 @@ async def callback(
# real owner, signed in through Google right now, sets a new password in Settings. # real owner, signed in through Google right now, sets a new password in Settings.
if canonical_email(email) in env_admin_canonicals(): if canonical_email(email) in env_admin_canonicals():
if user.role != "admin" and user.password_hash is not None: if user.role != "admin" and user.password_hash is not None:
_revoke_password(user) _revoke_password(db, user)
user.role = "admin" user.role = "admin"
elif not user.role: elif not user.role:
user.role = "user" user.role = "user"
@@ -1118,9 +1119,8 @@ def password_reset_confirm(body: TokenPasswordIn, db: Session = Depends(get_db))
# about. The person still confirms their address the normal way. # about. The person still confirms their address the normal way.
if kind == "reset": if kind == "reset":
user.email_verified = True user.email_verified = True
bump_session_epoch( # SA4: a reset (often a compromise response) kills all existing sessions — and API tokens.
user bump_session_epoch(db, user)
) # SA4: a reset (often a compromise response) kills all existing sessions.
# Burn any other outstanding reset tokens for this user. # Burn any other outstanding reset tokens for this user.
for row in db.execute( for row in db.execute(
select(AuthToken).where( select(AuthToken).where(
@@ -1143,13 +1143,17 @@ def _remember_epoch(request: Request, user: User) -> None:
request.session["epochs"] = epochs request.session["epochs"] = epochs
def bump_session_epoch(user: User) -> None: def bump_session_epoch(db: Session, user: User) -> None:
"""Invalidate every EXISTING signed cookie for this account by advancing its epoch (SA4). Cookies """Invalidate every EXISTING signed cookie for this account by advancing its epoch (SA4). Cookies
minted before the bump carry a now-stale epoch and are rejected by `current_user`.""" minted before the bump carry a now-stale epoch and are rejected by `current_user`. The account's
API tokens go with them (`apitokens.revoke_all`): a token minted from a stolen session must not
outlive the answer to that theft. Not committed — the caller commits it with its change."""
user.session_epoch = (user.session_epoch or 0) + 1 user.session_epoch = (user.session_epoch or 0) + 1
if user.id is not None:
apitokens.revoke_all(db, user.id)
def revoke_untrusted_preset_password(user: User) -> None: def revoke_untrusted_preset_password(db: Session, user: User) -> None:
"""SEC-10 (account pre-emption): a password set on an account whose email was NEVER verified is """SEC-10 (account pre-emption): a password set on an account whose email was NEVER verified is
untrusted — whoever registered the address (to pre-empt the real owner) could have chosen it. When untrusted — whoever registered the address (to pre-empt the real owner) could have chosen it. When
the real owner attests ownership through Google, drop that password and bump the epoch to revoke any the real owner attests ownership through Google, drop that password and bump the epoch to revoke any
@@ -1183,15 +1187,15 @@ def revoke_untrusted_preset_password(user: User) -> None:
_send_reset_if_eligible keyed off exactly that, so a revoked account silently got no reset link and _send_reset_if_eligible keyed off exactly that, so a revoked account silently got no reset link and
was locked out for good (it could not sign in either). The stamp separates the two cases.""" was locked out for good (it could not sign in either). The stamp separates the two cases."""
if not user.email_verified or (not user.is_active and user.password_hash is not None): if not user.email_verified or (not user.is_active and user.password_hash is not None):
_revoke_password(user) _revoke_password(db, user)
def _revoke_password(user: User) -> None: def _revoke_password(db: Session, user: User) -> None:
"""Drop the account's password, stamp why (so password-reset still accepts it) and end every """Drop the account's password, stamp why (so password-reset still accepts it) and end every
session it signed in.""" session it signed in."""
user.password_hash = None user.password_hash = None
user.password_revoked_at = datetime.now(timezone.utc) user.password_revoked_at = datetime.now(timezone.utc)
bump_session_epoch(user) bump_session_epoch(db, user)
@router.post("/logout-others") @router.post("/logout-others")
@@ -1203,7 +1207,7 @@ def logout_others(
in somewhere" / a suspected session compromise.""" in somewhere" / a suspected session compromise."""
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current # Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
# session at a newer epoch than the DB (see set_password). # session at a newer epoch than the DB (see set_password).
bump_session_epoch(user) bump_session_epoch(db, user)
db.commit() db.commit()
_remember_epoch(request, user) _remember_epoch(request, user)
log.info("Logout-others: uid=%s (epoch=%s)", user.id, user.session_epoch) log.info("Logout-others: uid=%s (epoch=%s)", user.id, user.session_epoch)
@@ -1255,7 +1259,7 @@ def set_password(
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so # SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
# the person making the change stays signed in here. Commit BEFORE re-stamping so a failed commit # the person making the change stays signed in here. Commit BEFORE re-stamping so a failed commit
# can't leave the cookie at a newer epoch than the DB (which would wrongly 401 THIS session). # can't leave the cookie at a newer epoch than the DB (which would wrongly 401 THIS session).
bump_session_epoch(user) bump_session_epoch(db, user)
db.commit() db.commit()
_remember_epoch(request, user) _remember_epoch(request, user)
log.info("Password set/changed: uid=%s", user.id) log.info("Password set/changed: uid=%s", user.id)
+9 -2
View File
@@ -27,6 +27,7 @@ from app.db import get_db
from app.downloads import clip as clipmod from app.downloads import clip as clipmod
from app.downloads import credentials as dlcreds from app.downloads import credentials as dlcreds
from app.downloads import edit as editmod from app.downloads import edit as editmod
from app.downloads import folders
from app.downloads import links as linksmod from app.downloads import links as linksmod
from app.downloads import quota, service, storage from app.downloads import quota, service, storage
from app.downloads import urlguard from app.downloads import urlguard
@@ -806,8 +807,14 @@ def create_link(
def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder: def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
"""One of the user's own folders. A source folder lets its files go on its own clock, and the Hub """One of the user's own folders. A source folder lets its files go on its own clock, and the Hub
is the user's channel to a machine of theirs: neither is a shelf to hand out.""" is the user's channel to a machine of theirs: neither is a shelf to hand out. Held (FOR SHARE)
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder") for the rest of the transaction, so a delete of it at the same moment waits and takes the new
link with it, rather than failing the link's foreign key."""
owned_or_404(db, FileFolder, folder_id, user, what="folder")
try:
folder = folders.own_folder(db, user.id, folder_id)
except folders.FolderError as e:
raise refused(409, e.reason) from e
if folder.kind != FolderKind.OWN: if folder.kind != FolderKind.OWN:
raise refused(409, "share_folder_kind") raise refused(409, "share_folder_kind")
return folder return folder
+12 -29
View File
@@ -9,7 +9,7 @@ a short-lived signed grant that the media URL carries (a `<video src>` can't sen
from fastapi import APIRouter, Depends, Request from fastapi import APIRouter, Depends, Request
from fastapi.responses import FileResponse from fastapi.responses import FileResponse
from pydantic import BaseModel from pydantic import BaseModel
from sqlalchemy import select from sqlalchemy import select, update
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.config import settings from app.config import settings
@@ -22,7 +22,6 @@ from app.ratelimit import RateLimiter, client_key
from app.security import verify_password from app.security import verify_password
from app.refusal import refused from app.refusal import refused
from app.routes._common import not_found from app.routes._common import not_found
from app.routes.files import MAX_LIST
router = APIRouter(prefix="/api/public", tags=["public"]) router = APIRouter(prefix="/api/public", tags=["public"])
@@ -37,9 +36,6 @@ _unlock_limit = RateLimiter(max_events=10, window_seconds=300)
# already running when it fills are still checked — as many as the route's threadpool runs at once. # already running when it fills are still checked — as many as the route's threadpool runs at once.
_unlock_failures = RateLimiter(max_events=50, window_seconds=300) _unlock_failures = RateLimiter(max_events=50, window_seconds=300)
# The most files a folder link lists: as many as the owner's own Files list shows.
_FOLDER_MAX = MAX_LIST
def _link_or_404(db: Session, token: str) -> DownloadLink: def _link_or_404(db: Session, token: str) -> DownloadLink:
link = linksmod.find(db, token) link = linksmod.find(db, token)
@@ -89,16 +85,6 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs
return _ready_job(db, db.get(DownloadJob, link.job_id)) return _ready_job(db, db.get(DownloadJob, link.job_id))
def _in_folder(link: DownloadLink, folder: FileFolder) -> tuple:
"""What a folder link shows: the done files in its folder NOW, the folder owner's. A file moved
out stops opening through the link at its next request; one moved in starts."""
return (
DownloadJob.folder_id == link.folder_id,
DownloadJob.user_id == folder.user_id,
DownloadJob.status == JobStatus.DONE,
)
def _link_folder(db: Session, link: DownloadLink) -> FileFolder: def _link_folder(db: Session, link: DownloadLink) -> FileFolder:
folder = db.get(FileFolder, link.folder_id) if link.folder_id is not None else None folder = db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
if folder is None: if folder is None:
@@ -109,7 +95,7 @@ def _link_folder(db: Session, link: DownloadLink) -> FileFolder:
def _folder_job(db: Session, link: DownloadLink, job_id: int) -> tuple[DownloadJob, MediaAsset]: def _folder_job(db: Session, link: DownloadLink, job_id: int) -> tuple[DownloadJob, MediaAsset]:
folder = _link_folder(db, link) folder = _link_folder(db, link)
job = db.execute( job = db.execute(
select(DownloadJob).where(DownloadJob.id == job_id, *_in_folder(link, folder)) select(DownloadJob).where(DownloadJob.id == job_id, *linksmod.in_folder(folder))
).scalar_one_or_none() ).scalar_one_or_none()
if job is None: if job is None:
raise refused(404, "video_gone") raise refused(404, "video_gone")
@@ -136,16 +122,9 @@ def _poster_url(
def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict: def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
"""A folder link's page: the folder's name and each ready file's own watch metadata, newest """A folder link's page: the folder's name and each file's own watch metadata
first. A file whose bytes are not on disk is left out, so the list never offers a dead row.""" (`linksmod.folder_files`: ready, on disk, newest first)."""
folder = _link_folder(db, link) folder = _link_folder(db, link)
rows = db.execute(
select(DownloadJob, MediaAsset)
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(*_in_folder(link, folder), MediaAsset.status == "ready")
.order_by(DownloadJob.ready_at.desc().nulls_last(), DownloadJob.id.desc())
.limit(_FOLDER_MAX)
).all()
files = [ files = [
{ {
"id": job.id, "id": job.id,
@@ -157,8 +136,7 @@ def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None)
_poster_url(token, asset, grant, job.id), _poster_url(token, asset, grant, job.id),
), ),
} }
for job, asset in rows for job, asset in linksmod.folder_files(db, folder)
if storage.safe_abs_path(settings.files_root, asset.rel_path) is not None
] ]
return { return {
"needs_password": False, "needs_password": False,
@@ -187,8 +165,13 @@ def _opened(db: Session, link: DownloadLink, token: str, grant: str | None = Non
if link.folder_id is not None if link.folder_id is not None
else _file_meta(db, link, token, grant) else _file_meta(db, link, token, grant)
) )
link.view_count += 1 # In SQL, not `+= 1` on the loaded row: two opens at once would both write the same count.
db.flush() db.execute(
update(DownloadLink)
.where(DownloadLink.id == link.id)
.values(view_count=DownloadLink.view_count + 1)
.execution_options(synchronize_session=False)
)
return meta return meta
+2 -2
View File
@@ -104,9 +104,9 @@ def _wanted_grants(db: Session, user: User, body: TokenIn) -> dict[int, set[Righ
for folder_id, rights in wanted.items(): for folder_id, rights in wanted.items():
try: try:
if Right.WRITE in rights: if Right.WRITE in rights:
folders.upload_target(db, user.id, folder_id, hold=False) folders.upload_target(db, user.id, folder_id)
else: else:
folders.own_folder(db, user.id, folder_id, hold=False) folders.own_folder(db, user.id, folder_id)
except folders.FolderError as e: except folders.FolderError as e:
raise refused(409, e.reason) from e raise refused(409, e.reason) from e
return wanted return wanted
+2 -1
View File
@@ -370,7 +370,8 @@ def _explore_cleanup_job() -> None:
def _download_gc_job() -> None: def _download_gc_job() -> None:
# Retention GC for the download center: pre-expiry warnings, TTL deletion, LRU eviction. # Retention GC for the download center: pre-expiry warnings, TTL deletion, LRU eviction — and
# the week-ahead warning before an API token runs out (`apitokens.warn_expiring`).
# Pure disk/DB work (no YouTube quota). # Pure disk/DB work (no YouTube quota).
_job("download_gc", run_download_gc) _job("download_gc", run_download_gc)
+75 -3
View File
@@ -32,6 +32,7 @@ from app.models import (
) )
from app.ratelimit import RateLimiter from app.ratelimit import RateLimiter
from app.routes import tokens as tokens_routes from app.routes import tokens as tokens_routes
from app.security import hash_token
@pytest.fixture @pytest.fixture
@@ -114,6 +115,17 @@ def test_a_token_has_its_shape_and_a_checksum_that_refuses_a_typo():
assert not apitokens.well_formed(flipped) assert not apitokens.well_formed(flipped)
assert not apitokens.well_formed(raw[:-1]) assert not apitokens.well_formed(raw[:-1])
assert not apitokens.well_formed("xxx_" + raw[4:]) assert not apitokens.well_formed("xxx_" + raw[4:])
# A header byte past ASCII arrives as one latin-1 character: refused, never a 500.
assert not apitokens.well_formed(raw[:-1] + "é")
def test_a_non_ascii_token_is_refused_as_any_bad_one(db, client):
# The test client re-sends a high byte as two UTF-8 bytes, which arrive as two latin-1
# characters: one "A" fewer keeps the header a token's length, so it is the checksum that meets
# them, not the length check.
raw = ("Bearer sfl_" + "A" * 47 + "é").encode("latin-1")
r = TestClient(app, headers={"authorization": raw}).get("/api/files/folders")
assert r.status_code == 401 and r.json()["detail"]["reason"] == "not_authenticated"
def test_the_hub_preset_makes_one_hub_folder_with_every_right_and_shows_the_token_once( def test_the_hub_preset_makes_one_hub_folder_with_every_right_and_shows_the_token_once(
@@ -248,12 +260,23 @@ def test_a_token_is_judged_by_its_grants_even_beside_a_signed_in_cookie(db, make
assert both.json() == [] # the token may not list: its grants decide assert both.json() == [] # the token may not list: its grants decide
def test_a_token_works_on_the_files_routes_only(db, make_user, as_user): def _signed_in(client, email="alice@example.com") -> TestClient:
alice = make_user() r = client.post(
raw = _make(as_user(alice), preset="hub")["token"] "/auth/password-login", json={"email": email, "password": "correct horse battery"}
)
assert r.status_code == 200, r.text
return client
def test_a_token_works_on_the_files_routes_only(db, make_user, client):
"""Through the real session stack: under `as_user` every route without its header answers 401
whatever `current_user` does with a token, so these refusals would prove nothing there."""
make_user(email="alice@example.com")
raw = _make(_signed_in(client), preset="hub")["token"]
m = _bearer(raw) m = _bearer(raw)
assert m.get("/api/files/folders").status_code == 200 assert m.get("/api/files/folders").status_code == 200
# Making a wider token, or reading the account, takes the user's session. # Making a wider token, or reading the account, takes the user's session.
assert m.get("/api/me").json() == {"authenticated": False} # the token is no one here
assert m.get("/api/tokens").status_code == 401 assert m.get("/api/tokens").status_code == 401
assert m.post("/api/tokens", json={"name": "wider", "preset": "hub"}).status_code == 401 assert m.post("/api/tokens", json={"name": "wider", "preset": "hub"}).status_code == 401
assert m.get("/api/files/usage").status_code == 401 assert m.get("/api/files/usage").status_code == 401
@@ -296,6 +319,55 @@ def test_only_the_files_routes_take_a_token():
} }
def _reset_password(db, client, alice):
from app.routes.auth import _issue_token
link = _issue_token(db, alice, "reset", timedelta(hours=1))
r = client.post(
"/auth/password-reset/confirm", json={"token": link, "password": "a brand new password"}
)
assert r.status_code == 200, r.text
def _change_password(db, client, alice):
r = client.post(
"/auth/set-password",
json={"password": "a brand new password", "current_password": "correct horse battery"},
)
assert r.status_code == 200, r.text
def _log_out_others(db, client, alice):
assert client.post("/auth/logout-others").status_code == 200
def _revoke_untrusted_password(db, client, alice):
from app.routes.auth import _revoke_password
_revoke_password(db, alice)
db.commit()
@pytest.mark.parametrize(
"end_sessions",
[_reset_password, _change_password, _log_out_others, _revoke_untrusted_password],
)
def test_whatever_ends_every_session_ends_every_token_too(db, make_user, client, end_sessions):
"""A token minted from a stolen session must not outlive the owner's answer to the theft."""
alice = make_user(email="alice@example.com")
bob = make_user(email="bob@example.com")
raw = _make(_signed_in(client), preset="hub")["token"]
bobs = apitokens.mint()
db.add(ApiToken(user_id=bob.id, name="b", prefix=bobs[:12], token_hash=hash_token(bobs)))
db.commit()
assert _bearer(raw).get("/api/files/folders").status_code == 200
end_sessions(db, client, alice)
assert _bearer(raw).get("/api/files/folders").status_code == 401
assert db.execute(select(ApiToken.user_id)).scalars().all() == [bob.id] # only hers went
[revoked] = _audits(db, "api_token_revoke")
assert revoked.actor_id == alice.id and "every session" in (revoked.summary or "")
# --- a token that does not hold ------------------------------------------------------------------ # --- a token that does not hold ------------------------------------------------------------------
+1 -1
View File
@@ -493,7 +493,7 @@ def _revoke_as_google_adoption(db, user) -> None:
Google callback adopting an account whose email was never verified.""" Google callback adopting an account whose email was never verified."""
from app.routes.auth import revoke_untrusted_preset_password from app.routes.auth import revoke_untrusted_preset_password
revoke_untrusted_preset_password(user) revoke_untrusted_preset_password(db, user)
db.commit() db.commit()
+51 -2
View File
@@ -7,6 +7,7 @@ a file link; a document is never served to open as a page.
""" """
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest import pytest
from alembic import command from alembic import command
@@ -15,9 +16,9 @@ from sqlalchemy import select, text
from app.config import settings from app.config import settings
from app.db import engine from app.db import engine
from app.downloads import og from app.downloads import folders, og
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset
_VIDEO = { _VIDEO = {
"streams": [ "streams": [
@@ -125,6 +126,12 @@ def test_only_a_folder_of_ones_own_can_be_shared(shelf, db, as_user, make_user):
r = owner.post(f"/api/downloads/folders/{source}/links", json={}) r = owner.post(f"/api/downloads/folders/{source}/links", json={})
assert r.status_code == 409 assert r.status_code == 409
assert r.json()["detail"]["reason"] == "share_folder_kind" assert r.json()["detail"]["reason"] == "share_folder_kind"
# Nor the Hub: whatever a machine drops there would be public the moment it lands.
hub = folders.hub_folder(db, shelf["user"].id).id
db.commit()
r = owner.post(f"/api/downloads/folders/{hub}/links", json={})
assert r.status_code == 409
assert r.json()["detail"]["reason"] == "share_folder_kind"
stranger = as_user(make_user()) stranger = as_user(make_user())
assert ( assert (
@@ -154,6 +161,43 @@ def test_a_password_folder_link_shows_nothing_until_unlocked(shelf, client):
assert client.get(files[0]["file_url"]).status_code == 200 assert client.get(files[0]["file_url"]).status_code == 200
def test_a_files_poster_opens_by_the_links_own_rules(shelf, db, client):
"""The poster route on its own: a password link's poster needs the unlock grant, and a file
outside the folder has none through it, grant or not."""
for job_id in (shelf["inside"][0], shelf["outside"]):
asset = db.get(MediaAsset, db.get(DownloadJob, job_id).asset_id)
asset.poster_path = f"poster-{job_id}.jpg"
(Path(settings.files_root) / asset.poster_path).write_bytes(b"a poster frame")
db.commit()
token = _link(shelf, password="open sesame")["token"]
inside = f"/api/public/watch/{token}/files/{shelf['inside'][0]}/poster.jpg"
assert client.get(inside).status_code == 403
unlocked = client.post(f"/api/public/watch/{token}/unlock", json={"password": "open sesame"})
[entry] = [f for f in unlocked.json()["files"] if f["id"] == shelf["inside"][0]]
assert entry["poster_url"].startswith(inside + "?g=")
got = client.get(entry["poster_url"])
assert got.status_code == 200 and got.content == b"a poster frame"
grant = entry["poster_url"].split("?g=", 1)[1]
outside = f"/api/public/watch/{token}/files/{shelf['outside']}/poster.jpg?g={grant}"
assert client.get(outside).status_code == 404
def test_every_open_counts_as_a_view(shelf, client):
made = _link(shelf)
for _ in range(3):
assert client.get(f"/api/public/watch/{made['token']}").status_code == 200
[listed] = shelf["owner"].get(f"/api/downloads/folders/{shelf['folder']}/links").json()
assert listed["view_count"] == 3
def test_the_folder_list_is_capped_like_the_owners_files_list():
from app.downloads import links as linksmod
from app.routes.files import MAX_LIST
assert linksmod.FOLDER_MAX == MAX_LIST
def test_a_document_in_a_shared_folder_only_downloads(shelf, client, probe): def test_a_document_in_a_shared_folder_only_downloads(shelf, client, probe):
probe["data"] = None probe["data"] = None
doc = shelf["upload"]("notes.html", shelf["folder"], b"<script>1</script>") doc = shelf["upload"]("notes.html", shelf["folder"], b"<script>1</script>")
@@ -234,6 +278,11 @@ def test_the_unfurl_names_the_folder_and_counts_its_files_only(shelf, db):
for name in names: for name in names:
assert not name or name not in tags assert not name or name not in tags
# A file whose bytes are gone is off the page, and so off the card's count.
gone = db.get(MediaAsset, db.get(DownloadJob, shelf["inside"][0]).asset_id)
(Path(settings.files_root) / gone.rel_path).unlink()
assert "1 file " in og._watch_tags(token, db)
locked = _link(shelf, password="pw")["token"] locked = _link(shelf, password="pw")["token"]
hidden = og._watch_tags(locked, db) hidden = og._watch_tags(locked, db)
assert "Friends" not in hidden and "Password-protected folder" in hidden assert "Friends" not in hidden and "Password-protected folder" in hidden
+40
View File
@@ -185,3 +185,43 @@ def test_the_downgrade_writes_the_tokens_back(db, key, owner_job):
assert db.execute(select(DownloadLink.token_hash)).scalars().all() == [ assert db.execute(select(DownloadLink.token_hash)).scalars().all() == [
security.hash_token(token) security.hash_token(token)
] ]
def test_without_a_key_the_migration_keeps_the_address_opening_but_not_shown(db, owner_job, client):
"""No key to seal with: the hash alone is kept, the link still opens, its address is gone."""
assert not security.can_encrypt()
owner, job_id = owner_job
db.commit() # an open transaction's locks would hold the downgrade's ALTER TABLE forever
cfg = Config("alembic.ini")
command.downgrade(cfg, "0106_api_tokens")
try:
with engine.begin() as conn:
conn.execute(
text("INSERT INTO download_links (job_id, token) VALUES (:j, 'an-old-token-sent')"),
{"j": job_id},
)
finally:
command.upgrade(cfg, "head")
(row,) = db.execute(text("SELECT token_hash, token_enc FROM download_links")).mappings().all()
assert row["token_hash"] == security.hash_token("an-old-token-sent")
assert row["token_enc"] is None
assert client.get("/api/public/watch/an-old-token-sent").status_code == 200
assert [e["url"] for e in owner.get(f"/api/downloads/{job_id}/links").json()] == [None]
def test_a_downgrade_with_no_readable_copy_writes_a_fresh_token(db, key, owner_job, monkeypatch):
"""A key since changed reads no copy: the old column gets a random token, so the link is
revoked rather than left with no token (the column is NOT NULL there)."""
owner, job_id = owner_job
token = owner.post(f"/api/downloads/{job_id}/links", json={}).json()["token"]
db.commit() # an open transaction's locks would hold the downgrade's ALTER TABLE forever
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
cfg = Config("alembic.ini")
command.downgrade(cfg, "0106_api_tokens")
try:
with engine.connect() as conn:
back = conn.execute(text("SELECT token FROM download_links")).scalars().all()
finally:
command.upgrade(cfg, "head")
assert len(back) == 1 and back[0] and back[0] != token
+3
View File
@@ -53,6 +53,9 @@ class TestGrant:
def test_malformed_grants_are_rejected_not_crashed(self): def test_malformed_grants_are_rejected_not_crashed(self):
for bad in [None, "", "no-dot", "notanumber.sig"]: for bad in [None, "", "no-dot", "notanumber.sig"]:
assert check_grant("tok", bad, 0) is False assert check_grant("tok", bad, 0) is False
# The query string decodes as UTF-8: a signature past ASCII is just a wrong one.
future = int(datetime.now(timezone.utc).timestamp()) + _GRANT_TTL
assert check_grant("tok", f"{future}.é", 0) is False
def _asset(**over): def _asset(**over):
+50
View File
@@ -0,0 +1,50 @@
"""Migration 0106's downgrade data step, DB-lane: a Hub folder holding a file becomes an own folder
that keeps the file — under a name its user's own folders do not already have (their names are
unique per user, case-blind), and with its token gone with the tokens table."""
from alembic import command
from alembic.config import Config
from sqlalchemy import text
from app.db import engine
def test_a_hub_folder_with_a_file_comes_back_as_an_own_folder_beside_a_namesake(db, make_user):
uid = make_user().id
with engine.begin() as conn:
hub = conn.execute(
text("INSERT INTO file_folders (user_id, kind) VALUES (:u, 'hub') RETURNING id"),
{"u": uid},
).scalar_one()
conn.execute(
text("INSERT INTO file_folders (user_id, kind, name) VALUES (:u, 'own', :n)"),
{"u": uid, "n": f"hub {hub}"}, # the name the downgrade would pick, in another case
)
held = conn.execute(
text(
"INSERT INTO download_jobs (user_id, source_kind, source_ref, profile_snapshot,"
" status, folder_id) VALUES (:u, 'upload', 'x', '{}', 'queued', :f) RETURNING id"
),
{"u": uid, "f": hub},
).scalar_one()
conn.execute(
text(
"INSERT INTO api_tokens (user_id, name, prefix, token_hash) VALUES"
" (:u, 'hub', 'sfl_abcd1234', 'h') "
),
{"u": uid},
)
cfg = Config("alembic.ini")
command.downgrade(cfg, "0105_job_source_key")
try:
with engine.connect() as conn:
kind, name = conn.execute(
text("SELECT kind, name FROM file_folders WHERE id = :i"), {"i": hub}
).one()
folder_of_job = conn.execute(
text("SELECT folder_id FROM download_jobs WHERE id = :j"), {"j": held}
).scalar_one()
finally:
command.upgrade(cfg, "head")
assert (kind, name) == ("own", f"Hub {hub} (2)")
assert folder_of_job == hub
+3
View File
@@ -96,6 +96,9 @@ class TestDownloadFilename:
def test_does_not_double_an_extension_already_present(self): def test_does_not_double_an_extension_already_present(self):
assert download_filename("clip.mp4", "mp4", Path("/x/f.mp4")) == "clip.mp4" assert download_filename("clip.mp4", "mp4", Path("/x/f.mp4")) == "clip.mp4"
def test_no_extension_is_served_without_a_trailing_dot(self):
assert download_filename("Makefile", None, Path("/x/upload-7")) == "Makefile"
def test_falls_back_to_the_file_suffix_when_no_container(self): def test_falls_back_to_the_file_suffix_when_no_container(self):
assert download_filename("song", None, Path("/x/f.m4a")) == "song.m4a" assert download_filename("song", None, Path("/x/f.m4a")) == "song.m4a"
+6 -2
View File
@@ -44,7 +44,9 @@ The installer asks for the **public URL** where the instance will be reached (ju
local `.env` file — keep that file private. **Do not change `TOKEN_ENCRYPTION_KEY` once the app has local `.env` file — keep that file private. **Do not change `TOKEN_ENCRYPTION_KEY` once the app has
stored any secret** (Google credentials, API keys, SMTP password, Plex token): those values are stored any secret** (Google credentials, API keys, SMTP password, Plex token): those values are
encrypted with it, and a changed key makes them unreadable. If it is ever lost or rotated, clear the encrypted with it, and a changed key makes them unreadable. If it is ever lost or rotated, clear the
affected rows from the `system_config` table and re-enter the secrets in the admin Configuration page, affected rows from the `system_config` table and re-enter the secrets in the admin Configuration page.
Share links are kept with it too: after a key change every existing link still opens, but its
address can no longer be shown — its owner makes a new link to share it again,
- pulls the image and starts the app + database, - pulls the image and starts the app + database,
- prints the **setup wizard URL**, which looks like `…/setup?token=…`. - prints the **setup wizard URL**, which looks like `…/setup?token=…`.
@@ -163,7 +165,9 @@ then cannot read its own secrets.
2. **`.env`** — in particular `TOKEN_ENCRYPTION_KEY`. Your Google client secret, SMTP password, Plex 2. **`.env`** — in particular `TOKEN_ENCRYPTION_KEY`. Your Google client secret, SMTP password, Plex
token and every stored refresh token are encrypted with it *inside* the database. Restore a dump token and every stored refresh token are encrypted with it *inside* the database. Restore a dump
against a different key and those values are lost — you would have to clear the affected rows from against a different key and those values are lost — you would have to clear the affected rows from
`system_config` and re-enter every secret by hand. Keep `.env` beside the dump, and just as private. `system_config` and re-enter every secret by hand — and no existing share link's address can be
shown again (the links still open; their owners make new ones to share). Keep `.env` beside the
dump, and just as private.
3. **Downloaded media** *(only if you use Files)* — the `siftlode_files` volume, or 3. **Downloaded media** *(only if you use Files)* — the `siftlode_files` volume, or
whatever `FILES_HOST_PATH` points at. Not needed to bring the instance back; the jobs and their whatever `FILES_HOST_PATH` points at. Not needed to bring the instance back; the jobs and their
metadata are in the database, the files themselves are not. metadata are in the database, the files themselves are not.
+26 -13
View File
@@ -17,7 +17,7 @@ import { retryConnect } from "./retryConnect";
const RUNS = 5; const RUNS = 5;
const OUT = "e2e/.speed/latest.json"; const OUT = "e2e/.speed/latest.json";
const CONTAINER = process.env.E2E_DB_CONTAINER || "siftlode-db-1"; const CONTAINER = process.env.E2E_API_CONTAINER || "siftlode-api-1";
const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] ?? NaN; const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] ?? NaN;
@@ -32,20 +32,33 @@ const scriptBytes = (page: Page) =>
/** Public share links on the local stack that could play: no password, not expired, asset ready. /** Public share links on the local stack that could play: no password, not expired, asset ready.
* The page itself is the judge of the last step (its file must be on THIS disk — a localdev DB * The page itself is the judge of the last step (its file must be on THIS disk — a localdev DB
* seeded from prod lists assets whose files never came along), so the caller probes them. */ * seeded from prod lists assets whose files never came along), so the caller probes them. Read
* through the API container: the database keeps a link's token only hashed and encrypted, and only
* the app holds the key to read it back (`links.recover`). */
const SHARE_TOKENS_PY = `
from sqlalchemy import select
from app.db import SessionLocal
from app.downloads import links
from app.models import DownloadJob, DownloadLink, MediaAsset
with SessionLocal() as db:
for link in db.execute(
select(DownloadLink)
.join(DownloadJob, DownloadJob.id == DownloadLink.job_id)
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadLink.password_hash.is_(None), MediaAsset.status == "ready")
.order_by(DownloadLink.id)
).scalars():
token = None if links.is_expired(link) else links.recover(link)
if token:
print(token)
`;
function shareTokens(): string[] { function shareTokens(): string[] {
const sql = // Not caught: an API container this cannot reach (no docker, a renamed container) is a failed
"select l.token from download_links l join download_jobs j on j.id = l.job_id" +
" join media_assets a on a.id = j.asset_id" +
" where l.password_hash is null and (l.expires_at is null or l.expires_at > now())" +
" and a.status = 'ready' order by l.id";
// Not caught: a database this cannot reach (no docker, a renamed container) is a failed
// measurement, not "no playable share link on the local stack". // measurement, not "no playable share link on the local stack".
const out = execFileSync( const out = execFileSync("docker", ["exec", CONTAINER, "python", "-c", SHARE_TOKENS_PY], {
"docker", encoding: "utf8",
["exec", CONTAINER, "psql", "-U", "siftlode", "-d", "siftlode", "-tAc", sql], });
{ encoding: "utf8" },
);
return out return out
.split("\n") .split("\n")
.map((t) => t.trim()) .map((t) => t.trim())
+1 -1
View File
@@ -71,7 +71,7 @@
"playlist_sync": "Mirrors each user's YouTube playlists into the app. If it stops, changes to your YouTube playlists don't show up locally.", "playlist_sync": "Mirrors each user's YouTube playlists into the app. If it stops, changes to your YouTube playlists don't show up locally.",
"maintenance": "Finds videos that can no longer be played (deleted, made private, abandoned premieres), hides them from the Feed and removes them after a grace period; re-checks the least-recently-checked videos. If it stops, dead videos linger in the catalogue.", "maintenance": "Finds videos that can no longer be played (deleted, made private, abandoned premieres), hides them from the Feed and removes them after a grace period; re-checks the least-recently-checked videos. If it stops, dead videos linger in the catalogue.",
"explore_cleanup": "Removes channels you explored but never subscribed to (and their videos) after a grace period, so curiosity browsing doesn't pile up in the catalogue. If it stops, abandoned explored channels linger.", "explore_cleanup": "Removes channels you explored but never subscribed to (and their videos) after a grace period, so curiosity browsing doesn't pile up in the catalogue. If it stops, abandoned explored channels linger.",
"download_gc": "Deletes downloaded files past their retention window, reclaims space no one is holding anymore, and warns owners before a shared file expires. If it stops, old downloads pile up and disk space isn't freed.", "download_gc": "Deletes downloaded files past their retention window, reclaims space no one is holding anymore, and warns owners before a shared file or an API token expires. If it stops, old downloads pile up, disk space isn't freed, and a token runs out unannounced.",
"plex_sync": "Mirrors the enabled Plex libraries into the app's catalogue — the metadata behind Plex browsing, search, filters and the info pages (genres, cast, ratings, artwork). If it stops, newly added or changed Plex titles don't appear until the next sync.", "plex_sync": "Mirrors the enabled Plex libraries into the app's catalogue — the metadata behind Plex browsing, search, filters and the info pages (genres, cast, ratings, artwork). If it stops, newly added or changed Plex titles don't appear until the next sync.",
"plex_watch_sync": "Pulls recent Plex watch changes (finished episodes, resume positions) into Siftlode and re-pushes any local watch changes that didn't reach Plex. The cheap two-way keep-in-sync between full reconciles. If it stops, watched/resume made in the Plex app takes longer to show here.", "plex_watch_sync": "Pulls recent Plex watch changes (finished episodes, resume positions) into Siftlode and re-pushes any local watch changes that didn't reach Plex. The cheap two-way keep-in-sync between full reconciles. If it stops, watched/resume made in the Plex app takes longer to show here.",
"plex_watch_reconcile": "A full watch-state reconcile that rescans every library — catches what the incremental pass can't, notably an episode un-marked as watched on the Plex side. Heavier, so it runs about once a day. If it stops, un-watches done in Plex aren't mirrored back.", "plex_watch_reconcile": "A full watch-state reconcile that rescans every library — catches what the incremental pass can't, notably an episode un-marked as watched on the Plex side. Heavier, so it runs about once a day. If it stops, un-watches done in Plex aren't mirrored back.",
+1 -1
View File
@@ -71,7 +71,7 @@
"playlist_sync": "Tükrözi a felhasználók YouTube lejátszási listáit az appba. Ha leáll, a YouTube-listák változásai nem jelennek meg lokálisan.", "playlist_sync": "Tükrözi a felhasználók YouTube lejátszási listáit az appba. Ha leáll, a YouTube-listák változásai nem jelennek meg lokálisan.",
"maintenance": "Megkeresi a már sehol le nem játszható videókat (törölt, priváttá tett, elhagyott premier), elrejti őket a Feedből, és türelmi idő után eltávolítja; újraellenőrzi a legrégebben ellenőrzött videókat. Ha leáll, a halott videók a katalógusban maradnak.", "maintenance": "Megkeresi a már sehol le nem játszható videókat (törölt, priváttá tett, elhagyott premier), elrejti őket a Feedből, és türelmi idő után eltávolítja; újraellenőrzi a legrégebben ellenőrzött videókat. Ha leáll, a halott videók a katalógusban maradnak.",
"explore_cleanup": "A türelmi idő után eltávolítja a felfedezett, de nem követett csatornákat (és videóikat), hogy a kíváncsiság-böngészés ne halmozódjon a katalógusban. Ha leáll, az elhagyott felfedezett csatornák megmaradnak.", "explore_cleanup": "A türelmi idő után eltávolítja a felfedezett, de nem követett csatornákat (és videóikat), hogy a kíváncsiság-böngészés ne halmozódjon a katalógusban. Ha leáll, az elhagyott felfedezett csatornák megmaradnak.",
"download_gc": "Törli a megőrzési időn túli letöltéseket, felszabadítja a már senki által nem használt helyet, és a lejárat előtt figyelmezteti a tulajdonost. Ha leáll, a régi letöltések felhalmozódnak, és a tárhely nem szabadul fel.", "download_gc": "Törli a megőrzési időn túli letöltéseket, felszabadítja a már senki által nem használt helyet, és figyelmezteti a tulajdonost, mielőtt egy megosztott fájl vagy egy API-token lejár. Ha leáll, a régi letöltések felhalmozódnak, a tárhely nem szabadul fel, és a token szó nélkül jár le.",
"plex_sync": "Tükrözi az engedélyezett Plex-könyvtárakat az app katalógusába — ez adja a Plex böngészés, keresés, szűrők és infóoldalak metaadatait (műfaj, szereplők, pontszám, borítók). Ha leáll, az újonnan hozzáadott vagy módosított Plex-címek csak a következő szinkron után jelennek meg.", "plex_sync": "Tükrözi az engedélyezett Plex-könyvtárakat az app katalógusába — ez adja a Plex böngészés, keresés, szűrők és infóoldalak metaadatait (műfaj, szereplők, pontszám, borítók). Ha leáll, az újonnan hozzáadott vagy módosított Plex-címek csak a következő szinkron után jelennek meg.",
"plex_watch_sync": "Áthozza a Plex-oldali friss megtekintés-változásokat (befejezett epizódok, folytatás-pozíciók) a Siftlode-ba, és felnyomja a Plexet el nem érő helyi változásokat. Ez az olcsó kétirányú szinkron a teljes egyeztetések között. Ha leáll, a Plex-appban jelölt megtekintés/folytatás lassabban jelenik meg itt.", "plex_watch_sync": "Áthozza a Plex-oldali friss megtekintés-változásokat (befejezett epizódok, folytatás-pozíciók) a Siftlode-ba, és felnyomja a Plexet el nem érő helyi változásokat. Ez az olcsó kétirányú szinkron a teljes egyeztetések között. Ha leáll, a Plex-appban jelölt megtekintés/folytatás lassabban jelenik meg itt.",
"plex_watch_reconcile": "A megtekintési állapot teljes egyeztetése, amely minden könyvtárat újraszken­nel — elkapja, amit az inkrementális menet nem, főleg ha egy epizódról a Plex oldalán levették a megtekintve jelet. Nehezebb, ezért kb. naponta fut. Ha leáll, a Plexben levett megtekintve jel nem tükröződik vissza.", "plex_watch_reconcile": "A megtekintési állapot teljes egyeztetése, amely minden könyvtárat újraszken­nel — elkapja, amit az inkrementális menet nem, főleg ha egy epizódról a Plex oldalán levették a megtekintve jelet. Nehezebb, ezért kb. naponta fut. Ha leáll, a Plexben levett megtekintve jel nem tükröződik vissza.",