feat(files): every file sits in a library folder, and keeps its own expiry
The Files library starts here. A user's every download, clip and upload now lands in a source folder made per origin (YouTube, Plex clips, uploads, one per other site), decided once in downloads/folders.py at every place a job is made. How long a file stays moves from the shared asset to the user's job: stamped from its folder when it is done, never for a file the user keeps. The GC warns each user once, then takes the expired file out of their library; the cache cap never evicts a kept file, and an evicted one leaves every holder's library. The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume siftlode_files); docs/self-hosting.md says how to carry an old volume over. Migration 0102 places every existing job and carries each done file's expiry over from its asset. safe_abs_path takes a missing path and answers None, which drops one baselined type error.
This commit is contained in:
@@ -395,7 +395,7 @@ def test_a_clip_job_reaches_the_clip_entry(monkeypatch):
|
||||
def worker(db, tmp_path, monkeypatch):
|
||||
"""The worker path with its external programs replaced: ffmpeg writes the file it was given, the
|
||||
probe answers what the test says, the phone-safe step and the poster pass through."""
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path / "root"))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path / "root"))
|
||||
seen: dict = {}
|
||||
|
||||
def ffmpeg(cmd, *_a, **_k):
|
||||
|
||||
@@ -80,7 +80,7 @@ def _asset_and_job(db, user, *, sig, asset_status, job_status, error=None):
|
||||
|
||||
|
||||
def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
cookiefile = tmp_path / "cookies.txt"
|
||||
cookiefile.write_text("# Netscape HTTP Cookie File\n")
|
||||
@@ -230,7 +230,7 @@ def test_a_download_yt_dlp_gave_no_duration_stores_the_files_duration(db, monkey
|
||||
# row lost its trim action. Through the real finalize, with a real 3-second file.
|
||||
u = _user(db, "fb-duration@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
@@ -350,7 +350,7 @@ def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeyp
|
||||
# file name and the share preview all said "Video". The post's text is in the description.
|
||||
u = _user(db, "fb-title@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
@@ -401,7 +401,7 @@ def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path):
|
||||
# video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg).
|
||||
u = _user(db, "fb-mp3@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
@@ -430,7 +430,7 @@ def test_a_video_download_is_still_made_browser_playable(db, monkeypatch, tmp_pa
|
||||
a, j = _asset_and_job(
|
||||
db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running"
|
||||
)
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
|
||||
@@ -259,7 +259,7 @@ def test_a_refusal_for_one_user_does_not_poison_the_shared_asset(db, monkeypatch
|
||||
that shared row — correct for an upstream failure, wrong for "YOUR storage is full", which
|
||||
`worker._process_job` would then hand verbatim to someone with an empty library.
|
||||
"""
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
over, other = User(email="over@example.com"), User(email="other@example.com")
|
||||
db.add_all([over, other])
|
||||
db.flush()
|
||||
@@ -306,7 +306,7 @@ def test_a_download_over_the_space_left_says_so_and_releases_the_shared_asset(
|
||||
# yt-dlp refuses a file over `max_filesize` by saying so and returning with no file (pinned in
|
||||
# test_formats). That is the owner's storage, not a processing fault: their refusal, in words
|
||||
# true of a cap with space still left, and the shared asset goes back for the next job.
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
u = User(email="nearcap@example.com")
|
||||
db.add(u)
|
||||
db.flush()
|
||||
@@ -352,7 +352,7 @@ def test_a_download_that_outgrows_the_space_left_unannounced_is_refused_and_not_
|
||||
# yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's
|
||||
# fragmented formats never meet it (measured on localdev: 718 MB fetched with 1 MB left). The
|
||||
# file it produced is held to what is left by its real size, as a clip is.
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
u = User(email="unannounced@example.com")
|
||||
db.add(u)
|
||||
db.flush()
|
||||
|
||||
@@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg n
|
||||
def test_an_editor_clip_larger_than_what_is_left_is_refused_and_not_placed(
|
||||
db, make_user, tmp_path, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
src = tmp_path / "in.mp4"
|
||||
subprocess.run(
|
||||
[
|
||||
|
||||
@@ -258,7 +258,7 @@ def test_an_edit_that_crashes_is_said_once(sink, finalized, monkeypatch, tmp_pat
|
||||
"""The line `_process_edit`'s handler used to log before `_fail_edit` logged it again."""
|
||||
(tmp_path / "src.mp4").write_bytes(b"x")
|
||||
monkeypatch.setattr(pipeline, "SessionLocal", _EditDb)
|
||||
monkeypatch.setattr(pipeline.settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.settings, "files_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging")
|
||||
|
||||
def odd_spec(*a, **k):
|
||||
@@ -300,7 +300,7 @@ def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink
|
||||
RuntimeError("dictionary changed size during iteration"),
|
||||
RecursionError("maximum recursion depth exceeded"),
|
||||
NotImplementedError(),
|
||||
FileNotFoundError(2, "No such file", "/downloads/Some_Channel/Season_2024/A_Private_Title"),
|
||||
FileNotFoundError(2, "No such file", "/files/Some_Channel/Season_2024/A_Private_Title"),
|
||||
],
|
||||
ids=lambda e: type(e).__name__,
|
||||
)
|
||||
@@ -499,11 +499,9 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
|
||||
@pytest.mark.parametrize(
|
||||
"err",
|
||||
[
|
||||
subprocess.TimeoutExpired(
|
||||
["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30
|
||||
),
|
||||
FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"),
|
||||
subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]),
|
||||
subprocess.TimeoutExpired(["ffmpeg", "-i", "/files/Some_Channel/A_Private_Title.mp4"], 30),
|
||||
FileNotFoundError(2, "No such file", "/files/Some_Channel/A_Private_Title.jpg"),
|
||||
subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/files/Some_Channel/x.mp4"]),
|
||||
],
|
||||
ids=lambda e: type(e).__name__,
|
||||
)
|
||||
@@ -523,9 +521,7 @@ def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_
|
||||
src.write_bytes(b"\xff\xd8\xff rest")
|
||||
|
||||
def refuse(a, b):
|
||||
raise PermissionError(
|
||||
13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg"
|
||||
)
|
||||
raise PermissionError(13, "Permission denied", "/files/Some_Channel/A_Private_Title.jpg")
|
||||
|
||||
monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
|
||||
assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
"""The Files library's folders and a file's retention: which folder a new file lands in, when a done
|
||||
file goes, and the GC that takes it — warning once, never a file its user kept, never a file another
|
||||
user still holds.
|
||||
|
||||
Pure-logic for the source-key rule; DB-backed for the rest.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import folders, formats, service
|
||||
from app.downloads import gc as gcmod
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "key"),
|
||||
[
|
||||
("https://www.tiktok.com/@a/video/1", "site:tiktok.com"),
|
||||
("https://m.facebook.com/groups/x/posts/2", "site:facebook.com"),
|
||||
("https://vimeo.com/123", "site:vimeo.com"),
|
||||
("https://youtu.be/abcdefghijk", "youtube"),
|
||||
("https://music.youtube.com/watch?v=abcdefghijk", "youtube"),
|
||||
("https://www.youtube.com/watch?v=abcdefghijk", "youtube"),
|
||||
# A front door is only stripped when a real host remains behind it.
|
||||
("https://m.co/x", "site:m.co"),
|
||||
("not a url", "site:unknown"),
|
||||
],
|
||||
)
|
||||
def test_a_url_lands_in_its_sites_folder(url, key):
|
||||
assert folders.site_key(url) == key
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _asset(db, kind="youtube", ref="abcdefghijk", status="ready", rel="a.mp4", size=10):
|
||||
asset = MediaAsset(
|
||||
source_kind=kind,
|
||||
source_ref=ref,
|
||||
format_sig=hashlib.sha256(f"{kind}:{ref}".encode()).hexdigest()[:32],
|
||||
status=status,
|
||||
rel_path=rel,
|
||||
size_bytes=size,
|
||||
ref_count=0,
|
||||
title="A title",
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
return asset
|
||||
|
||||
|
||||
def _job(db, user_id, asset, *, kind="youtube", ref=None, status=JobStatus.DONE, **kw):
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id if asset else None,
|
||||
source_kind=kind,
|
||||
source_ref=ref or (asset.source_ref if asset else "x"),
|
||||
profile_snapshot={},
|
||||
status=status,
|
||||
**kw,
|
||||
)
|
||||
db.add(job)
|
||||
db.flush()
|
||||
folders.place(db, job)
|
||||
if asset is not None:
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _folder(db, job) -> FileFolder:
|
||||
return db.get(FileFolder, job.folder_id)
|
||||
|
||||
|
||||
def test_every_kind_of_job_lands_in_its_source_folder(db, make_user):
|
||||
uid = make_user().id
|
||||
yt = _job(db, uid, _asset(db))
|
||||
site = _job(db, uid, _asset(db, "url", "https://www.tiktok.com/@a/video/1"), kind="url")
|
||||
up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
|
||||
plex = _job(
|
||||
db, uid, _asset(db, "clip", "c1"), kind="clip", edit_spec={"source": "plex", "ref": "9"}
|
||||
)
|
||||
ytclip = _job(
|
||||
db, uid, _asset(db, "clip", "c2"), kind="clip", edit_spec={"source": "youtube", "ref": "v"}
|
||||
)
|
||||
# An edit of a Plex clip: only its parent knows it came from Plex (the clip's asset says "clip").
|
||||
edit = _job(
|
||||
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=plex.id,
|
||||
source_asset_id=plex.asset_id,
|
||||
) # fmt: skip
|
||||
keys = {j.id: _folder(db, j).source_key for j in (yt, site, up, plex, ytclip, edit)}
|
||||
assert keys == {
|
||||
yt.id: "youtube",
|
||||
site.id: "site:tiktok.com",
|
||||
up.id: "upload",
|
||||
plex.id: "plex",
|
||||
ytclip.id: "youtube",
|
||||
edit.id: "plex", # an edit sits with the file it was cut from
|
||||
}
|
||||
# One folder per origin per user: the two YouTube files share theirs.
|
||||
assert _folder(db, yt).id == _folder(db, ytclip).id
|
||||
|
||||
|
||||
def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
|
||||
uid = make_user().id
|
||||
src = _asset(db, "url", "https://vimeo.com/1")
|
||||
edit = DownloadJob(
|
||||
user_id=uid, source_kind="edit", source_ref="e", profile_snapshot={},
|
||||
source_job_id=None, source_asset_id=src.id, status=JobStatus.DONE,
|
||||
) # fmt: skip
|
||||
assert folders.source_key_for(db, edit) == "site:vimeo.com"
|
||||
|
||||
|
||||
def test_folders_are_per_user(db, make_user):
|
||||
a, b = make_user().id, make_user().id
|
||||
asset = _asset(db)
|
||||
ja, jb = _job(db, a, asset), _job(db, b, asset)
|
||||
assert ja.folder_id != jb.folder_id
|
||||
assert (_folder(db, ja).user_id, _folder(db, jb).user_id) == (a, b)
|
||||
|
||||
|
||||
def test_a_source_folder_keeps_files_the_retention_days_and_uploads_forever(db, make_user):
|
||||
sysconfig.set_value(db, "download_retention_days", 12)
|
||||
db.commit()
|
||||
uid = make_user().id
|
||||
yt = _job(db, uid, _asset(db))
|
||||
up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
|
||||
assert _folder(db, yt).retention_days == 12
|
||||
assert _folder(db, up).retention_days is None
|
||||
|
||||
|
||||
def test_enqueue_places_the_job_and_a_cache_hit_is_stamped_at_once(db, make_user):
|
||||
uid = make_user().id
|
||||
spec = formats.normalize({})
|
||||
first = service.enqueue(db, uid, "youtube", "abcdefghijk", spec)
|
||||
assert _folder(db, first).source_key == "youtube"
|
||||
assert first.expires_at is None # queued: its retention starts when it is done
|
||||
asset = db.get(MediaAsset, first.asset_id)
|
||||
asset.status, asset.rel_path = "ready", "a.mp4"
|
||||
db.commit()
|
||||
other = make_user().id
|
||||
hit = service.enqueue(db, other, "youtube", "abcdefghijk", spec)
|
||||
assert hit.status == JobStatus.DONE
|
||||
days = _folder(db, hit).retention_days
|
||||
assert hit.expires_at is not None
|
||||
assert abs(hit.expires_at - (NOW + timedelta(days=days))) < timedelta(minutes=5)
|
||||
|
||||
|
||||
def test_a_finishing_worker_stamps_the_expiry_unless_kept(db, make_user):
|
||||
uid = make_user().id
|
||||
asset = _asset(db)
|
||||
plain = _job(db, uid, asset, status=JobStatus.RUNNING)
|
||||
kept = _job(db, uid, asset, status=JobStatus.RUNNING, kept=True)
|
||||
for job in (plain, kept):
|
||||
assert service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100)
|
||||
db.commit()
|
||||
db.refresh(plain)
|
||||
db.refresh(kept)
|
||||
assert plain.expires_at is not None
|
||||
assert kept.expires_at is None
|
||||
|
||||
|
||||
def _gc(db):
|
||||
return gcmod.run_download_gc(db)
|
||||
|
||||
|
||||
def _notices(db, uid, kind):
|
||||
return db.execute(
|
||||
select(func.count())
|
||||
.select_from(Notification)
|
||||
.where(Notification.user_id == uid, Notification.type == kind)
|
||||
).scalar()
|
||||
|
||||
|
||||
def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, _root):
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
uid = make_user().id
|
||||
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
|
||||
_gc(db)
|
||||
_gc(db)
|
||||
assert _notices(db, uid, "download_expiring") == 1
|
||||
job.expires_at = NOW - timedelta(minutes=1)
|
||||
db.commit()
|
||||
assert _gc(db)["expired"] == 1
|
||||
assert db.get(DownloadJob, job.id) is None
|
||||
assert _notices(db, uid, "download_expired") == 1
|
||||
# Its last holder gone, the shared file and its row go too.
|
||||
assert not (_root / "a.mp4").exists()
|
||||
assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
|
||||
|
||||
|
||||
def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
uid = make_user().id
|
||||
job = _job(db, uid, _asset(db), kept=True, expires_at=NOW - timedelta(days=1))
|
||||
assert _gc(db)["expired"] == 0
|
||||
assert db.get(DownloadJob, job.id) is not None
|
||||
|
||||
|
||||
def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
a, b = make_user().id, make_user().id
|
||||
asset = _asset(db)
|
||||
gone = _job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
|
||||
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
|
||||
_gc(db)
|
||||
assert db.get(DownloadJob, gone.id) is None
|
||||
assert db.get(DownloadJob, stays.id) is not None
|
||||
assert (_root / "a.mp4").exists()
|
||||
|
||||
|
||||
def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
|
||||
db, make_user, _root
|
||||
):
|
||||
sysconfig.set_value(db, "download_total_max_bytes", 15)
|
||||
db.commit()
|
||||
a, b = make_user().id, make_user().id
|
||||
for name in ("old.mp4", "new.mp4"):
|
||||
(_root / name).write_bytes(b"x")
|
||||
old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
|
||||
new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
|
||||
old.last_access_at = NOW - timedelta(days=9)
|
||||
new.last_access_at = NOW - timedelta(days=1)
|
||||
kept = _job(db, a, old, kept=True) # the older file, but kept: the newer one goes instead
|
||||
ja, jb = _job(db, a, new), _job(db, b, new)
|
||||
assert _gc(db)["evicted"] == 1
|
||||
assert db.get(DownloadJob, kept.id) is not None
|
||||
assert db.get(DownloadJob, ja.id) is None and db.get(DownloadJob, jb.id) is None
|
||||
assert _notices(db, a, "download_evicted") == 1
|
||||
assert _notices(db, b, "download_evicted") == 1
|
||||
assert (_root / "old.mp4").exists() and not (_root / "new.mp4").exists()
|
||||
|
||||
|
||||
def test_deleting_a_user_takes_their_folders_and_files_with_them(db, make_user):
|
||||
# Both tables cascade from users; the files' folder FK refuses deleting a folder that still holds
|
||||
# files, which must not stop the account's own deletion.
|
||||
user = make_user()
|
||||
_job(db, user.id, _asset(db))
|
||||
db.delete(user)
|
||||
db.commit()
|
||||
assert db.execute(select(func.count()).select_from(FileFolder)).scalar() == 0
|
||||
assert db.execute(select(func.count()).select_from(DownloadJob)).scalar() == 0
|
||||
|
||||
|
||||
def test_the_longest_host_has_room_for_its_folder(db, make_user):
|
||||
label = "a" * 63
|
||||
host = ".".join([label, label, label, "b" * 58]) + ".hu" # 253 characters, the DNS maximum
|
||||
assert len(host) == 253
|
||||
uid = make_user().id
|
||||
job = _job(db, uid, _asset(db, "url", f"https://{host}/v"), kind="url")
|
||||
assert _folder(db, job).source_key == f"site:{host}"
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Migration 0102's data step, DB-lane: every existing job lands in its source folder, and a done
|
||||
file's expiry moves from the shared asset to the user's job.
|
||||
|
||||
The schema half is checked by test_migrations.py; a row placed in the wrong folder is invisible there.
|
||||
"""
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import engine
|
||||
|
||||
|
||||
def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user):
|
||||
alice, bob = make_user().id, make_user().id
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0101_plex_text_plain")
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
|
||||
def asset(kind, ref, expires=None):
|
||||
return conn.execute(
|
||||
text(
|
||||
"INSERT INTO media_assets (source_kind, source_ref, format_sig, status,"
|
||||
" ref_count, expires_at) VALUES (:k, :r, :sig, 'ready', 1, :e)"
|
||||
" RETURNING id"
|
||||
),
|
||||
{"k": kind, "r": ref, "sig": kind + ref, "e": expires},
|
||||
).scalar_one()
|
||||
|
||||
def job(user, kind, ref, asset_id, status="done", spec=None, parent=None):
|
||||
return conn.execute(
|
||||
text(
|
||||
"INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
|
||||
" profile_snapshot, status, edit_spec, source_job_id)"
|
||||
" VALUES (:u, :a, :k, :r, '{}', :s, CAST(:spec AS json), :p) RETURNING id"
|
||||
),
|
||||
{"u": user, "a": asset_id, "k": kind, "r": ref, "s": status,
|
||||
"spec": spec, "p": parent},
|
||||
).scalar_one() # fmt: skip
|
||||
|
||||
yt = asset("youtube", "abcdefghijk", "2030-01-02T00:00:00+00:00")
|
||||
tt = asset("url", "https://www.tiktok.com/@a/video/1", "2030-01-03T00:00:00+00:00")
|
||||
up = asset("upload", "u1")
|
||||
cl = asset("clip", "c1", "2030-01-04T00:00:00+00:00")
|
||||
j_yt = job(alice, "youtube", "abcdefghijk", yt)
|
||||
j_yt_bob = job(bob, "youtube", "abcdefghijk", yt, status="queued")
|
||||
j_tt = job(alice, "url", "https://www.tiktok.com/@a/video/1", tt)
|
||||
j_up = job(alice, "upload", "u1", up)
|
||||
j_cl = job(alice, "clip", "c1", cl, spec='{"source": "plex", "ref": "9"}')
|
||||
j_ed = job(alice, "edit", "e1", cl, parent=j_tt)
|
||||
finally:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"SELECT j.id, f.user_id, f.source_key, f.retention_days, j.expires_at::date"
|
||||
" FROM download_jobs j JOIN file_folders f ON f.id = j.folder_id"
|
||||
)
|
||||
).all()
|
||||
folders = conn.execute(text("SELECT count(*) FROM file_folders")).scalar()
|
||||
got = {r[0]: (r[1], r[2], r[3], str(r[4]) if r[4] else None) for r in rows}
|
||||
assert got == {
|
||||
j_yt: (alice, "youtube", 30, "2030-01-02"),
|
||||
j_yt_bob: (bob, "youtube", 30, None), # not done yet: no expiry until it is
|
||||
j_tt: (alice, "site:tiktok.com", 30, "2030-01-03"),
|
||||
j_up: (alice, "upload", None, None), # an upload never expires
|
||||
j_cl: (alice, "plex", 30, "2030-01-04"),
|
||||
j_ed: (alice, "site:tiktok.com", 30, "2030-01-04"),
|
||||
}
|
||||
# alice: youtube, tiktok, upload, plex; bob: youtube.
|
||||
assert folders == 5
|
||||
@@ -46,7 +46,7 @@ def test_an_odd_sized_4_4_4_video_converts_to_the_even_frame_below_it(tmp_path):
|
||||
|
||||
|
||||
def test_an_odd_crop_makes_a_clip_and_stores_the_frame_it_has(db, make_user, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
src = tmp_path / "in.mp4"
|
||||
_ffmpeg(
|
||||
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2",
|
||||
|
||||
@@ -127,7 +127,7 @@ _FILES = {
|
||||
def media_root(tmp_path, monkeypatch):
|
||||
root = tmp_path / "media"
|
||||
root.mkdir()
|
||||
monkeypatch.setattr(settings, "download_root", str(root))
|
||||
monkeypatch.setattr(settings, "files_root", str(root))
|
||||
return root
|
||||
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ _PW = "open sesame"
|
||||
|
||||
@pytest.fixture
|
||||
def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"}
|
||||
audio = {"codec_type": "audio", "codec_name": "aac"}
|
||||
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
|
||||
|
||||
@@ -27,7 +27,7 @@ def pages(tmp_path, monkeypatch):
|
||||
|
||||
@pytest.fixture
|
||||
def link_token(as_user, make_user, tmp_path, monkeypatch, upload_headers):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path / "media"))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path / "media"))
|
||||
video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"}
|
||||
audio = {"codec_type": "audio", "codec_name": "aac"}
|
||||
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
|
||||
|
||||
@@ -100,7 +100,7 @@ class TestDownloadFilename:
|
||||
|
||||
|
||||
class TestSafeAbsPath:
|
||||
"""The path-traversal guard: only paths that resolve INSIDE download_root AND exist are returned."""
|
||||
"""The path-traversal guard: only paths that resolve INSIDE files_root AND exist are returned."""
|
||||
|
||||
def test_returns_the_path_for_a_file_inside_the_root(self, tmp_path: Path):
|
||||
(tmp_path / "sub").mkdir()
|
||||
|
||||
@@ -5,7 +5,7 @@ generic file (force-download so uploaded HTML/SVG can't execute inline), quota a
|
||||
size known up front, the per-file cap enforced while streaming, filename sanitization, and the two
|
||||
guards that keep an upload out of paths that assume a re-fetchable source (edit / redownload).
|
||||
|
||||
DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is
|
||||
DB-backed (real Postgres) — skipped when no database is reachable. `settings.files_root` is
|
||||
pointed at a per-test tmp dir so ingest's on-disk moves are isolated. The ingest/route tests
|
||||
monkeypatch the shared `probe.probe_json` so they don't shell out to ffprobe; the classifier's OWN
|
||||
parsing (image → file, cover-art skip, dims, duration) is covered directly by `TestProbeClassifier`
|
||||
@@ -51,7 +51,7 @@ def test_derive_media_kind():
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
@@ -761,7 +761,7 @@ def test_a_refused_upload_leaves_nothing_in_the_staging_directory(
|
||||
|
||||
sysconfig.set_value(db, "download_upload_max_bytes", 8)
|
||||
db.commit()
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging = Path(settings.files_root) / ".staging" / "uploads"
|
||||
c = as_user(make_user())
|
||||
|
||||
r = c.post("/api/downloads/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin"))
|
||||
|
||||
@@ -29,7 +29,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension(
|
||||
):
|
||||
# A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an
|
||||
# audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file.
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
data = {
|
||||
"streams": [{"codec_type": "audio", "codec_name": "aac"}],
|
||||
"format": {"format_name": "mov,mp4", "duration": "10"},
|
||||
|
||||
Reference in New Issue
Block a user