peter
54a6bbf07f
- GC: the expiry warning marks each file conditionally on the expiry it saw, one commit each (a file deleted meanwhile no longer fails the run; one re-timed keeps its fresh expiry unwarned); eviction counts the disk again when a candidate was freed meanwhile, so it never evicts for space already free. - Folders: a retention change locks the folder and its files, and a finishing file reads the retention under a share lock, so neither misses the other; a folder delete locks the files it takes home (a file moved out meanwhile stays put, one deleted meanwhile no longer fails the delete). - Upload: the folder is checked before the probe and held only once the job names it. - Dedup enqueue locks the cached asset, so a release meanwhile yields a fresh one instead of a 500. - Mount check reads the kernel's mount table instead of comparing devices (the vfs storage driver gave a mounted volume the device of /), and only inside a container; the API also logs a fresh, empty volume in the library's place. - Compose files fall back to DOWNLOAD_HOST_PATH; the upgrade notes cover the from-source pull. - Folder routes call owned_or_404 directly.
647 lines
24 KiB
Python
647 lines
24 KiB
Python
"""The Files library's folders and a file's retention: which folder a new file lands in, when a done
|
|
file goes, and the GC that takes it — warning once, never a file its user kept, never a file another
|
|
user still holds.
|
|
|
|
Pure-logic for the source-key rule; DB-backed for the rest.
|
|
"""
|
|
|
|
import hashlib
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
|
|
from app import sysconfig
|
|
from app.config import settings
|
|
from app.downloads import folders, formats, service
|
|
from app.downloads import gc as gcmod
|
|
from app.db import SessionLocal
|
|
from app.downloads.status import JobStatus
|
|
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("url", "key"),
|
|
[
|
|
("https://www.tiktok.com/@a/video/1", "site:tiktok.com"),
|
|
("https://m.facebook.com/groups/x/posts/2", "site:facebook.com"),
|
|
("https://vimeo.com/123", "site:vimeo.com"),
|
|
("https://youtu.be/abcdefghijk", "youtube"),
|
|
("https://music.youtube.com/watch?v=abcdefghijk", "youtube"),
|
|
("https://www.youtube.com/watch?v=abcdefghijk", "youtube"),
|
|
# A front door is only stripped when a real host remains behind it.
|
|
("https://m.co/x", "site:m.co"),
|
|
("not a url", "site:unknown"),
|
|
],
|
|
)
|
|
def test_a_url_lands_in_its_sites_folder(url, key):
|
|
assert folders.site_key(url) == key
|
|
|
|
|
|
@pytest.fixture
|
|
def _root(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
|
return tmp_path
|
|
|
|
|
|
def _asset(db, kind="youtube", ref="abcdefghijk", status="ready", rel="a.mp4", size=10):
|
|
asset = MediaAsset(
|
|
source_kind=kind,
|
|
source_ref=ref,
|
|
format_sig=hashlib.sha256(f"{kind}:{ref}".encode()).hexdigest()[:32],
|
|
status=status,
|
|
rel_path=rel,
|
|
size_bytes=size,
|
|
ref_count=0,
|
|
title="A title",
|
|
)
|
|
db.add(asset)
|
|
db.flush()
|
|
return asset
|
|
|
|
|
|
def _job(db, user_id, asset, *, kind="youtube", ref=None, status=JobStatus.DONE, **kw):
|
|
job = DownloadJob(
|
|
user_id=user_id,
|
|
asset_id=asset.id if asset else None,
|
|
source_kind=kind,
|
|
source_ref=ref or (asset.source_ref if asset else "x"),
|
|
profile_snapshot={},
|
|
status=status,
|
|
**kw,
|
|
)
|
|
db.add(job)
|
|
db.flush()
|
|
folders.place(db, job)
|
|
if asset is not None:
|
|
service.incr_ref_count(db, asset.id)
|
|
db.commit()
|
|
return job
|
|
|
|
|
|
def _folder(db, job) -> FileFolder:
|
|
return db.get(FileFolder, job.folder_id)
|
|
|
|
|
|
def test_every_kind_of_job_lands_in_its_source_folder(db, make_user):
|
|
uid = make_user().id
|
|
yt = _job(db, uid, _asset(db))
|
|
site = _job(db, uid, _asset(db, "url", "https://www.tiktok.com/@a/video/1"), kind="url")
|
|
up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
|
|
plex = _job(
|
|
db, uid, _asset(db, "clip", "c1"), kind="clip", edit_spec={"source": "plex", "ref": "9"}
|
|
)
|
|
ytclip = _job(
|
|
db, uid, _asset(db, "clip", "c2"), kind="clip", edit_spec={"source": "youtube", "ref": "v"}
|
|
)
|
|
# An edit of a Plex clip: only its parent knows it came from Plex (the clip's asset says "clip").
|
|
edit = _job(
|
|
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=plex.id,
|
|
source_asset_id=plex.asset_id,
|
|
) # fmt: skip
|
|
keys = {j.id: _folder(db, j).source_key for j in (yt, site, up, plex, ytclip, edit)}
|
|
assert keys == {
|
|
yt.id: "youtube",
|
|
site.id: "site:tiktok.com",
|
|
up.id: "upload",
|
|
plex.id: "plex",
|
|
ytclip.id: "youtube",
|
|
edit.id: "plex", # an edit sits with the file it was cut from
|
|
}
|
|
# One folder per origin per user: the two YouTube files share theirs.
|
|
assert _folder(db, yt).id == _folder(db, ytclip).id
|
|
|
|
|
|
def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
|
|
uid = make_user().id
|
|
src = _asset(db, "url", "https://vimeo.com/1")
|
|
edit = DownloadJob(
|
|
user_id=uid, source_kind="edit", source_ref="e", profile_snapshot={},
|
|
source_job_id=None, source_asset_id=src.id, status=JobStatus.DONE,
|
|
) # fmt: skip
|
|
assert folders.source_key_for(db, edit) == "site:vimeo.com"
|
|
|
|
|
|
def test_an_edit_keeps_its_origin_after_its_parent_and_its_file_are_gone(db, make_user):
|
|
# The parent download expires, its file goes with its last holder, and both of the edit's
|
|
# pointers to it are cleared: the edit must still go back to the YouTube folder it came from.
|
|
uid = make_user().id
|
|
parent = _job(db, uid, _asset(db))
|
|
edit = _job(
|
|
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=parent.id,
|
|
source_asset_id=parent.asset_id,
|
|
) # fmt: skip
|
|
home = _folder(db, edit)
|
|
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
|
|
db.add(own)
|
|
db.flush()
|
|
folders.move(db, edit, own)
|
|
edit.source_job_id = edit.source_asset_id = None # what ON DELETE SET NULL leaves
|
|
db.delete(parent)
|
|
db.commit()
|
|
folders.move(db, edit, home)
|
|
assert edit.folder_id == home.id
|
|
folders.move(db, edit, own)
|
|
folders.delete_own(db, own)
|
|
assert edit.folder_id == home.id
|
|
|
|
|
|
def test_a_folder_deleted_while_a_file_goes_in_takes_that_file_home_too(db, make_user):
|
|
# Before, the delete missed the file still on its way in and failed on the foreign key (a 500),
|
|
# or the file's insert did after the folder went.
|
|
import threading
|
|
|
|
uid = make_user().id
|
|
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
|
|
db.add(own)
|
|
db.commit()
|
|
own_id = own.id
|
|
job = _job(db, uid, _asset(db))
|
|
home = job.folder_id
|
|
target = folders.own_folder(db, uid, own_id) # held from the check until this commits
|
|
outcome: list[object] = []
|
|
|
|
def delete_it():
|
|
other = SessionLocal()
|
|
try:
|
|
outcome.append(folders.delete_own(other, other.get(FileFolder, own_id)))
|
|
other.commit()
|
|
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
|
|
outcome.append(e)
|
|
finally:
|
|
other.close()
|
|
|
|
t = threading.Thread(target=delete_it)
|
|
t.start()
|
|
t.join(timeout=1)
|
|
assert t.is_alive() # the delete waits for the file, even before it is written
|
|
folders.move(db, job, target)
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
assert outcome == [1]
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, job.id).folder_id == home
|
|
assert db.get(FileFolder, own_id) is None
|
|
|
|
|
|
def test_folders_are_per_user(db, make_user):
|
|
a, b = make_user().id, make_user().id
|
|
asset = _asset(db)
|
|
ja, jb = _job(db, a, asset), _job(db, b, asset)
|
|
assert ja.folder_id != jb.folder_id
|
|
assert (_folder(db, ja).user_id, _folder(db, jb).user_id) == (a, b)
|
|
|
|
|
|
def test_a_source_folder_keeps_files_the_retention_days_and_uploads_forever(db, make_user):
|
|
sysconfig.set_value(db, "download_retention_days", 12)
|
|
db.commit()
|
|
uid = make_user().id
|
|
yt = _job(db, uid, _asset(db))
|
|
up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
|
|
assert _folder(db, yt).retention_days == 12
|
|
assert _folder(db, up).retention_days is None
|
|
|
|
|
|
def test_enqueue_places_the_job_and_a_cache_hit_is_stamped_at_once(db, make_user):
|
|
uid = make_user().id
|
|
spec = formats.normalize({})
|
|
first = service.enqueue(db, uid, "youtube", "abcdefghijk", spec)
|
|
assert _folder(db, first).source_key == "youtube"
|
|
assert first.expires_at is None # queued: its retention starts when it is done
|
|
asset = db.get(MediaAsset, first.asset_id)
|
|
asset.status, asset.rel_path = "ready", "a.mp4"
|
|
db.commit()
|
|
other = make_user().id
|
|
hit = service.enqueue(db, other, "youtube", "abcdefghijk", spec)
|
|
assert hit.status == JobStatus.DONE
|
|
days = _folder(db, hit).retention_days
|
|
assert hit.expires_at is not None
|
|
assert abs(hit.expires_at - (NOW + timedelta(days=days))) < timedelta(minutes=5)
|
|
|
|
|
|
def test_a_finishing_worker_stamps_the_expiry_unless_kept(db, make_user):
|
|
uid = make_user().id
|
|
asset = _asset(db)
|
|
plain = _job(db, uid, asset, status=JobStatus.RUNNING)
|
|
kept = _job(db, uid, asset, status=JobStatus.RUNNING, kept=True)
|
|
for job in (plain, kept):
|
|
assert service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100)
|
|
db.commit()
|
|
db.refresh(plain)
|
|
db.refresh(kept)
|
|
assert plain.expires_at is not None
|
|
assert kept.expires_at is None
|
|
|
|
|
|
def _gc(db):
|
|
return gcmod.run_download_gc(db)
|
|
|
|
|
|
def _notices(db, uid, kind):
|
|
return db.execute(
|
|
select(func.count())
|
|
.select_from(Notification)
|
|
.where(Notification.user_id == uid, Notification.type == kind)
|
|
).scalar()
|
|
|
|
|
|
def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, _root):
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
uid = make_user().id
|
|
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
|
|
_gc(db)
|
|
_gc(db)
|
|
assert _notices(db, uid, "download_expiring") == 1
|
|
job.expires_at = NOW - timedelta(minutes=1)
|
|
db.commit()
|
|
assert _gc(db)["expired"] == 1
|
|
assert db.get(DownloadJob, job.id) is None
|
|
assert _notices(db, uid, "download_expired") == 1
|
|
# Its last holder gone, the shared file and its row go too.
|
|
assert not (_root / "a.mp4").exists()
|
|
assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
|
|
|
|
|
|
@pytest.mark.parametrize("restart", ["unkeep", "move", "retention"])
|
|
def test_a_file_whose_clock_starts_again_is_warned_again(db, make_user, _root, restart):
|
|
# The GC warns once per expiry; a file whose expiry was moved must be warned before the new one.
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
uid = make_user().id
|
|
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
|
|
_gc(db)
|
|
assert job.expiry_warned
|
|
if restart == "unkeep":
|
|
folders.set_kept(db, job, True)
|
|
folders.set_kept(db, job, False)
|
|
elif restart == "move":
|
|
home = _folder(db, job)
|
|
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
|
|
db.add(own)
|
|
db.flush()
|
|
folders.move(db, job, own)
|
|
folders.move(db, job, home)
|
|
else:
|
|
folders.set_retention(db, _folder(db, job), 90)
|
|
db.commit()
|
|
assert not job.expiry_warned
|
|
|
|
|
|
def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
uid = make_user().id
|
|
job = _job(db, uid, _asset(db), kept=True, expires_at=NOW - timedelta(days=1))
|
|
assert _gc(db)["expired"] == 0
|
|
assert db.get(DownloadJob, job.id) is not None
|
|
|
|
|
|
def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
a, b = make_user().id, make_user().id
|
|
asset = _asset(db)
|
|
gone = _job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
|
|
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
|
|
_gc(db)
|
|
assert db.get(DownloadJob, gone.id) is None
|
|
assert db.get(DownloadJob, stays.id) is not None
|
|
assert (_root / "a.mp4").exists()
|
|
|
|
|
|
def _meanwhile(monkeypatch, act, step="_relock_expired"):
|
|
"""Run `act` in another session (another request) between a GC pass's SELECT and its handling of
|
|
each row it found — the window a user's click can land in. `step` is the pass's per-row re-read
|
|
(expiry, warning, eviction); `act` gets that row's id."""
|
|
real = getattr(gcmod, step)
|
|
|
|
def wrapped(db, key, *rest):
|
|
other = SessionLocal()
|
|
try:
|
|
act(other, key)
|
|
other.commit()
|
|
finally:
|
|
other.close()
|
|
return real(db, key, *rest)
|
|
|
|
monkeypatch.setattr(gcmod, step, wrapped)
|
|
|
|
|
|
def test_a_file_kept_while_the_gc_runs_stays(db, make_user, _root, monkeypatch):
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
job = _job(db, make_user().id, _asset(db), expires_at=NOW - timedelta(minutes=1))
|
|
_meanwhile(monkeypatch, lambda other, jid: setattr(other.get(DownloadJob, jid), "kept", True))
|
|
assert _gc(db)["expired"] == 0
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, job.id) is not None
|
|
assert (_root / "a.mp4").exists()
|
|
|
|
|
|
def test_a_file_deleted_while_the_gc_runs_is_released_once(db, make_user, _root, monkeypatch):
|
|
# Released twice, the shared file would go from under the other user's copy.
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
a, b = make_user().id, make_user().id
|
|
asset = _asset(db)
|
|
_job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
|
|
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
|
|
|
|
def user_deletes(other, jid):
|
|
service.delete(other, other.get(DownloadJob, jid))
|
|
|
|
_meanwhile(monkeypatch, user_deletes)
|
|
_gc(db)
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, stays.id).asset_id == asset.id
|
|
assert db.get(MediaAsset, asset.id).ref_count == 1
|
|
assert (_root / "a.mp4").exists()
|
|
|
|
|
|
def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
|
|
db, make_user, _root
|
|
):
|
|
sysconfig.set_value(db, "download_total_max_bytes", 15)
|
|
db.commit()
|
|
a, b = make_user().id, make_user().id
|
|
for name in ("old.mp4", "new.mp4"):
|
|
(_root / name).write_bytes(b"x")
|
|
old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
|
|
new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
|
|
old.last_access_at = NOW - timedelta(days=9)
|
|
new.last_access_at = NOW - timedelta(days=1)
|
|
kept = _job(db, a, old, kept=True) # the older file, but kept: the newer one goes instead
|
|
ja, jb = _job(db, a, new), _job(db, b, new)
|
|
assert _gc(db)["evicted"] == 1
|
|
assert db.get(DownloadJob, kept.id) is not None
|
|
assert db.get(DownloadJob, ja.id) is None and db.get(DownloadJob, jb.id) is None
|
|
assert _notices(db, a, "download_evicted") == 1
|
|
assert _notices(db, b, "download_evicted") == 1
|
|
assert (_root / "old.mp4").exists() and not (_root / "new.mp4").exists()
|
|
|
|
|
|
def test_a_file_deleted_while_the_gc_warns_does_not_stop_the_run(db, make_user, _root, monkeypatch):
|
|
# Written back from a stale read, the warning of a deleted file failed the whole run: nothing
|
|
# expired or was evicted that cycle, nor the next while the race lasted.
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
(_root / "b.mp4").write_bytes(b"x")
|
|
uid = make_user().id
|
|
_job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=1))
|
|
gone = _job(
|
|
db, uid, _asset(db, ref="bbbbbbbbbbb", rel="b.mp4"), expires_at=NOW - timedelta(minutes=1)
|
|
)
|
|
|
|
def user_deletes(other, jid):
|
|
service.delete(other, other.get(DownloadJob, jid))
|
|
|
|
_meanwhile(monkeypatch, user_deletes, step="_mark_warned")
|
|
result = _gc(db)
|
|
assert (result["warned"], result["expired"]) == (0, 1)
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, gone.id) is None
|
|
assert _notices(db, uid, "download_expiring") == 0
|
|
|
|
|
|
def test_a_file_re_timed_while_the_gc_warns_keeps_its_new_expiry_unwarned(
|
|
db, make_user, _root, monkeypatch
|
|
):
|
|
# Moved, un-kept or its folder's retention changed: a fresh expiry, warned when it comes — not
|
|
# marked warned now for a date it no longer has.
|
|
uid = make_user().id
|
|
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=1))
|
|
later = NOW + timedelta(days=30)
|
|
|
|
def re_timed(other, jid):
|
|
moved = other.get(DownloadJob, jid)
|
|
moved.expires_at = later
|
|
moved.expiry_warned = False
|
|
|
|
_meanwhile(monkeypatch, re_timed, step="_mark_warned")
|
|
assert _gc(db)["warned"] == 0
|
|
db.expire_all()
|
|
fresh = db.get(DownloadJob, job.id)
|
|
assert (fresh.expires_at, fresh.expiry_warned) == (later, False)
|
|
assert _notices(db, uid, "download_expiring") == 0
|
|
|
|
|
|
def _two_over_the_cap(db, make_user, _root):
|
|
sysconfig.set_value(db, "download_total_max_bytes", 15)
|
|
db.commit()
|
|
a, b = make_user().id, make_user().id
|
|
for name in ("old.mp4", "new.mp4"):
|
|
(_root / name).write_bytes(b"x")
|
|
old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
|
|
new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
|
|
old.last_access_at = NOW - timedelta(days=9)
|
|
new.last_access_at = NOW - timedelta(days=1)
|
|
return _job(db, a, old), _job(db, b, new)
|
|
|
|
|
|
def test_a_file_deleted_while_the_cache_is_over_its_cap_frees_that_space_and_no_more(
|
|
db, make_user, _root, monkeypatch
|
|
):
|
|
# Its bytes still counted, the pass went on to evict another user's file for space already free.
|
|
jold, jnew = _two_over_the_cap(db, make_user, _root)
|
|
|
|
def holder_deletes(other, asset_id):
|
|
if asset_id == jold.asset_id:
|
|
service.delete(other, other.get(DownloadJob, jold.id))
|
|
|
|
_meanwhile(monkeypatch, holder_deletes, step="_relock_evictable")
|
|
assert _gc(db)["evicted"] == 0
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, jnew.id) is not None
|
|
assert (_root / "new.mp4").exists()
|
|
|
|
|
|
def test_a_file_kept_while_the_cache_is_over_its_cap_stays(db, make_user, _root, monkeypatch):
|
|
jold, jnew = _two_over_the_cap(db, make_user, _root)
|
|
|
|
def holder_keeps(other, asset_id):
|
|
if asset_id == jold.asset_id:
|
|
other.get(DownloadJob, jold.id).kept = True
|
|
|
|
_meanwhile(monkeypatch, holder_keeps, step="_relock_evictable")
|
|
assert _gc(db)["evicted"] == 1 # the newer one goes in its place
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, jold.id) is not None
|
|
assert (_root / "old.mp4").exists()
|
|
assert db.get(DownloadJob, jnew.id) is None
|
|
|
|
|
|
def _in_thread(work):
|
|
"""Run `work(other_session)` in another thread and session; returns (thread, outcome list)."""
|
|
import threading
|
|
|
|
outcome: list[object] = []
|
|
|
|
def run():
|
|
other = SessionLocal()
|
|
try:
|
|
outcome.append(work(other))
|
|
other.commit()
|
|
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
|
|
outcome.append(e)
|
|
finally:
|
|
other.close()
|
|
|
|
t = threading.Thread(target=run)
|
|
t.start()
|
|
return t, outcome
|
|
|
|
|
|
def test_a_retention_changed_while_a_file_goes_in_times_that_file_too(db, make_user):
|
|
# The change used to list the folder's files before the arriving one was in, so that one kept the
|
|
# old retention — deleted later from a folder now set to keep files forever.
|
|
uid = make_user().id
|
|
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7)
|
|
db.add(own)
|
|
db.commit()
|
|
own_id = own.id
|
|
job = _job(db, uid, _asset(db))
|
|
target = folders.own_folder(db, uid, own_id) # held from the check until this commits
|
|
t, outcome = _in_thread(
|
|
lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
|
|
)
|
|
t.join(timeout=1)
|
|
folders.move(db, job, target) # stamped with the 7 days it sees
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
assert outcome == [None]
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, job.id).expires_at is None
|
|
|
|
|
|
def test_a_retention_changed_while_a_file_finishes_there_times_that_file_too(db, make_user):
|
|
# The worker's finish read the folder's old retention unlocked, while the change listed the done
|
|
# files before this one was among them.
|
|
uid = make_user().id
|
|
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7)
|
|
db.add(own)
|
|
db.commit()
|
|
own_id = own.id
|
|
job = _job(db, uid, _asset(db), status=JobStatus.RUNNING)
|
|
folders.move(db, job, own)
|
|
db.commit()
|
|
job.status = JobStatus.DONE
|
|
folders.stamp_done(db, job) # the finish: stamped with the 7 days it reads
|
|
db.flush()
|
|
t, outcome = _in_thread(
|
|
lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
|
|
)
|
|
t.join(timeout=1)
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
assert outcome == [None]
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, job.id).expires_at is None
|
|
|
|
|
|
def test_a_file_moved_out_while_its_folder_is_deleted_stays_where_it_went(db, make_user):
|
|
uid = make_user().id
|
|
f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F")
|
|
g = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="G")
|
|
db.add_all([f, g])
|
|
db.commit()
|
|
f_id, g_id = f.id, g.id
|
|
job = _job(db, uid, _asset(db))
|
|
folders.move(db, job, f)
|
|
db.commit()
|
|
folders.move(db, job, folders.own_folder(db, uid, g_id))
|
|
db.flush() # the move is written, not yet committed
|
|
t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id)))
|
|
t.join(timeout=1)
|
|
assert t.is_alive() # the delete waits for the file it would take home
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
assert outcome == [0]
|
|
db.expire_all()
|
|
assert db.get(DownloadJob, job.id).folder_id == g_id
|
|
|
|
|
|
def test_a_file_deleted_while_its_folder_is_deleted_does_not_fail_the_delete(db, make_user, _root):
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
uid = make_user().id
|
|
f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F")
|
|
db.add(f)
|
|
db.commit()
|
|
f_id = f.id
|
|
job = _job(db, uid, _asset(db))
|
|
folders.move(db, job, f)
|
|
db.commit()
|
|
service.delete(db, db.get(DownloadJob, job.id))
|
|
db.flush() # deleted, not yet committed
|
|
t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id)))
|
|
t.join(timeout=1)
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
assert outcome == [0]
|
|
db.expire_all()
|
|
assert db.get(FileFolder, f_id) is None
|
|
|
|
|
|
def test_a_file_asked_for_while_its_cached_copy_is_freed_gets_a_fresh_one(db, make_user):
|
|
# The enqueue read the cached copy unlocked: freed meanwhile, its job pointed at a row that was gone
|
|
# (a foreign-key failure, a 500).
|
|
asset = _asset(db)
|
|
asset_id, sig = asset.id, asset.format_sig
|
|
db.commit()
|
|
db.execute(select(MediaAsset).where(MediaAsset.id == asset_id).with_for_update())
|
|
db.delete(db.get(MediaAsset, asset_id))
|
|
db.flush() # the release, not yet committed
|
|
t, outcome = _in_thread(
|
|
lambda other: service.get_or_create_asset(other, "youtube", "abcdefghijk", sig).id
|
|
)
|
|
t.join(timeout=1)
|
|
assert t.is_alive() # it waits for the release
|
|
db.commit()
|
|
t.join(timeout=10)
|
|
[fresh] = outcome
|
|
assert isinstance(fresh, int) and fresh != asset_id
|
|
|
|
|
|
def test_an_empty_volume_in_the_librarys_place_is_noticed(db, make_user, _root):
|
|
# A fresh volume is a real, writable mount: only the files the library names tell it is the wrong one.
|
|
assert service.library_left_behind(db, str(_root)) is False # nothing named yet
|
|
_asset(db, rel="a.mp4")
|
|
db.commit()
|
|
assert service.library_left_behind(db, str(_root)) is True
|
|
(_root / "a.mp4").write_bytes(b"x")
|
|
assert service.library_left_behind(db, str(_root)) is False
|
|
|
|
|
|
def test_deleting_a_user_takes_their_folders_and_files_with_them(db, make_user):
|
|
# Both tables cascade from users; the files' folder FK refuses deleting a folder that still holds
|
|
# files, which must not stop the account's own deletion.
|
|
user = make_user()
|
|
_job(db, user.id, _asset(db))
|
|
db.delete(user)
|
|
db.commit()
|
|
assert db.execute(select(func.count()).select_from(FileFolder)).scalar() == 0
|
|
assert db.execute(select(func.count()).select_from(DownloadJob)).scalar() == 0
|
|
|
|
|
|
def test_the_longest_host_has_room_for_its_folder(db, make_user):
|
|
label = "a" * 63
|
|
host = ".".join([label, label, label, "b" * 58]) + ".hu" # 253 characters, the DNS maximum
|
|
assert len(host) == 253
|
|
uid = make_user().id
|
|
job = _job(db, uid, _asset(db, "url", f"https://{host}/v"), kind="url")
|
|
assert _folder(db, job).source_key == f"site:{host}"
|
|
|
|
|
|
def test_the_ready_notice_names_the_file_and_its_folder(db, make_user):
|
|
# The notice's "Open" goes to the file in its folder, so both ride along as data.
|
|
from app.downloads import pipeline
|
|
|
|
user = make_user()
|
|
job = _job(db, user.id, _asset(db))
|
|
pipeline._notify_download_done(db, user.id, "A title", job.id, kind="download")
|
|
db.commit()
|
|
note = db.scalars(select(Notification).where(Notification.user_id == user.id)).one()
|
|
assert note.type == "download_done"
|
|
assert note.data == {
|
|
"title": "A title",
|
|
"kind": "download",
|
|
"job_id": job.id,
|
|
"folder_id": job.folder_id,
|
|
}
|
|
assert job.folder_id is not None
|