peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
c8247c0dde4d62d021b87af31f077e02c0f6d92b
siftlode/backend/tests/test_file_folders.py
T
peter c9baf68b16 fix(files): review round 3, the server — a fetched locked re-read, a finish that follows its file, an eviction that counts again 
- set_retention and delete_own re-read the folder under its lock and use that row (an unfetched ORM
  result never refreshes the object); a folder gone meanwhile is refused (409), and the route audits
  the retention actually replaced
- a finishing download is stamped only while it is still in the folder (and keep) it was read with:
  a move or a folder delete landing meanwhile sends the read round again, so no file is done with no
  expiry
- the eviction pass counts the disk before each candidate and skips one asked for since it began
- docs: source builds get their own volume-copy snippet
- race tests assert the lock wait; one built on a committing delete never raced
2026-10-13 03:15:59 +02:00

808 lines
30 KiB
Python
Raw Blame History

"""The Files library's folders and a file's retention: which folder a new file lands in, when a done
file goes, and the GC that takes it — warning once, never a file its user kept, never a file another
user still holds.
Pure-logic for the source-key rule; DB-backed for the rest.
"""
import hashlib
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import func, select
from app import sysconfig
from app.config import settings
from app.downloads import folders, formats, service
from app.downloads import gc as gcmod
from app.db import SessionLocal
from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
NOW = datetime.now(timezone.utc)
@pytest.mark.parametrize(
("url", "key"),
[
("https://www.tiktok.com/@a/video/1", "site:tiktok.com"),
("https://m.facebook.com/groups/x/posts/2", "site:facebook.com"),
("https://vimeo.com/123", "site:vimeo.com"),
("https://youtu.be/abcdefghijk", "youtube"),
("https://music.youtube.com/watch?v=abcdefghijk", "youtube"),
("https://www.youtube.com/watch?v=abcdefghijk", "youtube"),
# A front door is only stripped when a real host remains behind it.
("https://m.co/x", "site:m.co"),
("not a url", "site:unknown"),
],
)
def test_a_url_lands_in_its_sites_folder(url, key):
assert folders.site_key(url) == key
@pytest.fixture
def _root(tmp_path, monkeypatch):
monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
def _asset(db, kind="youtube", ref="abcdefghijk", status="ready", rel="a.mp4", size=10):
asset = MediaAsset(
source_kind=kind,
source_ref=ref,
format_sig=hashlib.sha256(f"{kind}:{ref}".encode()).hexdigest()[:32],
status=status,
rel_path=rel,
size_bytes=size,
ref_count=0,
title="A title",
)
db.add(asset)
db.flush()
return asset
def _job(db, user_id, asset, *, kind="youtube", ref=None, status=JobStatus.DONE, **kw):
job = DownloadJob(
user_id=user_id,
asset_id=asset.id if asset else None,
source_kind=kind,
source_ref=ref or (asset.source_ref if asset else "x"),
profile_snapshot={},
status=status,
**kw,
)
db.add(job)
db.flush()
folders.place(db, job)
if asset is not None:
service.incr_ref_count(db, asset.id)
db.commit()
return job
def _folder(db, job) -> FileFolder:
return db.get(FileFolder, job.folder_id)
def test_every_kind_of_job_lands_in_its_source_folder(db, make_user):
uid = make_user().id
yt = _job(db, uid, _asset(db))
site = _job(db, uid, _asset(db, "url", "https://www.tiktok.com/@a/video/1"), kind="url")
up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
plex = _job(
db, uid, _asset(db, "clip", "c1"), kind="clip", edit_spec={"source": "plex", "ref": "9"}
)
ytclip = _job(
db, uid, _asset(db, "clip", "c2"), kind="clip", edit_spec={"source": "youtube", "ref": "v"}
)
# An edit of a Plex clip: only its parent knows it came from Plex (the clip's asset says "clip").
edit = _job(
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=plex.id,
source_asset_id=plex.asset_id,
) # fmt: skip
keys = {j.id: _folder(db, j).source_key for j in (yt, site, up, plex, ytclip, edit)}
assert keys == {
yt.id: "youtube",
site.id: "site:tiktok.com",
up.id: "upload",
plex.id: "plex",
ytclip.id: "youtube",
edit.id: "plex", # an edit sits with the file it was cut from
}
# One folder per origin per user: the two YouTube files share theirs.
assert _folder(db, yt).id == _folder(db, ytclip).id
def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
uid = make_user().id
src = _asset(db, "url", "https://vimeo.com/1")
edit = DownloadJob(
user_id=uid, source_kind="edit", source_ref="e", profile_snapshot={},
source_job_id=None, source_asset_id=src.id, status=JobStatus.DONE,
) # fmt: skip
assert folders.source_key_for(db, edit) == "site:vimeo.com"
def test_an_edit_keeps_its_origin_after_its_parent_and_its_file_are_gone(db, make_user):
# The parent download expires, its file goes with its last holder, and both of the edit's
# pointers to it are cleared: the edit must still go back to the YouTube folder it came from.
uid = make_user().id
parent = _job(db, uid, _asset(db))
edit = _job(
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=parent.id,
source_asset_id=parent.asset_id,
) # fmt: skip
home = _folder(db, edit)
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
db.add(own)
db.flush()
folders.move(db, edit, own)
edit.source_job_id = edit.source_asset_id = None # what ON DELETE SET NULL leaves
db.delete(parent)
db.commit()
folders.move(db, edit, home)
assert edit.folder_id == home.id
folders.move(db, edit, own)
folders.delete_own(db, own)
assert edit.folder_id == home.id
def test_a_folder_deleted_while_a_file_goes_in_takes_that_file_home_too(db, make_user):
# Before, the delete missed the file still on its way in and failed on the foreign key (a 500),
# or the file's insert did after the folder went.
import threading
uid = make_user().id
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
db.add(own)
db.commit()
own_id = own.id
job = _job(db, uid, _asset(db))
home = job.folder_id
target = folders.own_folder(db, uid, own_id) # held from the check until this commits
outcome: list[object] = []
def delete_it():
other = SessionLocal()
try:
outcome.append(folders.delete_own(other, other.get(FileFolder, own_id)))
other.commit()
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
outcome.append(e)
finally:
other.close()
t = threading.Thread(target=delete_it)
t.start()
t.join(timeout=1)
assert t.is_alive() # the delete waits for the file, even before it is written
folders.move(db, job, target)
db.commit()
t.join(timeout=10)
assert outcome == [1]
db.expire_all()
assert db.get(DownloadJob, job.id).folder_id == home
assert db.get(FileFolder, own_id) is None
def test_folders_are_per_user(db, make_user):
a, b = make_user().id, make_user().id
asset = _asset(db)
ja, jb = _job(db, a, asset), _job(db, b, asset)
assert ja.folder_id != jb.folder_id
assert (_folder(db, ja).user_id, _folder(db, jb).user_id) == (a, b)
def test_a_source_folder_keeps_files_the_retention_days_and_uploads_forever(db, make_user):
sysconfig.set_value(db, "download_retention_days", 12)
db.commit()
uid = make_user().id
yt = _job(db, uid, _asset(db))
up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
assert _folder(db, yt).retention_days == 12
assert _folder(db, up).retention_days is None
def test_enqueue_places_the_job_and_a_cache_hit_is_stamped_at_once(db, make_user):
uid = make_user().id
spec = formats.normalize({})
first = service.enqueue(db, uid, "youtube", "abcdefghijk", spec)
assert _folder(db, first).source_key == "youtube"
assert first.expires_at is None # queued: its retention starts when it is done
asset = db.get(MediaAsset, first.asset_id)
asset.status, asset.rel_path = "ready", "a.mp4"
db.commit()
other = make_user().id
hit = service.enqueue(db, other, "youtube", "abcdefghijk", spec)
assert hit.status == JobStatus.DONE
days = _folder(db, hit).retention_days
assert hit.expires_at is not None
assert abs(hit.expires_at - (NOW + timedelta(days=days))) < timedelta(minutes=5)
def test_a_finishing_worker_stamps_the_expiry_unless_kept(db, make_user):
uid = make_user().id
asset = _asset(db)
plain = _job(db, uid, asset, status=JobStatus.RUNNING)
kept = _job(db, uid, asset, status=JobStatus.RUNNING, kept=True)
for job in (plain, kept):
assert service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100)
db.commit()
db.refresh(plain)
db.refresh(kept)
assert plain.expires_at is not None
assert kept.expires_at is None
def _gc(db):
return gcmod.run_download_gc(db)
def _notices(db, uid, kind):
return db.execute(
select(func.count())
.select_from(Notification)
.where(Notification.user_id == uid, Notification.type == kind)
).scalar()
def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, _root):
(_root / "a.mp4").write_bytes(b"x")
uid = make_user().id
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
_gc(db)
_gc(db)
assert _notices(db, uid, "download_expiring") == 1
job.expires_at = NOW - timedelta(minutes=1)
db.commit()
assert _gc(db)["expired"] == 1
assert db.get(DownloadJob, job.id) is None
assert _notices(db, uid, "download_expired") == 1
# Its last holder gone, the shared file and its row go too.
assert not (_root / "a.mp4").exists()
assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
@pytest.mark.parametrize("restart", ["unkeep", "move", "retention"])
def test_a_file_whose_clock_starts_again_is_warned_again(db, make_user, _root, restart):
# The GC warns once per expiry; a file whose expiry was moved must be warned before the new one.
(_root / "a.mp4").write_bytes(b"x")
uid = make_user().id
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
_gc(db)
assert job.expiry_warned
if restart == "unkeep":
folders.set_kept(db, job, True)
folders.set_kept(db, job, False)
elif restart == "move":
home = _folder(db, job)
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
db.add(own)
db.flush()
folders.move(db, job, own)
folders.move(db, job, home)
else:
folders.set_retention(db, _folder(db, job), 90)
db.commit()
assert not job.expiry_warned
def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
(_root / "a.mp4").write_bytes(b"x")
uid = make_user().id
job = _job(db, uid, _asset(db), kept=True, expires_at=NOW - timedelta(days=1))
assert _gc(db)["expired"] == 0
assert db.get(DownloadJob, job.id) is not None
def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
(_root / "a.mp4").write_bytes(b"x")
a, b = make_user().id, make_user().id
asset = _asset(db)
gone = _job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
_gc(db)
assert db.get(DownloadJob, gone.id) is None
assert db.get(DownloadJob, stays.id) is not None
assert (_root / "a.mp4").exists()
def _meanwhile(monkeypatch, act, step="_relock_expired"):
"""Run `act` in another session (another request) between a GC pass's SELECT and its handling of
each row it found — the window a user's click can land in. `step` is the pass's per-row re-read
(expiry, warning, eviction); `act` gets that row's id."""
real = getattr(gcmod, step)
def wrapped(db, key, *rest):
other = SessionLocal()
try:
act(other, key)
other.commit()
finally:
other.close()
return real(db, key, *rest)
monkeypatch.setattr(gcmod, step, wrapped)
def test_a_file_kept_while_the_gc_runs_stays(db, make_user, _root, monkeypatch):
(_root / "a.mp4").write_bytes(b"x")
job = _job(db, make_user().id, _asset(db), expires_at=NOW - timedelta(minutes=1))
_meanwhile(monkeypatch, lambda other, jid: setattr(other.get(DownloadJob, jid), "kept", True))
assert _gc(db)["expired"] == 0
db.expire_all()
assert db.get(DownloadJob, job.id) is not None
assert (_root / "a.mp4").exists()
def test_a_file_deleted_while_the_gc_runs_is_released_once(db, make_user, _root, monkeypatch):
# Released twice, the shared file would go from under the other user's copy.
(_root / "a.mp4").write_bytes(b"x")
a, b = make_user().id, make_user().id
asset = _asset(db)
_job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
def user_deletes(other, jid):
service.delete(other, other.get(DownloadJob, jid))
_meanwhile(monkeypatch, user_deletes)
_gc(db)
db.expire_all()
assert db.get(DownloadJob, stays.id).asset_id == asset.id
assert db.get(MediaAsset, asset.id).ref_count == 1
assert (_root / "a.mp4").exists()
def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
db, make_user, _root
):
sysconfig.set_value(db, "download_total_max_bytes", 15)
db.commit()
a, b = make_user().id, make_user().id
for name in ("old.mp4", "new.mp4"):
(_root / name).write_bytes(b"x")
old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
old.last_access_at = NOW - timedelta(days=9)
new.last_access_at = NOW - timedelta(days=1)
kept = _job(db, a, old, kept=True) # the older file, but kept: the newer one goes instead
ja, jb = _job(db, a, new), _job(db, b, new)
assert _gc(db)["evicted"] == 1
assert db.get(DownloadJob, kept.id) is not None
assert db.get(DownloadJob, ja.id) is None and db.get(DownloadJob, jb.id) is None
assert _notices(db, a, "download_evicted") == 1
assert _notices(db, b, "download_evicted") == 1
assert (_root / "old.mp4").exists() and not (_root / "new.mp4").exists()
def test_a_file_deleted_while_the_gc_warns_does_not_stop_the_run(db, make_user, _root, monkeypatch):
# Written back from a stale read, the warning of a deleted file failed the whole run: nothing
# expired or was evicted that cycle, nor the next while the race lasted.
(_root / "a.mp4").write_bytes(b"x")
(_root / "b.mp4").write_bytes(b"x")
uid = make_user().id
_job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=1))
gone = _job(
db, uid, _asset(db, ref="bbbbbbbbbbb", rel="b.mp4"), expires_at=NOW - timedelta(minutes=1)
)
def user_deletes(other, jid):
service.delete(other, other.get(DownloadJob, jid))
_meanwhile(monkeypatch, user_deletes, step="_mark_warned")
result = _gc(db)
assert (result["warned"], result["expired"]) == (0, 1)
db.expire_all()
assert db.get(DownloadJob, gone.id) is None
assert _notices(db, uid, "download_expiring") == 0
def test_a_file_re_timed_while_the_gc_warns_keeps_its_new_expiry_unwarned(
db, make_user, _root, monkeypatch
):
# Moved, un-kept or its folder's retention changed: a fresh expiry, warned when it comes — not
# marked warned now for a date it no longer has.
uid = make_user().id
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=1))
later = NOW + timedelta(days=30)
def re_timed(other, jid):
moved = other.get(DownloadJob, jid)
moved.expires_at = later
moved.expiry_warned = False
_meanwhile(monkeypatch, re_timed, step="_mark_warned")
assert _gc(db)["warned"] == 0
db.expire_all()
fresh = db.get(DownloadJob, job.id)
assert (fresh.expires_at, fresh.expiry_warned) == (later, False)
assert _notices(db, uid, "download_expiring") == 0
def _two_over_the_cap(db, make_user, _root):
sysconfig.set_value(db, "download_total_max_bytes", 15)
db.commit()
a, b = make_user().id, make_user().id
for name in ("old.mp4", "new.mp4"):
(_root / name).write_bytes(b"x")
old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
old.last_access_at = NOW - timedelta(days=9)
new.last_access_at = NOW - timedelta(days=1)
return _job(db, a, old), _job(db, b, new)
def test_a_file_deleted_while_the_cache_is_over_its_cap_frees_that_space_and_no_more(
db, make_user, _root, monkeypatch
):
# Its bytes still counted, the pass went on to evict another user's file for space already free.
jold, jnew = _two_over_the_cap(db, make_user, _root)
def holder_deletes(other, asset_id):
if asset_id == jold.asset_id:
service.delete(other, other.get(DownloadJob, jold.id))
_meanwhile(monkeypatch, holder_deletes, step="_relock_evictable")
assert _gc(db)["evicted"] == 0
db.expire_all()
assert db.get(DownloadJob, jnew.id) is not None
assert (_root / "new.mp4").exists()
def test_a_file_kept_while_the_cache_is_over_its_cap_stays(db, make_user, _root, monkeypatch):
jold, jnew = _two_over_the_cap(db, make_user, _root)
def holder_keeps(other, asset_id):
if asset_id == jold.asset_id:
other.get(DownloadJob, jold.id).kept = True
_meanwhile(monkeypatch, holder_keeps, step="_relock_evictable")
assert _gc(db)["evicted"] == 1 # the newer one goes in its place
db.expire_all()
assert db.get(DownloadJob, jold.id) is not None
assert (_root / "old.mp4").exists()
assert db.get(DownloadJob, jnew.id) is None
def _in_thread(work):
"""Run `work(other_session)` in another thread and session; returns (thread, outcome list)."""
import threading
outcome: list[object] = []
def run():
other = SessionLocal()
try:
outcome.append(work(other))
other.commit()
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
outcome.append(e)
finally:
other.close()
t = threading.Thread(target=run)
t.start()
return t, outcome
def test_a_retention_changed_while_a_file_goes_in_times_that_file_too(db, make_user):
# The change used to list the folder's files before the arriving one was in, so that one kept the
# old retention — deleted later from a folder now set to keep files forever.
uid = make_user().id
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7)
db.add(own)
db.commit()
own_id = own.id
job = _job(db, uid, _asset(db))
target = folders.own_folder(db, uid, own_id) # held from the check until this commits
t, outcome = _in_thread(
lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
)
t.join(timeout=1)
assert t.is_alive() # the change waits for the file being put in
folders.move(db, job, target) # stamped with the 7 days it sees
db.commit()
t.join(timeout=10)
assert outcome == [7] # the retention it replaced
db.expire_all()
assert db.get(DownloadJob, job.id).expires_at is None
def test_a_retention_changed_while_a_file_finishes_there_times_that_file_too(db, make_user):
# The worker's finish read the folder's old retention unlocked, while the change listed the done
# files before this one was among them.
uid = make_user().id
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7)
db.add(own)
db.commit()
own_id = own.id
job = _job(db, uid, _asset(db), status=JobStatus.RUNNING)
folders.move(db, job, own)
db.commit()
job.status = JobStatus.DONE
folders.stamp_done(db, job) # the finish: stamped with the 7 days it reads
db.flush()
t, outcome = _in_thread(
lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
)
t.join(timeout=1)
assert t.is_alive() # the change waits for the file being stamped
db.commit()
t.join(timeout=10)
assert outcome == [7] # the retention it replaced
db.expire_all()
assert db.get(DownloadJob, job.id).expires_at is None
def _deleted_uncommitted(db, job):
"""`service.delete` up to its commit: the file is gone in this session, its row still locked."""
service.release_asset(db, job)
db.delete(job)
db.flush()
def test_a_file_moved_out_while_its_folder_is_deleted_stays_where_it_went(db, make_user):
uid = make_user().id
f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F")
g = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="G")
db.add_all([f, g])
db.commit()
f_id, g_id = f.id, g.id
job = _job(db, uid, _asset(db))
folders.move(db, job, f)
db.commit()
folders.move(db, job, folders.own_folder(db, uid, g_id))
db.flush() # the move is written, not yet committed
t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id)))
t.join(timeout=1)
assert t.is_alive() # the delete waits for the file it would take home
db.commit()
t.join(timeout=10)
assert outcome == [0]
db.expire_all()
assert db.get(DownloadJob, job.id).folder_id == g_id
def test_a_file_deleted_while_its_folder_is_deleted_does_not_fail_the_delete(db, make_user, _root):
(_root / "a.mp4").write_bytes(b"x")
uid = make_user().id
f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F")
db.add(f)
db.commit()
f_id = f.id
job = _job(db, uid, _asset(db))
folders.move(db, job, f)
db.commit()
_deleted_uncommitted(db, db.get(DownloadJob, job.id))
t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id)))
t.join(timeout=1)
assert t.is_alive() # the delete waits for the file being deleted
db.commit()
t.join(timeout=10)
assert outcome == [0]
db.expire_all()
assert db.get(FileFolder, f_id) is None
def _own_with_a_file(db, uid, retention=30):
"""A user's own folder keeping files `retention` days, holding one done file due in as many."""
own = FileFolder(
user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=retention
)
db.add(own)
db.flush()
job = _job(db, uid, _asset(db))
job.folder_id = own.id
job.expires_at = NOW + timedelta(days=retention)
db.commit()
return own.id, job.id
def _elsewhere(act):
other = SessionLocal()
try:
act(other)
other.commit()
finally:
other.close()
def test_a_retention_change_works_from_what_a_change_before_it_left(db, make_user):
# The locked re-read was never fetched, so the change shifted the files from the value read before
# its wait: 30 -> 10 -> 60 left them due in 40 days under a folder saying 60.
own_id, job_id = _own_with_a_file(db, make_user().id)
stale = db.get(FileFolder, own_id) # read before the other change lands
_elsewhere(lambda other: folders.set_retention(other, other.get(FileFolder, own_id), 10))
assert folders.set_retention(db, stale, 60) == 10
db.commit()
db.expire_all()
due = db.get(DownloadJob, job_id).expires_at - NOW
assert timedelta(days=59) < due < timedelta(days=61)
def test_a_retention_change_to_a_folder_deleted_meanwhile_is_refused(db, make_user):
own_id, _ = _own_with_a_file(db, make_user().id)
stale = db.get(FileFolder, own_id)
_elsewhere(lambda other: folders.delete_own(other, other.get(FileFolder, own_id)))
with pytest.raises(folders.FolderError, match="file_folder_gone"):
folders.set_retention(db, stale, 60)
def test_a_file_moved_out_while_its_folder_retention_changes_keeps_its_new_folders(db, make_user):
uid = make_user().id
own_id, job_id = _own_with_a_file(db, uid)
g = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="G", retention_days=7)
db.add(g)
db.commit()
g_id = g.id
folders.move(db, db.get(DownloadJob, job_id), folders.own_folder(db, uid, g_id))
db.flush() # the move is written, not yet committed
t, outcome = _in_thread(
lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
)
t.join(timeout=1)
assert t.is_alive() # the change waits for the file it would time
db.commit()
t.join(timeout=10)
assert outcome == [30]
db.expire_all()
job = db.get(DownloadJob, job_id)
assert job.folder_id == g_id
assert job.expires_at is not None # G's 7 days, not the forever of the folder it left
def test_a_file_deleted_while_its_folder_retention_changes_does_not_fail_the_change(
db, make_user, _root
):
(_root / "a.mp4").write_bytes(b"x")
own_id, job_id = _own_with_a_file(db, make_user().id)
_deleted_uncommitted(db, db.get(DownloadJob, job_id))
t, outcome = _in_thread(
lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
)
t.join(timeout=1)
assert t.is_alive() # the change waits for the file it would time
db.commit()
t.join(timeout=10)
assert outcome == [30]
def test_a_file_finishing_while_its_folder_is_deleted_gets_its_new_folders_retention(db, make_user):
# The finish read the folder unlocked, then waited out the delete and found no folder: the file
# was done with no expiry at all, in a source folder that keeps files 30 days.
uid = make_user().id
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7)
db.add(own)
db.commit()
own_id = own.id
job = _job(db, uid, _asset(db), status=JobStatus.RUNNING)
job_id = job.id
folders.move(db, job, own)
db.commit()
folders.delete_own(db, db.get(FileFolder, own_id))
db.flush() # the delete, not yet committed
t, outcome = _in_thread(
lambda other: service.finalize_if_running(other, job_id, status=JobStatus.DONE)
)
t.join(timeout=1)
assert t.is_alive() # the finish waits for the delete
db.commit()
t.join(timeout=10)
assert outcome == [True]
db.expire_all()
done = db.get(DownloadJob, job_id)
assert done.folder_id != own_id
assert done.expires_at is not None
def _an_upload_over_the_cap_too(db, uid, _root):
(_root / "u.bin").write_bytes(b"x")
return _job(db, uid, _asset(db, "upload", "u1", rel="u.bin", size=10), kind="upload")
def test_space_freed_elsewhere_while_the_cache_is_over_its_cap_evicts_no_more(
db, make_user, _root, monkeypatch
):
# The pass counted the disk once: an upload deleted meanwhile still counted, so a second user's
# file went for space already free.
jold, jnew = _two_over_the_cap(db, make_user, _root)
up = _an_upload_over_the_cap_too(db, make_user().id, _root)
def upload_deleted(other, asset_id):
if asset_id == jold.asset_id:
service.delete(other, other.get(DownloadJob, up.id))
_meanwhile(monkeypatch, upload_deleted, step="_relock_evictable")
assert _gc(db)["evicted"] == 1
db.expire_all()
assert db.get(DownloadJob, jnew.id) is not None
assert (_root / "new.mp4").exists()
def test_a_file_asked_for_again_while_the_cache_is_over_its_cap_stays(
db, make_user, _root, monkeypatch
):
# A cache hit between the pass's SELECT and its lock: the file the user just asked for went.
jold, jnew = _two_over_the_cap(db, make_user, _root)
def asked_again(other, asset_id):
if asset_id == jold.asset_id:
other.get(MediaAsset, asset_id).last_access_at = datetime.now(timezone.utc)
_meanwhile(monkeypatch, asked_again, step="_relock_evictable")
_gc(db)
db.expire_all()
assert db.get(DownloadJob, jold.id) is not None
assert (_root / "old.mp4").exists()
def test_a_file_asked_for_while_its_cached_copy_is_freed_gets_a_fresh_one(db, make_user):
# The enqueue read the cached copy unlocked: freed meanwhile, its job pointed at a row that was gone
# (a foreign-key failure, a 500).
asset = _asset(db)
asset_id, sig = asset.id, asset.format_sig
db.commit()
db.execute(select(MediaAsset).where(MediaAsset.id == asset_id).with_for_update())
db.delete(db.get(MediaAsset, asset_id))
db.flush() # the release, not yet committed
t, outcome = _in_thread(
lambda other: service.get_or_create_asset(other, "youtube", "abcdefghijk", sig).id
)
t.join(timeout=1)
assert t.is_alive() # it waits for the release
db.commit()
t.join(timeout=10)
[fresh] = outcome
assert isinstance(fresh, int) and fresh != asset_id
def test_an_empty_volume_in_the_librarys_place_is_noticed(db, make_user, _root):
# A fresh volume is a real, writable mount: only the files the library names tell it is the wrong one.
assert service.library_left_behind(db, str(_root)) is False # nothing named yet
_asset(db, rel="a.mp4")
db.commit()
assert service.library_left_behind(db, str(_root)) is True
(_root / "a.mp4").write_bytes(b"x")
assert service.library_left_behind(db, str(_root)) is False
def test_deleting_a_user_takes_their_folders_and_files_with_them(db, make_user):
# Both tables cascade from users; the files' folder FK refuses deleting a folder that still holds
# files, which must not stop the account's own deletion.
user = make_user()
_job(db, user.id, _asset(db))
db.delete(user)
db.commit()
assert db.execute(select(func.count()).select_from(FileFolder)).scalar() == 0
assert db.execute(select(func.count()).select_from(DownloadJob)).scalar() == 0
def test_the_longest_host_has_room_for_its_folder(db, make_user):
label = "a" * 63
host = ".".join([label, label, label, "b" * 58]) + ".hu" # 253 characters, the DNS maximum
assert len(host) == 253
uid = make_user().id
job = _job(db, uid, _asset(db, "url", f"https://{host}/v"), kind="url")
assert _folder(db, job).source_key == f"site:{host}"
def test_the_ready_notice_names_the_file_and_its_folder(db, make_user):
# The notice's "Open" goes to the file in its folder, so both ride along as data.
from app.downloads import pipeline
user = make_user()
job = _job(db, user.id, _asset(db))
pipeline._notify_download_done(db, user.id, "A title", job.id, kind="download")
db.commit()
note = db.scalars(select(Notification).where(Notification.user_id == user.id)).one()
assert note.type == "download_done"
assert note.data == {
"title": "A title",
"kind": "download",
"job_id": job.id,
"folder_id": job.folder_id,
}
assert job.folder_id is not None
Reference in New Issue View Git Blame Copy Permalink