Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c82cf25bcf | ||
|
|
634e9bb17c | ||
|
|
ce2954cf2b | ||
|
|
2229b01a91 | ||
|
|
b17b47f54e | ||
|
|
1e2b3c5d29 | ||
|
|
d065902404 | ||
|
|
a300977ed4 | ||
|
|
ddb7818d38 | ||
|
|
48787af11c | ||
|
|
b733ee84fa |
@@ -6,6 +6,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Channel, Subscription, User
|
||||
from app.utils import normalize_description
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
@@ -55,7 +56,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
|
||||
|
||||
channel.title = snippet.get("title") or channel.title
|
||||
channel.handle = snippet.get("customUrl")
|
||||
channel.description = snippet.get("description")
|
||||
channel.description = normalize_description(snippet.get("description"))
|
||||
channel.thumbnail_url = (
|
||||
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
||||
)
|
||||
@@ -104,7 +105,7 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
channel = Channel(
|
||||
id=cid,
|
||||
title=sub.get("title"),
|
||||
description=sub.get("description"),
|
||||
description=normalize_description(sub.get("description")),
|
||||
thumbnail_url=sub.get("thumbnail_url"),
|
||||
)
|
||||
db.add(channel)
|
||||
|
||||
@@ -36,3 +36,18 @@ def to_tsquery_str(q: str) -> str | None:
|
||||
return None
|
||||
tokens[-1] = tokens[-1] + ":*"
|
||||
return " & ".join(tokens)
|
||||
|
||||
|
||||
def normalize_description(text: str | None) -> str | None:
|
||||
"""Strip whitespace-only lines from a CHANNEL description before storing. YouTube channel
|
||||
descriptions pad their contact/link blocks with runs of blank lines; keeping them wastes storage,
|
||||
bloats the About popover, and adds nothing to the search document. Drops purely-empty lines and
|
||||
right-strips each kept line; `str.splitlines()` also folds CR / CRLF and Unicode line separators
|
||||
to "\\n" (a benign line-ending normalization). None stays None; all-blank collapses to "".
|
||||
|
||||
Deliberately CHANNEL-only — video descriptions are left with their original formatting: they are
|
||||
user-facing prose where paragraph breaks carry meaning, unlike a channel's metadata block."""
|
||||
if text is None:
|
||||
return None
|
||||
kept = [line.rstrip() for line in text.splitlines() if line.strip()]
|
||||
return "\n".join(kept)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from app.utils import iso
|
||||
from app.utils import iso, normalize_description
|
||||
|
||||
|
||||
def test_none_maps_to_none():
|
||||
@@ -28,3 +28,30 @@ def test_matches_datetime_isoformat_contract():
|
||||
# hand-roll `x.isoformat() if x else None`.
|
||||
dt = datetime(2020, 1, 2, 3, 4, 5, 678901, tzinfo=timezone.utc)
|
||||
assert iso(dt) == dt.isoformat()
|
||||
|
||||
|
||||
def test_normalize_description_none_stays_none():
|
||||
assert normalize_description(None) is None
|
||||
|
||||
|
||||
def test_normalize_description_drops_blank_lines():
|
||||
# A YouTube contact block padded with runs of blank lines — the waste we strip before storing.
|
||||
raw = "Contact us\n\n\nEmail: a@b.com\n \nInstagram: @x\n\n\n"
|
||||
assert normalize_description(raw) == "Contact us\nEmail: a@b.com\nInstagram: @x"
|
||||
|
||||
|
||||
def test_normalize_description_right_strips_and_trims():
|
||||
assert normalize_description(" hi there \n\n bye ") == " hi there\n bye"
|
||||
|
||||
|
||||
def test_normalize_description_all_blank_collapses_to_empty():
|
||||
assert normalize_description("\n \n\t\n") == ""
|
||||
|
||||
|
||||
def test_normalize_description_single_line_unchanged():
|
||||
assert normalize_description("just one line") == "just one line"
|
||||
|
||||
|
||||
def test_normalize_description_folds_crlf_line_endings():
|
||||
# splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization.
|
||||
assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc"
|
||||
|
||||
@@ -28,6 +28,10 @@ export default function AboutCell({ text, channelId }: { text: string | null; ch
|
||||
offset: 6,
|
||||
// Pop instantly like the old cell did; the close grace lets the cursor reach the card.
|
||||
hoverDelay: { open: 0, close: 160 },
|
||||
// Cap the panel to the space between the trigger and the viewport edge (publishes --fit-max-h) so a
|
||||
// long About never runs its top off-screen — at high browser zoom the short viewport made a fixed
|
||||
// max-height overflow above the top edge, where the scroll container can't reach.
|
||||
fitHeight: true,
|
||||
});
|
||||
|
||||
const { data: ch } = useQuery({
|
||||
@@ -51,7 +55,10 @@ export default function AboutCell({ text, channelId }: { text: string | null; ch
|
||||
</span>
|
||||
<PopoverSurface
|
||||
popover={pop}
|
||||
className="glass w-[480px] max-h-112 overflow-y-auto px-3.5 py-3 rounded-lg animate-[fadeIn_0.12s_ease]"
|
||||
className="glass w-[480px] overflow-y-auto px-3.5 py-3 rounded-lg animate-[fadeIn_0.12s_ease]"
|
||||
// min(design cap, available space): keeps the 28rem cap on a tall viewport, shrinks to fit on a
|
||||
// short one (high zoom). --fit-max-h is set by the fitHeight `size` middleware once positioned.
|
||||
style={{ maxHeight: "min(28rem, var(--fit-max-h, 28rem))" }}
|
||||
>
|
||||
{/* Full About once the detail loads; until then the list's truncated description. */}
|
||||
{ch ? (
|
||||
|
||||
@@ -131,7 +131,14 @@ export function useAnchoredPopover(opts: AnchoredPopoverOptions) {
|
||||
size({
|
||||
padding: 8,
|
||||
apply({ availableHeight, elements }) {
|
||||
// Cap the positioned element, AND publish the available space as a custom property so
|
||||
// the INNER styled/scroll wrapper (PopoverSurface renders `className` on a child, not on
|
||||
// the floating element) can cap itself too — e.g. `maxHeight: min(28rem, var(--fit-max-h))`
|
||||
// keeps a caller's design cap while still shrinking to fit at high zoom / short viewports.
|
||||
// Capping only the outer element would leave the inner overflow-y-auto box at its fixed
|
||||
// height, spilling its top off-screen (the whole point of fitHeight).
|
||||
elements.floating.style.maxHeight = `${availableHeight}px`;
|
||||
elements.floating.style.setProperty("--fit-max-h", `${availableHeight}px`);
|
||||
},
|
||||
}),
|
||||
]
|
||||
|
||||
@@ -84,10 +84,18 @@ function StatusPill({ job }: { job: DownloadJob }) {
|
||||
}
|
||||
|
||||
function Thumb({ job }: { job: DownloadJob }) {
|
||||
// Prefer the source thumbnail; fall back to our generated poster for thumbnail-less sources.
|
||||
const primary = job.asset?.thumbnail_url || job.poster_url;
|
||||
// A remote i.ytimg.com thumbnail 404s once the source video is deleted/privated, leaving a
|
||||
// broken image. On load failure fall back to our always-present self-hosted poster.
|
||||
// Prefer OUR self-hosted poster: it's always present on disk, never expires, and is same-origin, so
|
||||
// a source-CDN signature expiry (Facebook's fbcdn URLs expire within days) or an ad-blocker/Shields
|
||||
// rule on the source domain can't break it. The source's remote thumbnail_url is only a fallback for
|
||||
// the rare poster-less asset (a thumbnail-less source whose ffmpeg frame-cut also failed).
|
||||
const remote = job.asset?.thumbnail_url ?? null;
|
||||
// `job.poster_url` only SIGNALS a poster exists; the actual <img> src must carry ?account= (see
|
||||
// api.posterImageUrl). A plain <img> can't send the X-Siftlode-Account header, so without it the
|
||||
// poster resolves to the session-default account and 404s for a per-tab account's download — which,
|
||||
// now that the poster is the primary image, reads as a blank Library thumbnail.
|
||||
const poster = job.poster_url ? api.posterImageUrl(job.id) : null;
|
||||
const primary = poster || remote;
|
||||
// On load failure (a genuinely missing poster file) fall back to the remote source thumbnail.
|
||||
const [src, setSrc] = useState(primary);
|
||||
useEffect(() => setSrc(primary), [primary]);
|
||||
return (
|
||||
@@ -99,7 +107,7 @@ function Thumb({ job }: { job: DownloadJob }) {
|
||||
loading="lazy"
|
||||
className="w-full h-full object-cover"
|
||||
onError={() => {
|
||||
if (job.poster_url && src !== job.poster_url) setSrc(job.poster_url);
|
||||
if (remote && src !== remote) setSrc(remote);
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import { useVirtualizer } from "@tanstack/react-virtual";
|
||||
import { shouldPrefetchNext } from "./readAhead";
|
||||
|
||||
// The virtualization engine, lifted out of VirtualFeed so any long client- or server-side list can
|
||||
// reuse it (the feed's video grid, the channel manager's card/row views). It owns ONLY the layout
|
||||
@@ -23,9 +24,6 @@ import { useVirtualizer } from "@tanstack/react-virtual";
|
||||
// (grid-cols-[repeat(auto-fill,minmax(260px,1fr))] with a gap-4 / 1rem gutter).
|
||||
const GRID_GAP = 16;
|
||||
const LIST_GAP = 4;
|
||||
// Start fetching the next page this many rows before the end scrolls into view
|
||||
// (keeps the old IntersectionObserver's generous prefetch feel).
|
||||
const PREFETCH_ROWS = 4;
|
||||
|
||||
/** Columns that fit `width`, or 1 for the single-column layout (`minCol == null`). */
|
||||
function columnsFor(minCol: number | null, width: number): number {
|
||||
@@ -140,13 +138,17 @@ export default function VirtualGrid<T>({
|
||||
|
||||
const virtualRows = virtualizer.getVirtualItems();
|
||||
|
||||
// Infinite-scroll trigger: once the last rendered row is within PREFETCH_ROWS of the end, pull
|
||||
// the next page. A no-op for finite lists (no fetchNextPage / hasNextPage false).
|
||||
// Infinite-scroll read-ahead: pull the next page once the last rendered row is within the
|
||||
// read-ahead window of the loaded end (see readAhead.ts — the window scales with the rendered row
|
||||
// count, clamped so a tall window doesn't prefetch unboundedly). A no-op for finite lists (no
|
||||
// fetchNextPage / hasNextPage false). Because this effect re-runs when a page settles
|
||||
// (isFetchingNextPage flips false) it CHAINS — refilling the buffer as the user consumes it, so a
|
||||
// fast scroll reaches the loaded end far less often.
|
||||
useEffect(() => {
|
||||
if (!fetchNextPage) return;
|
||||
if (!fetchNextPage || !hasNextPage || isFetchingNextPage) return;
|
||||
const last = virtualRows[virtualRows.length - 1];
|
||||
if (!last) return;
|
||||
if (last.index >= rows.length - 1 - PREFETCH_ROWS && hasNextPage && !isFetchingNextPage) {
|
||||
if (shouldPrefetchNext(last.index, rows.length, virtualRows.length)) {
|
||||
fetchNextPage();
|
||||
}
|
||||
}, [virtualRows, rows.length, hasNextPage, isFetchingNextPage, fetchNextPage]);
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
MAX_LOOKAHEAD_ROWS,
|
||||
PREFETCH_ROWS,
|
||||
PREFETCH_VIEWPORTS,
|
||||
readAheadLookahead,
|
||||
shouldPrefetchNext,
|
||||
} from "./readAhead";
|
||||
|
||||
describe("readAheadLookahead", () => {
|
||||
it("floors at PREFETCH_ROWS when nothing (or little) is rendered", () => {
|
||||
expect(readAheadLookahead(0)).toBe(PREFETCH_ROWS);
|
||||
expect(readAheadLookahead(1)).toBe(PREFETCH_ROWS); // 1*2=2 < floor 4
|
||||
});
|
||||
|
||||
it("scales with the rendered row count for a normal viewport", () => {
|
||||
expect(readAheadLookahead(10)).toBe(10 * PREFETCH_VIEWPORTS); // 20
|
||||
expect(readAheadLookahead(15)).toBe(30);
|
||||
});
|
||||
|
||||
it("caps at MAX_LOOKAHEAD_ROWS so a tall window doesn't prefetch unboundedly", () => {
|
||||
// 30 rendered rows would be 60 uncapped — the cap is what bounds per-mount feed prefetch.
|
||||
expect(readAheadLookahead(30)).toBe(MAX_LOOKAHEAD_ROWS);
|
||||
expect(readAheadLookahead(1000)).toBe(MAX_LOOKAHEAD_ROWS);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldPrefetchNext", () => {
|
||||
it("fetches once the last rendered row enters the read-ahead window", () => {
|
||||
// 100 rows loaded, viewport renders 10 → lookahead 20. Fetch when lastIndex >= 100-1-20 = 79.
|
||||
expect(shouldPrefetchNext(78, 100, 10)).toBe(false);
|
||||
expect(shouldPrefetchNext(79, 100, 10)).toBe(true);
|
||||
expect(shouldPrefetchNext(99, 100, 10)).toBe(true);
|
||||
});
|
||||
|
||||
it("fetches eagerly while fewer rows are loaded than the lookahead (early pages)", () => {
|
||||
// Only 15 rows loaded, lookahead 20 → the whole list is inside the window.
|
||||
expect(shouldPrefetchNext(8, 15, 10)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not fetch near the top of a long loaded list", () => {
|
||||
expect(shouldPrefetchNext(12, 500, 10)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
// Infinite-scroll read-ahead policy for VirtualGrid, extracted as a pure function so the decision is
|
||||
// unit-testable without a layout engine.
|
||||
//
|
||||
// The next page is pulled once the last rendered row comes within `lookahead` rows of the loaded end,
|
||||
// where lookahead scales with the number of rows the viewport currently renders (so it adapts to
|
||||
// window height / column count / row height) but is clamped to a floor and a ceiling:
|
||||
// - floor PREFETCH_ROWS: the finite-list / first-paint case where nothing is rendered yet.
|
||||
// - ceiling MAX_LOOKAHEAD_ROWS: a tall window renders many rows, and an uncapped lookahead would
|
||||
// eagerly prefetch several pages on every mount / filter change — wasted work on the app's
|
||||
// heaviest query (the feed). The cap bounds that while still buffering a couple of screens ahead.
|
||||
export const PREFETCH_ROWS = 4;
|
||||
export const PREFETCH_VIEWPORTS = 2;
|
||||
export const MAX_LOOKAHEAD_ROWS = 40;
|
||||
|
||||
/** How many rows ahead of the last rendered row to keep buffered, for a viewport rendering
|
||||
* `renderedRows` rows. Clamped to [PREFETCH_ROWS, MAX_LOOKAHEAD_ROWS]. */
|
||||
export function readAheadLookahead(renderedRows: number): number {
|
||||
return Math.min(MAX_LOOKAHEAD_ROWS, Math.max(PREFETCH_ROWS, renderedRows * PREFETCH_VIEWPORTS));
|
||||
}
|
||||
|
||||
/** Whether to fetch the next page: true once the last rendered row (`lastIndex`) is within the
|
||||
* read-ahead window of the loaded end (`rowCount` total loaded rows). Because the caller re-runs
|
||||
* this as each page settles, it chains — refilling the buffer as the user consumes it. */
|
||||
export function shouldPrefetchNext(
|
||||
lastIndex: number,
|
||||
rowCount: number,
|
||||
renderedRows: number,
|
||||
): boolean {
|
||||
return lastIndex >= rowCount - 1 - readAheadLookahead(renderedRows);
|
||||
}
|
||||
@@ -225,6 +225,15 @@ export const downloadsApi = {
|
||||
? `/api/downloads/${id}/storyboard.jpg?account=${a}`
|
||||
: `/api/downloads/${id}/storyboard.jpg`;
|
||||
},
|
||||
// The locally-generated poster (an <img> src → direct nav, so same ?account= wallet gate). Without
|
||||
// it the poster resolves to the session-default account and 404s for a per-tab account's download —
|
||||
// which reads as a blank Library thumbnail once the poster is the card's primary image.
|
||||
posterImageUrl: (id: number): string => {
|
||||
const a = getActiveAccount();
|
||||
return a != null
|
||||
? `/api/downloads/${id}/poster.jpg?account=${a}`
|
||||
: `/api/downloads/${id}/poster.jpg`;
|
||||
},
|
||||
|
||||
// admin
|
||||
adminDownloads: (): Promise<DownloadJob[]> => req("/api/admin/downloads"),
|
||||
|
||||
@@ -14,6 +14,26 @@ export interface ReleaseEntry {
|
||||
}
|
||||
|
||||
export const RELEASE_NOTES: ReleaseEntry[] = [
|
||||
{
|
||||
version: "0.69.2",
|
||||
date: "2026-08-19",
|
||||
summary: "Download thumbnails now load for everyone, including multi-account sessions.",
|
||||
fixes: [
|
||||
"Fixed Library thumbnails staying blank when you're signed into more than one account. The poster image is loaded directly by the browser, which can't send the active-account header, so it resolved to your default account and returned “not found” for downloads owned by another of your accounts. The poster URL now carries the active account (the same way the file download and filmstrip already do), so the local poster loads regardless of which of your accounts is active.",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "0.69.1",
|
||||
date: "2026-08-19",
|
||||
summary:
|
||||
"Smoother fast-scrolling in the feed, reliable download thumbnails, and a tidier channel About popover.",
|
||||
fixes: [
|
||||
"Download thumbnails are reliable again. The Library now shows the copy saved on the server instead of the source's own link — which for Facebook videos expires within days (and can be blocked by a browser tracker shield), leaving a blank thumbnail even though the image was on disk the whole time.",
|
||||
"Fast-scrolling the feed no longer stalls. A quick flick of the scroll wheel used to outrun the loader and leave you waiting several seconds on “Load more”; the feed now keeps a couple of screens of videos loaded ahead, so the content is usually already there when you reach it.",
|
||||
"The channel About popover stays on screen at higher browser zoom. A long description could push the top of the box above the edge of the window where you couldn't read it; it now shrinks to fit the available height and scrolls inside.",
|
||||
"Channel descriptions are tidier: the runs of blank lines YouTube pads its contact/link blocks with are stripped, so the About box is more compact.",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "0.69.0",
|
||||
date: "2026-08-19",
|
||||
|
||||
Reference in New Issue
Block a user