Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2229b01a91 | ||
|
|
b17b47f54e | ||
|
|
1e2b3c5d29 | ||
|
|
d065902404 | ||
|
|
a300977ed4 | ||
|
|
ddb7818d38 | ||
|
|
48787af11c | ||
|
|
b733ee84fa |
@@ -6,6 +6,7 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models import Channel, Subscription, User
|
from app.models import Channel, Subscription, User
|
||||||
|
from app.utils import normalize_description
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.sync")
|
log = logging.getLogger("siftlode.sync")
|
||||||
@@ -55,7 +56,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
|
|||||||
|
|
||||||
channel.title = snippet.get("title") or channel.title
|
channel.title = snippet.get("title") or channel.title
|
||||||
channel.handle = snippet.get("customUrl")
|
channel.handle = snippet.get("customUrl")
|
||||||
channel.description = snippet.get("description")
|
channel.description = normalize_description(snippet.get("description"))
|
||||||
channel.thumbnail_url = (
|
channel.thumbnail_url = (
|
||||||
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
||||||
)
|
)
|
||||||
@@ -104,7 +105,7 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
|||||||
channel = Channel(
|
channel = Channel(
|
||||||
id=cid,
|
id=cid,
|
||||||
title=sub.get("title"),
|
title=sub.get("title"),
|
||||||
description=sub.get("description"),
|
description=normalize_description(sub.get("description")),
|
||||||
thumbnail_url=sub.get("thumbnail_url"),
|
thumbnail_url=sub.get("thumbnail_url"),
|
||||||
)
|
)
|
||||||
db.add(channel)
|
db.add(channel)
|
||||||
|
|||||||
@@ -36,3 +36,18 @@ def to_tsquery_str(q: str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
tokens[-1] = tokens[-1] + ":*"
|
tokens[-1] = tokens[-1] + ":*"
|
||||||
return " & ".join(tokens)
|
return " & ".join(tokens)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_description(text: str | None) -> str | None:
|
||||||
|
"""Strip whitespace-only lines from a CHANNEL description before storing. YouTube channel
|
||||||
|
descriptions pad their contact/link blocks with runs of blank lines; keeping them wastes storage,
|
||||||
|
bloats the About popover, and adds nothing to the search document. Drops purely-empty lines and
|
||||||
|
right-strips each kept line; `str.splitlines()` also folds CR / CRLF and Unicode line separators
|
||||||
|
to "\\n" (a benign line-ending normalization). None stays None; all-blank collapses to "".
|
||||||
|
|
||||||
|
Deliberately CHANNEL-only — video descriptions are left with their original formatting: they are
|
||||||
|
user-facing prose where paragraph breaks carry meaning, unlike a channel's metadata block."""
|
||||||
|
if text is None:
|
||||||
|
return None
|
||||||
|
kept = [line.rstrip() for line in text.splitlines() if line.strip()]
|
||||||
|
return "\n".join(kept)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
from app.utils import iso
|
from app.utils import iso, normalize_description
|
||||||
|
|
||||||
|
|
||||||
def test_none_maps_to_none():
|
def test_none_maps_to_none():
|
||||||
@@ -28,3 +28,30 @@ def test_matches_datetime_isoformat_contract():
|
|||||||
# hand-roll `x.isoformat() if x else None`.
|
# hand-roll `x.isoformat() if x else None`.
|
||||||
dt = datetime(2020, 1, 2, 3, 4, 5, 678901, tzinfo=timezone.utc)
|
dt = datetime(2020, 1, 2, 3, 4, 5, 678901, tzinfo=timezone.utc)
|
||||||
assert iso(dt) == dt.isoformat()
|
assert iso(dt) == dt.isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_none_stays_none():
|
||||||
|
assert normalize_description(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_drops_blank_lines():
|
||||||
|
# A YouTube contact block padded with runs of blank lines — the waste we strip before storing.
|
||||||
|
raw = "Contact us\n\n\nEmail: a@b.com\n \nInstagram: @x\n\n\n"
|
||||||
|
assert normalize_description(raw) == "Contact us\nEmail: a@b.com\nInstagram: @x"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_right_strips_and_trims():
|
||||||
|
assert normalize_description(" hi there \n\n bye ") == " hi there\n bye"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_all_blank_collapses_to_empty():
|
||||||
|
assert normalize_description("\n \n\t\n") == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_single_line_unchanged():
|
||||||
|
assert normalize_description("just one line") == "just one line"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_folds_crlf_line_endings():
|
||||||
|
# splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization.
|
||||||
|
assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc"
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ export default function AboutCell({ text, channelId }: { text: string | null; ch
|
|||||||
offset: 6,
|
offset: 6,
|
||||||
// Pop instantly like the old cell did; the close grace lets the cursor reach the card.
|
// Pop instantly like the old cell did; the close grace lets the cursor reach the card.
|
||||||
hoverDelay: { open: 0, close: 160 },
|
hoverDelay: { open: 0, close: 160 },
|
||||||
|
// Cap the panel to the space between the trigger and the viewport edge (publishes --fit-max-h) so a
|
||||||
|
// long About never runs its top off-screen — at high browser zoom the short viewport made a fixed
|
||||||
|
// max-height overflow above the top edge, where the scroll container can't reach.
|
||||||
|
fitHeight: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
const { data: ch } = useQuery({
|
const { data: ch } = useQuery({
|
||||||
@@ -51,7 +55,10 @@ export default function AboutCell({ text, channelId }: { text: string | null; ch
|
|||||||
</span>
|
</span>
|
||||||
<PopoverSurface
|
<PopoverSurface
|
||||||
popover={pop}
|
popover={pop}
|
||||||
className="glass w-[480px] max-h-112 overflow-y-auto px-3.5 py-3 rounded-lg animate-[fadeIn_0.12s_ease]"
|
className="glass w-[480px] overflow-y-auto px-3.5 py-3 rounded-lg animate-[fadeIn_0.12s_ease]"
|
||||||
|
// min(design cap, available space): keeps the 28rem cap on a tall viewport, shrinks to fit on a
|
||||||
|
// short one (high zoom). --fit-max-h is set by the fitHeight `size` middleware once positioned.
|
||||||
|
style={{ maxHeight: "min(28rem, var(--fit-max-h, 28rem))" }}
|
||||||
>
|
>
|
||||||
{/* Full About once the detail loads; until then the list's truncated description. */}
|
{/* Full About once the detail loads; until then the list's truncated description. */}
|
||||||
{ch ? (
|
{ch ? (
|
||||||
|
|||||||
@@ -131,7 +131,14 @@ export function useAnchoredPopover(opts: AnchoredPopoverOptions) {
|
|||||||
size({
|
size({
|
||||||
padding: 8,
|
padding: 8,
|
||||||
apply({ availableHeight, elements }) {
|
apply({ availableHeight, elements }) {
|
||||||
|
// Cap the positioned element, AND publish the available space as a custom property so
|
||||||
|
// the INNER styled/scroll wrapper (PopoverSurface renders `className` on a child, not on
|
||||||
|
// the floating element) can cap itself too — e.g. `maxHeight: min(28rem, var(--fit-max-h))`
|
||||||
|
// keeps a caller's design cap while still shrinking to fit at high zoom / short viewports.
|
||||||
|
// Capping only the outer element would leave the inner overflow-y-auto box at its fixed
|
||||||
|
// height, spilling its top off-screen (the whole point of fitHeight).
|
||||||
elements.floating.style.maxHeight = `${availableHeight}px`;
|
elements.floating.style.maxHeight = `${availableHeight}px`;
|
||||||
|
elements.floating.style.setProperty("--fit-max-h", `${availableHeight}px`);
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -84,10 +84,13 @@ function StatusPill({ job }: { job: DownloadJob }) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Thumb({ job }: { job: DownloadJob }) {
|
function Thumb({ job }: { job: DownloadJob }) {
|
||||||
// Prefer the source thumbnail; fall back to our generated poster for thumbnail-less sources.
|
// Prefer OUR self-hosted poster: it's always present on disk, never expires, and is same-origin, so
|
||||||
const primary = job.asset?.thumbnail_url || job.poster_url;
|
// a source-CDN signature expiry (Facebook's fbcdn URLs expire within days) or an ad-blocker/Shields
|
||||||
// A remote i.ytimg.com thumbnail 404s once the source video is deleted/privated, leaving a
|
// rule on the source domain can't break it. The source's remote thumbnail_url is only a fallback for
|
||||||
// broken image. On load failure fall back to our always-present self-hosted poster.
|
// the rare poster-less asset (a thumbnail-less source whose ffmpeg frame-cut also failed).
|
||||||
|
const remote = job.asset?.thumbnail_url ?? null;
|
||||||
|
const primary = job.poster_url || remote;
|
||||||
|
// On load failure (a genuinely missing poster file) fall back to the remote source thumbnail.
|
||||||
const [src, setSrc] = useState(primary);
|
const [src, setSrc] = useState(primary);
|
||||||
useEffect(() => setSrc(primary), [primary]);
|
useEffect(() => setSrc(primary), [primary]);
|
||||||
return (
|
return (
|
||||||
@@ -99,7 +102,7 @@ function Thumb({ job }: { job: DownloadJob }) {
|
|||||||
loading="lazy"
|
loading="lazy"
|
||||||
className="w-full h-full object-cover"
|
className="w-full h-full object-cover"
|
||||||
onError={() => {
|
onError={() => {
|
||||||
if (job.poster_url && src !== job.poster_url) setSrc(job.poster_url);
|
if (remote && src !== remote) setSrc(remote);
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
type ReactNode,
|
type ReactNode,
|
||||||
} from "react";
|
} from "react";
|
||||||
import { useVirtualizer } from "@tanstack/react-virtual";
|
import { useVirtualizer } from "@tanstack/react-virtual";
|
||||||
|
import { shouldPrefetchNext } from "./readAhead";
|
||||||
|
|
||||||
// The virtualization engine, lifted out of VirtualFeed so any long client- or server-side list can
|
// The virtualization engine, lifted out of VirtualFeed so any long client- or server-side list can
|
||||||
// reuse it (the feed's video grid, the channel manager's card/row views). It owns ONLY the layout
|
// reuse it (the feed's video grid, the channel manager's card/row views). It owns ONLY the layout
|
||||||
@@ -23,9 +24,6 @@ import { useVirtualizer } from "@tanstack/react-virtual";
|
|||||||
// (grid-cols-[repeat(auto-fill,minmax(260px,1fr))] with a gap-4 / 1rem gutter).
|
// (grid-cols-[repeat(auto-fill,minmax(260px,1fr))] with a gap-4 / 1rem gutter).
|
||||||
const GRID_GAP = 16;
|
const GRID_GAP = 16;
|
||||||
const LIST_GAP = 4;
|
const LIST_GAP = 4;
|
||||||
// Start fetching the next page this many rows before the end scrolls into view
|
|
||||||
// (keeps the old IntersectionObserver's generous prefetch feel).
|
|
||||||
const PREFETCH_ROWS = 4;
|
|
||||||
|
|
||||||
/** Columns that fit `width`, or 1 for the single-column layout (`minCol == null`). */
|
/** Columns that fit `width`, or 1 for the single-column layout (`minCol == null`). */
|
||||||
function columnsFor(minCol: number | null, width: number): number {
|
function columnsFor(minCol: number | null, width: number): number {
|
||||||
@@ -140,13 +138,17 @@ export default function VirtualGrid<T>({
|
|||||||
|
|
||||||
const virtualRows = virtualizer.getVirtualItems();
|
const virtualRows = virtualizer.getVirtualItems();
|
||||||
|
|
||||||
// Infinite-scroll trigger: once the last rendered row is within PREFETCH_ROWS of the end, pull
|
// Infinite-scroll read-ahead: pull the next page once the last rendered row is within the
|
||||||
// the next page. A no-op for finite lists (no fetchNextPage / hasNextPage false).
|
// read-ahead window of the loaded end (see readAhead.ts — the window scales with the rendered row
|
||||||
|
// count, clamped so a tall window doesn't prefetch unboundedly). A no-op for finite lists (no
|
||||||
|
// fetchNextPage / hasNextPage false). Because this effect re-runs when a page settles
|
||||||
|
// (isFetchingNextPage flips false) it CHAINS — refilling the buffer as the user consumes it, so a
|
||||||
|
// fast scroll reaches the loaded end far less often.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!fetchNextPage) return;
|
if (!fetchNextPage || !hasNextPage || isFetchingNextPage) return;
|
||||||
const last = virtualRows[virtualRows.length - 1];
|
const last = virtualRows[virtualRows.length - 1];
|
||||||
if (!last) return;
|
if (!last) return;
|
||||||
if (last.index >= rows.length - 1 - PREFETCH_ROWS && hasNextPage && !isFetchingNextPage) {
|
if (shouldPrefetchNext(last.index, rows.length, virtualRows.length)) {
|
||||||
fetchNextPage();
|
fetchNextPage();
|
||||||
}
|
}
|
||||||
}, [virtualRows, rows.length, hasNextPage, isFetchingNextPage, fetchNextPage]);
|
}, [virtualRows, rows.length, hasNextPage, isFetchingNextPage, fetchNextPage]);
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
MAX_LOOKAHEAD_ROWS,
|
||||||
|
PREFETCH_ROWS,
|
||||||
|
PREFETCH_VIEWPORTS,
|
||||||
|
readAheadLookahead,
|
||||||
|
shouldPrefetchNext,
|
||||||
|
} from "./readAhead";
|
||||||
|
|
||||||
|
describe("readAheadLookahead", () => {
|
||||||
|
it("floors at PREFETCH_ROWS when nothing (or little) is rendered", () => {
|
||||||
|
expect(readAheadLookahead(0)).toBe(PREFETCH_ROWS);
|
||||||
|
expect(readAheadLookahead(1)).toBe(PREFETCH_ROWS); // 1*2=2 < floor 4
|
||||||
|
});
|
||||||
|
|
||||||
|
it("scales with the rendered row count for a normal viewport", () => {
|
||||||
|
expect(readAheadLookahead(10)).toBe(10 * PREFETCH_VIEWPORTS); // 20
|
||||||
|
expect(readAheadLookahead(15)).toBe(30);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("caps at MAX_LOOKAHEAD_ROWS so a tall window doesn't prefetch unboundedly", () => {
|
||||||
|
// 30 rendered rows would be 60 uncapped — the cap is what bounds per-mount feed prefetch.
|
||||||
|
expect(readAheadLookahead(30)).toBe(MAX_LOOKAHEAD_ROWS);
|
||||||
|
expect(readAheadLookahead(1000)).toBe(MAX_LOOKAHEAD_ROWS);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("shouldPrefetchNext", () => {
|
||||||
|
it("fetches once the last rendered row enters the read-ahead window", () => {
|
||||||
|
// 100 rows loaded, viewport renders 10 → lookahead 20. Fetch when lastIndex >= 100-1-20 = 79.
|
||||||
|
expect(shouldPrefetchNext(78, 100, 10)).toBe(false);
|
||||||
|
expect(shouldPrefetchNext(79, 100, 10)).toBe(true);
|
||||||
|
expect(shouldPrefetchNext(99, 100, 10)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fetches eagerly while fewer rows are loaded than the lookahead (early pages)", () => {
|
||||||
|
// Only 15 rows loaded, lookahead 20 → the whole list is inside the window.
|
||||||
|
expect(shouldPrefetchNext(8, 15, 10)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not fetch near the top of a long loaded list", () => {
|
||||||
|
expect(shouldPrefetchNext(12, 500, 10)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
// Infinite-scroll read-ahead policy for VirtualGrid, extracted as a pure function so the decision is
|
||||||
|
// unit-testable without a layout engine.
|
||||||
|
//
|
||||||
|
// The next page is pulled once the last rendered row comes within `lookahead` rows of the loaded end,
|
||||||
|
// where lookahead scales with the number of rows the viewport currently renders (so it adapts to
|
||||||
|
// window height / column count / row height) but is clamped to a floor and a ceiling:
|
||||||
|
// - floor PREFETCH_ROWS: the finite-list / first-paint case where nothing is rendered yet.
|
||||||
|
// - ceiling MAX_LOOKAHEAD_ROWS: a tall window renders many rows, and an uncapped lookahead would
|
||||||
|
// eagerly prefetch several pages on every mount / filter change — wasted work on the app's
|
||||||
|
// heaviest query (the feed). The cap bounds that while still buffering a couple of screens ahead.
|
||||||
|
export const PREFETCH_ROWS = 4;
|
||||||
|
export const PREFETCH_VIEWPORTS = 2;
|
||||||
|
export const MAX_LOOKAHEAD_ROWS = 40;
|
||||||
|
|
||||||
|
/** How many rows ahead of the last rendered row to keep buffered, for a viewport rendering
|
||||||
|
* `renderedRows` rows. Clamped to [PREFETCH_ROWS, MAX_LOOKAHEAD_ROWS]. */
|
||||||
|
export function readAheadLookahead(renderedRows: number): number {
|
||||||
|
return Math.min(MAX_LOOKAHEAD_ROWS, Math.max(PREFETCH_ROWS, renderedRows * PREFETCH_VIEWPORTS));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether to fetch the next page: true once the last rendered row (`lastIndex`) is within the
|
||||||
|
* read-ahead window of the loaded end (`rowCount` total loaded rows). Because the caller re-runs
|
||||||
|
* this as each page settles, it chains — refilling the buffer as the user consumes it. */
|
||||||
|
export function shouldPrefetchNext(
|
||||||
|
lastIndex: number,
|
||||||
|
rowCount: number,
|
||||||
|
renderedRows: number,
|
||||||
|
): boolean {
|
||||||
|
return lastIndex >= rowCount - 1 - readAheadLookahead(renderedRows);
|
||||||
|
}
|
||||||
@@ -14,6 +14,18 @@ export interface ReleaseEntry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const RELEASE_NOTES: ReleaseEntry[] = [
|
export const RELEASE_NOTES: ReleaseEntry[] = [
|
||||||
|
{
|
||||||
|
version: "0.69.1",
|
||||||
|
date: "2026-08-19",
|
||||||
|
summary:
|
||||||
|
"Smoother fast-scrolling in the feed, reliable download thumbnails, and a tidier channel About popover.",
|
||||||
|
fixes: [
|
||||||
|
"Download thumbnails are reliable again. The Library now shows the copy saved on the server instead of the source's own link — which for Facebook videos expires within days (and can be blocked by a browser tracker shield), leaving a blank thumbnail even though the image was on disk the whole time.",
|
||||||
|
"Fast-scrolling the feed no longer stalls. A quick flick of the scroll wheel used to outrun the loader and leave you waiting several seconds on “Load more”; the feed now keeps a couple of screens of videos loaded ahead, so the content is usually already there when you reach it.",
|
||||||
|
"The channel About popover stays on screen at higher browser zoom. A long description could push the top of the box above the edge of the window where you couldn't read it; it now shrinks to fit the available height and scrolls inside.",
|
||||||
|
"Channel descriptions are tidier: the runs of blank lines YouTube pads its contact/link blocks with are stripped, so the About box is more compact.",
|
||||||
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
version: "0.69.0",
|
version: "0.69.0",
|
||||||
date: "2026-08-19",
|
date: "2026-08-19",
|
||||||
|
|||||||
Reference in New Issue
Block a user