perf(search): a one- or two-letter last word is matched whole, not as a prefix
A 1-2 letter prefix matched most of the library and ts_rank scored every match: "m" took ~4-8 s on prod against ~150 ms now. The prefix match starts at three characters (MIN_PREFIX_LEN); shared by the YouTube feed and the Plex library search.
This commit is contained in:
+10
-2
@@ -60,16 +60,24 @@ def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# The shortest last word that is prefix-matched. A 1–2 letter prefix matches a large share of the
|
||||
# library and ts_rank then scores every match: "m" took ~5–8 s on prod against ~100 ms for a whole
|
||||
# word (E31 S4e). A shorter last word is matched whole instead ("tv", "4k" still find themselves).
|
||||
MIN_PREFIX_LEN = 3
|
||||
|
||||
|
||||
def to_tsquery_str(q: str) -> str | None:
|
||||
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
|
||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*" —
|
||||
once it has MIN_PREFIX_LEN characters ("ti amo m" → "ti & amo & m").
|
||||
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
||||
(the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed
|
||||
search and the Plex library search, hence this neutral home rather than either domain module."""
|
||||
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
||||
if not tokens:
|
||||
return None
|
||||
tokens[-1] = tokens[-1] + ":*"
|
||||
if len(tokens[-1]) >= MIN_PREFIX_LEN:
|
||||
tokens[-1] = tokens[-1] + ":*"
|
||||
return " & ".join(tokens)
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
||||
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a); to_tsquery_str()."""
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from app.utils import iso, normalize_description
|
||||
from app.utils import iso, normalize_description, to_tsquery_str
|
||||
|
||||
|
||||
def test_none_maps_to_none():
|
||||
@@ -56,3 +56,19 @@ def test_normalize_description_single_line_unchanged():
|
||||
def test_normalize_description_folds_crlf_line_endings():
|
||||
# splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization.
|
||||
assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc"
|
||||
|
||||
|
||||
def test_tsquery_prefix_matches_a_last_word_of_three_or_more():
|
||||
assert to_tsquery_str("ti amo magyar") == "ti & amo & magyar:*"
|
||||
assert to_tsquery_str("min") == "min:*"
|
||||
|
||||
|
||||
def test_tsquery_matches_a_short_last_word_whole():
|
||||
# A 1-2 letter prefix made ts_rank score most of the library (5-8 s on prod, E31 S4e).
|
||||
assert to_tsquery_str("m") == "m"
|
||||
assert to_tsquery_str("minecraft 4k") == "minecraft & 4k"
|
||||
assert to_tsquery_str("ti amo m") == "ti & amo & m"
|
||||
|
||||
|
||||
def test_tsquery_without_a_word_is_none():
|
||||
assert to_tsquery_str("?! --") is None
|
||||
|
||||
Reference in New Issue
Block a user