perf(search): a one- or two-letter last word is matched whole, not as a prefix
A 1-2 letter prefix matched most of the library and ts_rank scored every match: "m" took ~4-8 s on prod against ~150 ms now. The prefix match starts at three characters (MIN_PREFIX_LEN); shared by the YouTube feed and the Plex library search.
This commit is contained in:
+10
-2
@@ -60,16 +60,24 @@ def now_utc() -> datetime:
|
|||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
# The shortest last word that is prefix-matched. A 1–2 letter prefix matches a large share of the
|
||||||
|
# library and ts_rank then scores every match: "m" took ~5–8 s on prod against ~100 ms for a whole
|
||||||
|
# word (E31 S4e). A shorter last word is matched whole instead ("tv", "4k" still find themselves).
|
||||||
|
MIN_PREFIX_LEN = 3
|
||||||
|
|
||||||
|
|
||||||
def to_tsquery_str(q: str) -> str | None:
|
def to_tsquery_str(q: str) -> str | None:
|
||||||
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
||||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
|
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*" —
|
||||||
|
once it has MIN_PREFIX_LEN characters ("ti amo m" → "ti & amo & m").
|
||||||
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
||||||
(the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed
|
(the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed
|
||||||
search and the Plex library search, hence this neutral home rather than either domain module."""
|
search and the Plex library search, hence this neutral home rather than either domain module."""
|
||||||
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
||||||
if not tokens:
|
if not tokens:
|
||||||
return None
|
return None
|
||||||
tokens[-1] = tokens[-1] + ":*"
|
if len(tokens[-1]) >= MIN_PREFIX_LEN:
|
||||||
|
tokens[-1] = tokens[-1] + ":*"
|
||||||
return " & ".join(tokens)
|
return " & ".join(tokens)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a); to_tsquery_str()."""
|
||||||
|
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
from app.utils import iso, normalize_description
|
from app.utils import iso, normalize_description, to_tsquery_str
|
||||||
|
|
||||||
|
|
||||||
def test_none_maps_to_none():
|
def test_none_maps_to_none():
|
||||||
@@ -56,3 +56,19 @@ def test_normalize_description_single_line_unchanged():
|
|||||||
def test_normalize_description_folds_crlf_line_endings():
|
def test_normalize_description_folds_crlf_line_endings():
|
||||||
# splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization.
|
# splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization.
|
||||||
assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc"
|
assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc"
|
||||||
|
|
||||||
|
|
||||||
|
def test_tsquery_prefix_matches_a_last_word_of_three_or_more():
|
||||||
|
assert to_tsquery_str("ti amo magyar") == "ti & amo & magyar:*"
|
||||||
|
assert to_tsquery_str("min") == "min:*"
|
||||||
|
|
||||||
|
|
||||||
|
def test_tsquery_matches_a_short_last_word_whole():
|
||||||
|
# A 1-2 letter prefix made ts_rank score most of the library (5-8 s on prod, E31 S4e).
|
||||||
|
assert to_tsquery_str("m") == "m"
|
||||||
|
assert to_tsquery_str("minecraft 4k") == "minecraft & 4k"
|
||||||
|
assert to_tsquery_str("ti amo m") == "ti & amo & m"
|
||||||
|
|
||||||
|
|
||||||
|
def test_tsquery_without_a_word_is_none():
|
||||||
|
assert to_tsquery_str("?! --") is None
|
||||||
|
|||||||
Reference in New Issue
Block a user