peter
1f010328df
int() refuses a decimal run past 4300 digits, so one such reference in a summary raised out of the sync and the text migration. Any number past seven significant digits is beyond U+10FFFF, so it becomes U+FFFD like any out-of-range reference; leading zeros are dropped first. The rails test's comment now says what it guards: both rails agree with the rollup.
52 lines
2.3 KiB
Python
52 lines
2.3 KiB
Python
"""Plex text fields as plain text.
|
|
|
|
Plex hands a summary over as the agent wrote it, and some agents (a Hungarian TV listing, for one)
|
|
write HTML: `<p>…</p>` paragraphs and `á`-style entities. The app renders summaries as text,
|
|
so the tags showed literally on the info page and the hover card, and a card's 300-character cut
|
|
could land inside one. Every summary and tagline is stored through `plain_text`, and the migration
|
|
that cleaned the existing rows calls it too, so the two cannot disagree.
|
|
"""
|
|
|
|
import html
|
|
import re
|
|
from html.entities import html5
|
|
|
|
_BREAK = re.compile(r"<br\s*/?>", re.IGNORECASE)
|
|
# A paragraph or block boundary: a blank line between the two. Only a tag name ends the match, so a
|
|
# bare "<" in prose ("a < b", "<3") is left alone.
|
|
_BLOCK = re.compile(r"</?(?:p|div|li|ul|ol|h[1-6]|blockquote)\b[^>]*>", re.IGNORECASE)
|
|
_TAG = re.compile(r"</?[a-zA-Z][^>]*>")
|
|
_BLANKS = re.compile(r"[ \t]*\n[ \t]*\n\s*")
|
|
|
|
# A well-formed character reference only. `html.unescape` alone also decodes the legacy forms that
|
|
# have no semicolon, so prose like "AT¬ice" or "R©" came out as "AT¬ice" and "R©". A named
|
|
# one counts only when its whole name is an entity: for "¬ice;" `html.unescape` still falls back
|
|
# to the legacy "¬" inside it.
|
|
_REF = re.compile(r"&(?:#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);")
|
|
|
|
|
|
def _decode(m: re.Match[str]) -> str:
|
|
ref = m.group()
|
|
if ref[1] != "#":
|
|
return html.unescape(ref) if ref[1:] in html5 else ref
|
|
if ref[2] in "xX":
|
|
return html.unescape(ref)
|
|
# `int()` refuses a decimal run past 4300 digits, and any number past seven significant digits is
|
|
# beyond U+10FFFF, which the parser turns into U+FFFD anyway.
|
|
digits = ref[2:-1].lstrip("0") or "0"
|
|
return "�" if len(digits) > 7 else html.unescape(f"&#{digits};")
|
|
|
|
|
|
def plain_text(value: str | None) -> str | None:
|
|
"""`value` with its markup gone: line breaks kept as newlines, paragraphs as a blank line,
|
|
every other tag dropped and entities decoded. None or an all-blank result stays None."""
|
|
if value is None:
|
|
return None
|
|
if "<" in value or "&" in value:
|
|
value = _BREAK.sub("\n", value)
|
|
value = _BLOCK.sub("\n\n", value)
|
|
value = _TAG.sub("", value)
|
|
value = _REF.sub(_decode, value)
|
|
value = _BLANKS.sub("\n\n", value)
|
|
return value.strip() or None
|