67 lines
2.0 KiB
Python
67 lines
2.0 KiB
Python
"""Safe, conservative Persian text normalization for human-authored fields."""
|
|
|
|
import re
|
|
from functools import lru_cache
|
|
|
|
from hazm import Normalizer
|
|
|
|
|
|
# Hazm's spacing rules are intended for prose and can add spaces inside URLs and
|
|
# technical identifiers. Temporarily replace those spans before normalization.
|
|
_PROTECTED_SPAN = re.compile(
|
|
r"https?://\S+|www\.\S+|[\w.+-]+@[\w.-]+\.\w+|"
|
|
r"(?<![\w\u0600-\u06ff])[A-Za-z0-9][A-Za-z0-9._:/?#%+\\=@-]*"
|
|
)
|
|
_TOKEN_PREFIX = "HAZMPROTECTEDTOKEN"
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _normalizer():
|
|
return Normalizer(
|
|
remove_diacritics=False,
|
|
remove_specials_chars=False,
|
|
decrease_repeated_chars=False,
|
|
persian_numbers=False,
|
|
)
|
|
|
|
|
|
def _alpha_index(index):
|
|
"""Return an ASCII-letter-only stable suffix (A, B, ..., AA)."""
|
|
result = ""
|
|
while True:
|
|
index, remainder = divmod(index, 26)
|
|
result = chr(65 + remainder) + result
|
|
if index == 0:
|
|
return result
|
|
index -= 1
|
|
|
|
|
|
def normalize_persian_text(value, *, multiline=False):
|
|
"""Normalize Persian orthography without rewriting meaning or technical data.
|
|
|
|
Multiline values retain paragraph boundaries. Empty/None values become an
|
|
empty string, matching the project's non-null textual model fields.
|
|
"""
|
|
if value is None:
|
|
return ""
|
|
text = str(value).replace("\r\n", "\n").replace("\r", "\n")
|
|
protected = {}
|
|
|
|
def protect(match):
|
|
token = f"{_TOKEN_PREFIX}{_alpha_index(len(protected))}"
|
|
protected[token] = match.group(0)
|
|
return token
|
|
|
|
text = _PROTECTED_SPAN.sub(protect, text)
|
|
if multiline:
|
|
normalized = "\n".join(
|
|
_normalizer().normalize(line).strip() if line.strip() else ""
|
|
for line in text.split("\n")
|
|
).strip()
|
|
else:
|
|
normalized = _normalizer().normalize(" ".join(text.splitlines())).strip()
|
|
|
|
for token, original in protected.items():
|
|
normalized = normalized.replace(token, original)
|
|
return normalized
|