first

apps/core/persian_text.py

@@ -0,0 +1,66 @@
+"""Safe, conservative Persian text normalization for human-authored fields."""
+
+import re
+from functools import lru_cache
+
+from hazm import Normalizer
+
+
+# Hazm's spacing rules are intended for prose and can add spaces inside URLs and
+# technical identifiers.  Temporarily replace those spans before normalization.
+_PROTECTED_SPAN = re.compile(
+    r"https?://\S+|www\.\S+|[\w.+-]+@[\w.-]+\.\w+|"
+    r"(?<![\w\u0600-\u06ff])[A-Za-z0-9][A-Za-z0-9._:/?#%+\\=@-]*"
+)
+_TOKEN_PREFIX = "HAZMPROTECTEDTOKEN"
+
+
+@lru_cache(maxsize=1)
+def _normalizer():
+    return Normalizer(
+        remove_diacritics=False,
+        remove_specials_chars=False,
+        decrease_repeated_chars=False,
+        persian_numbers=False,
+    )
+
+
+def _alpha_index(index):
+    """Return an ASCII-letter-only stable suffix (A, B, ..., AA)."""
+    result = ""
+    while True:
+        index, remainder = divmod(index, 26)
+        result = chr(65 + remainder) + result
+        if index == 0:
+            return result
+        index -= 1
+
+
+def normalize_persian_text(value, *, multiline=False):
+    """Normalize Persian orthography without rewriting meaning or technical data.
+
+    Multiline values retain paragraph boundaries. Empty/None values become an
+    empty string, matching the project's non-null textual model fields.
+    """
+    if value is None:
+        return ""
+    text = str(value).replace("\r\n", "\n").replace("\r", "\n")
+    protected = {}
+
+    def protect(match):
+        token = f"{_TOKEN_PREFIX}{_alpha_index(len(protected))}"
+        protected[token] = match.group(0)
+        return token
+
+    text = _PROTECTED_SPAN.sub(protect, text)
+    if multiline:
+        normalized = "\n".join(
+            _normalizer().normalize(line).strip() if line.strip() else ""
+            for line in text.split("\n")
+        ).strip()
+    else:
+        normalized = _normalizer().normalize(" ".join(text.splitlines())).strip()
+
+    for token, original in protected.items():
+        normalized = normalized.replace(token, original)
+    return normalized