Files
khatibi/tests/test_customer_name_fix.py
2026-10-21 10:50:42 +03:30

479 lines
22 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""تست‌های قابلیت «افزودن پیشوند آقای/خانم به نام مشتریان».
همهٔ تست‌ها روی دیتابیس تستی Django اجرا می‌شوند (SQLite در حافظه) و هیچ‌گاه به
``data9/kt.sqlite3`` (دیتابیس واقعی) دست نمی‌زنند. فایل دادهٔ واقعی هم دست‌نخورده
می‌ماند؛ هر تست روی یک فایل JSON ساختگیِ موقت کار می‌کند.
"""
import hashlib
import io
import itertools
import json
import shutil
import tempfile
from datetime import timedelta
from pathlib import Path
from unittest import mock
from django.core.management import call_command
from django.core.management.base import CommandError
from django.db import connections
from django.db.models.query import QuerySet
from django.db.utils import OperationalError
from django.test import SimpleTestCase, TestCase
from django.utils import timezone
from admin_panel.models import Customer, CustomerNameFixJob
from admin_panel.name_fix import (ALGORITHM_VERSION, DatasetError,
build_name_index, decide_prefix, load_name_index,
normalize_name_key)
from admin_panel.management.commands.fix_customer_names import (LOCK_TTL,
MAX_DB_RETRIES)
from .factories import make_customer
DATASET_RECORDS = [
{"Names": "محمد", "Gender": "M"},
{"Names": "مریم", "Gender": "F"},
{"Names": "علی", "Gender": "M"},
{"Names": "علي", "Gender": "M"}, # ي عربی؛ همان کلید نرمال‌شده
{"Names": "كاظم", "Gender": "M"}, # ك عربی
{"Names": "کاظم", "Gender": "M"}, # ک فارسی (همان کلید)
{"Names": "سارا", "Gender": "F"},
{"Names": "سارا", "Gender": "F"}, # تکراری هم‌جنسیت
{"Names": "آرش", "Gender": "M"},
{"Names": "آرش", "Gender": "F"}, # تکراری متناقض ⇒ مبهم
{"Names": "نازنین", "Gender": "D"}, # جنسیت نامعتبر
{"Names": 123, "Gender": "M"}, # رکورد malformed
{"Names": "", "Gender": "M"}, # نام خالی
{"Names": "بی‌جنس"}, # کلید Gender ندارد
]
def sha256_of(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
class DatasetFixtureMixin:
"""ساخت فایل JSON موقت و پاک‌سازی آن پس از تست."""
def setUp(self):
super().setUp()
self.tmpdir = tempfile.mkdtemp(prefix="namefix-")
self.addCleanup(shutil.rmtree, self.tmpdir, ignore_errors=True)
def write_dataset(self, records=DATASET_RECORDS, name="iranianNamesDataset.json"):
path = Path(self.tmpdir) / name
path.write_text(json.dumps({"RECORDS": records}, ensure_ascii=False),
encoding="utf-8")
return path
class NameIndexTests(DatasetFixtureMixin, SimpleTestCase):
"""اعتبارسنجی فایل، نرمال‌سازی و ساخت index."""
def test_load_builds_conservative_index_and_report(self):
path = self.write_dataset()
name_index = load_name_index(path)
self.assertEqual(name_index.sha256, sha256_of(path))
self.assertEqual(name_index.lookup("محمد"), "M")
self.assertEqual(name_index.lookup("مریم"), "F")
# ي/ی و ك/ک یکسان‌سازی می‌شوند.
self.assertEqual(name_index.lookup("علی"), "M")
self.assertEqual(name_index.lookup("کاظم"), "M")
# تکراری متناقض و جنسیت نامعتبر کنار گذاشته می‌شوند.
self.assertIsNone(name_index.lookup("آرش"))
self.assertIsNone(name_index.lookup("نازنین"))
rep = name_index.report
self.assertEqual(rep["records_total"], 14)
self.assertEqual(rep["rejected_malformed"], 2)
self.assertEqual(rep["rejected_empty_name"], 1)
self.assertEqual(rep["rejected_unknown_gender"], 1)
self.assertEqual(rep["usable_keys"], 5)
self.assertEqual(rep["unusable_keys"], 2)
self.assertEqual(rep["duplicate_keys"], 4)
self.assertEqual(rep["duplicate_same_gender"], 3)
self.assertEqual(rep["duplicate_conflict"], 1)
def test_normalization_unifies_arabic_and_persian_letters(self):
self.assertEqual(normalize_name_key(" علي "), "علی")
self.assertEqual(normalize_name_key("كاظم"), "کاظم")
self.assertEqual(normalize_name_key("محمد"), "محمد")
self.assertEqual(normalize_name_key(None), "")
def test_missing_records_key_is_rejected(self):
path = Path(self.tmpdir) / "bad.json"
path.write_text(json.dumps({"DATA": []}), encoding="utf-8")
with self.assertRaises(DatasetError):
load_name_index(path)
def test_records_must_be_a_list(self):
path = Path(self.tmpdir) / "bad.json"
path.write_text(json.dumps({"RECORDS": {"Names": "x"}}), encoding="utf-8")
with self.assertRaises(DatasetError):
load_name_index(path)
def test_invalid_json_is_rejected(self):
path = Path(self.tmpdir) / "bad.json"
path.write_text("{not json", encoding="utf-8")
with self.assertRaises(DatasetError):
load_name_index(path)
def test_normalization_collision_with_conflicting_gender_is_ambiguous(self):
name_index = build_name_index([
{"Names": "علي", "Gender": "M"},
{"Names": "علی", "Gender": "F"}, # پس از نرمال‌سازی همان کلید
])
self.assertIsNone(name_index.lookup("علی"))
self.assertIn("علی", name_index.ambiguous)
class DecidePrefixTests(DatasetFixtureMixin, SimpleTestCase):
"""الگوریتم تصمیم، مستقل از دیتابیس."""
def setUp(self):
super().setUp()
self.name_index = load_name_index(self.write_dataset())
def test_expected_behaviour_table(self):
cases = [
("محمد یاوری", "آقای محمد یاوری", "male"),
("مریم احمدی", "خانم مریم احمدی", "female"),
(" محمد یاوری ", "آقای محمد یاوری", "male"),
("\tمریم احمدی\n", "خانم مریم احمدی", "female"),
("یاوری", None, "single_word"),
("محمد", None, "single_word"),
("مریم", None, "single_word"),
("آقا محمد یاوری", None, "has_prefix"),
("آقای محمد یاوری", None, "has_prefix"),
("خانم مریم احمدی", None, "has_prefix"),
("نام ناشناخته احمدی", None, "unknown_name"),
("آرش محمدی", None, "ambiguous_gender"),
("نازنین کریمی", None, "ambiguous_gender"),
("علي رضایی", "آقای علي رضایی", "male"), # نام اصلی دست‌نخورده می‌ماند
("", None, "empty"),
(" ", None, "empty"),
]
for raw, expected_new, expected_reason in cases:
decision = decide_prefix(raw, self.name_index)
self.assertEqual(decision.new_name, expected_new, msg=repr(raw))
self.assertEqual(decision.reason, expected_reason, msg=repr(raw))
def test_none_value_is_handled(self):
decision = decide_prefix(None, self.name_index)
self.assertIsNone(decision.new_name)
self.assertEqual(decision.reason, "empty")
def test_internal_spaces_are_not_rewritten(self):
decision = decide_prefix("محمد یاوری", self.name_index)
self.assertEqual(decision.new_name, "آقای محمد یاوری")
# (نام اولیه، نتیجهٔ مورد انتظار) — ترتیب = ترتیب ساخت ⇒ ترتیب pk
CASES = [
("محمد یاوری", "آقای محمد یاوری"),
("مریم احمدی", "خانم مریم احمدی"),
(" محمد یاوری ", "آقای محمد یاوری"),
("یاوری", "یاوری"),
("محمد", "محمد"),
("مریم", "مریم"),
("آقا محمد یاوری", "آقا محمد یاوری"),
("آقای محمد یاوری", "آقای محمد یاوری"),
("خانم مریم احمدی", "خانم مریم احمدی"),
("نام ناشناخته احمدی", "نام ناشناخته احمدی"),
("آرش محمدی", "آرش محمدی"),
("نازنین کریمی", "نازنین کریمی"),
("علي رضایی", "آقای علي رضایی"),
(" نام ناشناخته احمدی ", " نام ناشناخته احمدی "),
]
EXPECTED_CHANGED = 4 # موارد ۱، ۲، ۳، ۱۳
EXPECTED_SINGLE_WORD = 3 # یاوری، محمد، مریم
EXPECTED_HAS_PREFIX = 3
EXPECTED_UNKNOWN = 2 # «نام ناشناخته …» دو مورد (با/بی فاصله)
EXPECTED_AMBIGUOUS = 2 # آرش، نازنین
class CommandTestMixin(DatasetFixtureMixin):
"""ساخت مشتریان نمونه + فایل دادهٔ موقت برای تست‌های فرمان."""
def setUp(self):
super().setUp()
self.json_path = self.write_dataset()
self.customers = [make_customer(name) for name, _ in CASES]
def run_command(self, **options):
options.setdefault("json_path", str(self.json_path))
options.setdefault("stdout", self._stdout())
return call_command("fix_customer_names", **options)
def _stdout(self):
return io.StringIO()
def names(self):
return list(Customer.objects.order_by("pk")
.values_list("name_family", flat=True))
def expected_names(self):
return [expected for _, expected in CASES]
def max_pk(self):
return max(c.pk for c in self.customers)
class DryRunTests(CommandTestMixin, TestCase):
"""پیش‌فرض فرمان نباید هیچ‌چیز بنویسد."""
def test_dry_run_changes_nothing(self):
out = self._stdout()
self.run_command(stdout=out)
self.assertEqual(self.names(), [name for name, _ in CASES])
self.assertEqual(CustomerNameFixJob.objects.count(), 0)
self.assertIn("dry-run", out.getvalue())
def test_dry_run_flag_conflicts_with_apply(self):
with self.assertRaises(CommandError):
self.run_command(dry_run=True, apply=True, confirm_production=True)
self.assertEqual(self.names(), [name for name, _ in CASES])
def test_dry_run_reports_expected_counts(self):
out = self._stdout()
self.run_command(stdout=out)
text = out.getvalue()
self.assertIn(f"مشتریان بررسی‌شده: {len(CASES)}", text)
self.assertIn(f"قابل اصلاح / اصلاح‌شده: {EXPECTED_CHANGED}", text)
self.assertIn(f"تک‌کلمه‌ای: {EXPECTED_SINGLE_WORD}", text)
self.assertIn(f"ناشناخته در JSON: {EXPECTED_UNKNOWN}", text)
self.assertIn(f"جنسیت مبهم/نامعتبر: {EXPECTED_AMBIGUOUS}", text)
# نمونهٔ قبل/بعد نمایش داده می‌شود.
self.assertIn("«محمد یاوری» → «آقای محمد یاوری»", text)
class ApplyTests(CommandTestMixin, TestCase):
"""اجرای واقعی: گارد تأیید، تغییر تنها مشتریان واجد شرایط، اتمیک بودن."""
APPLY = {"apply": True, "confirm_production": True}
def test_apply_without_confirm_production_is_refused(self):
with self.assertRaises(CommandError):
self.run_command(apply=True)
self.assertEqual(self.names(), [name for name, _ in CASES])
self.assertEqual(CustomerNameFixJob.objects.count(), 0)
def test_apply_changes_only_eligible_names(self):
self.run_command(batch_size=3, **self.APPLY)
self.assertEqual(self.names(), self.expected_names())
job = CustomerNameFixJob.objects.get()
self.assertEqual(job.status, CustomerNameFixJob.Status.COMPLETED)
self.assertEqual(job.algorithm_version, ALGORITHM_VERSION)
self.assertEqual(job.source_hash, sha256_of(self.json_path))
self.assertEqual(job.initial_max_pk, self.max_pk())
self.assertEqual(job.last_processed_pk, self.max_pk())
self.assertEqual(job.stats["scanned"], len(CASES))
self.assertEqual(job.stats["changed"], EXPECTED_CHANGED)
self.assertEqual(job.stats["single_word"], EXPECTED_SINGLE_WORD)
self.assertEqual(job.stats["has_prefix"], EXPECTED_HAS_PREFIX)
self.assertEqual(job.stats["unknown_name"], EXPECTED_UNKNOWN)
self.assertEqual(job.stats["ambiguous_gender"], EXPECTED_AMBIGUOUS)
self.assertEqual(job.stats["batches"], 5) # ۱۴ مشتری، batch=3
def test_other_customer_fields_and_json_file_remain_untouched(self):
target = self.customers[0]
Customer.objects.filter(pk=target.pk).update(
address="تهران، خیابان تست", phone1="09120000000",
notes="یادداشت مهم", is_active=False)
before_hash = sha256_of(self.json_path)
self.run_command(**self.APPLY)
target.refresh_from_db()
self.assertEqual(target.name_family, "آقای محمد یاوری")
self.assertEqual(target.address, "تهران، خیابان تست")
self.assertEqual(target.phone1, "09120000000")
self.assertEqual(target.notes, "یادداشت مهم")
self.assertFalse(target.is_active)
self.assertEqual(sha256_of(self.json_path), before_hash)
def test_customer_model_has_no_gender_field(self):
# فقط فیلدهای واقعی ستون‌دار مدل (بدون روابط معکوس).
field_names = {f.name for f in Customer._meta.concrete_fields}
self.assertEqual(field_names, {
"id", "name_family", "address", "notes", "phone1", "phone2",
"phone3", "idCode", "installDate", "create_date", "last_bazdid_dt",
"laghv_bazdid", "is_active", "sms_primary_phones",
# «قرار بعدی» (تعویق بازدید از فرم هماهنگی) + زمان ثبت برای حسابرسی.
"visit_hold_until", "visit_hold_set_at",
})
self.assertNotIn("gender", field_names)
def test_rerun_of_completed_job_requires_new_job(self):
self.run_command(**self.APPLY)
with self.assertRaises(CommandError):
self.run_command(**self.APPLY)
self.assertEqual(self.names(), self.expected_names())
self.assertEqual(CustomerNameFixJob.objects.count(), 1)
def test_new_job_rerun_is_idempotent(self):
self.run_command(**self.APPLY)
self.run_command(new_job=True, **self.APPLY)
# پیشوندهای موجود تکرار نشده‌اند.
self.assertEqual(self.names(), self.expected_names())
self.assertEqual(CustomerNameFixJob.objects.count(), 2)
self.assertEqual(CustomerNameFixJob.objects.latest("pk").stats["changed"], 0)
def test_missing_json_path_is_reported(self):
with self.assertRaises(CommandError):
self.run_command(json_path=str(Path(self.tmpdir) / "nope.json"),
apply=True, confirm_production=True)
self.assertEqual(self.names(), [name for name, _ in CASES])
def test_invalid_dataset_aborts_before_any_change(self):
bad = Path(self.tmpdir) / "bad.json"
bad.write_text("{oops", encoding="utf-8")
with self.assertRaises(CommandError):
self.run_command(json_path=str(bad), apply=True, confirm_production=True)
self.assertEqual(self.names(), [name for name, _ in CASES])
self.assertEqual(CustomerNameFixJob.objects.count(), 0)
def test_missing_job_table_is_reported(self):
with mock.patch.object(connections["default"].introspection,
"table_names", return_value=[]):
with self.assertRaises(CommandError):
self.run_command(**self.APPLY)
self.assertEqual(self.names(), [name for name, _ in CASES])
class ResilienceTests(CommandTestMixin, TestCase):
"""قفل، بازیابی پس از crash، hash، اتمیک بودن و retry قفل دیتابیس."""
APPLY = {"apply": True, "confirm_production": True}
def _job(self, **overrides):
values = {
"source_hash": sha256_of(self.json_path),
"algorithm_version": ALGORITHM_VERSION,
"status": CustomerNameFixJob.Status.PAUSED,
"batch_size": 3,
"initial_max_pk": self.max_pk(),
"last_processed_pk": 0,
"stats": {},
"heartbeat_at": timezone.now(),
}
values.update(overrides)
return CustomerNameFixJob.objects.create(**values)
def test_concurrent_run_is_blocked(self):
self._job(status=CustomerNameFixJob.Status.RUNNING,
heartbeat_at=timezone.now())
with self.assertRaises(CommandError):
self.run_command(**self.APPLY)
self.assertEqual(self.names(), [name for name, _ in CASES])
def test_stale_running_lock_is_recovered_and_resumed(self):
stale = self._job(status=CustomerNameFixJob.Status.RUNNING,
heartbeat_at=timezone.now() - LOCK_TTL - timedelta(minutes=1))
self.run_command(**self.APPLY)
stale.refresh_from_db()
self.assertEqual(stale.status, CustomerNameFixJob.Status.COMPLETED)
self.assertEqual(self.names(), self.expected_names())
def test_resume_continues_from_checkpoint(self):
first_pk = self.customers[0].pk
job = self._job(last_processed_pk=first_pk)
self.run_command(resume=job.pk, **self.APPLY)
names = self.names()
# مشتری قبل از checkpoint دست‌نخورده می‌ماند
self.assertEqual(names[0], "محمد یاوری")
self.assertEqual(names[1:], self.expected_names()[1:])
job.refresh_from_db()
self.assertEqual(job.status, CustomerNameFixJob.Status.COMPLETED)
self.assertEqual(job.last_processed_pk, self.max_pk())
self.assertEqual(job.stats["scanned"], len(CASES) - 1)
def test_resume_with_different_hash_is_refused(self):
job = self._job(source_hash="0" * 64)
with self.assertRaises(CommandError):
self.run_command(resume=job.pk, **self.APPLY)
self.assertEqual(self.names(), [name for name, _ in CASES])
job.refresh_from_db()
self.assertEqual(job.status, CustomerNameFixJob.Status.PAUSED)
def test_unknown_resume_id_is_refused(self):
with self.assertRaises(CommandError):
self.run_command(resume=999999, **self.APPLY)
self.assertEqual(self.names(), [name for name, _ in CASES])
def test_batch_failure_rolls_back_names_and_checkpoint_atomically(self):
original = QuerySet.bulk_update
calls = itertools.count(1)
def flaky(self, objs, fields, batch_size=None):
if next(calls) == 2:
raise RuntimeError("boom")
return original(self, objs, fields, batch_size)
with mock.patch.object(QuerySet, "bulk_update", flaky):
with self.assertRaises(CommandError):
self.run_command(batch_size=1, **self.APPLY)
names = self.names()
# batch اول (مشتری ۱) commit شده، batch دوم (مشتری ۲) rollback شده است.
self.assertEqual(names[0], "آقای محمد یاوری")
self.assertEqual(names[1], "مریم احمدی")
job = CustomerNameFixJob.objects.get()
self.assertEqual(job.status, CustomerNameFixJob.Status.FAILED)
# checkpoint دقیقاً روی پایان batch اول است ⇒ ادامه بدون تکرار/پرت شدن داده.
self.assertEqual(job.last_processed_pk, self.customers[0].pk)
def test_locked_database_is_retried_then_succeeds(self):
original = QuerySet.bulk_update
calls = itertools.count(1)
def flaky(self, objs, fields, batch_size=None):
if next(calls) == 1:
raise OperationalError("database is locked")
return original(self, objs, fields, batch_size)
with mock.patch.object(QuerySet, "bulk_update", flaky), \
mock.patch("admin_panel.management.commands.fix_customer_names.time.sleep"):
self.run_command(**self.APPLY)
self.assertEqual(self.names(), self.expected_names())
self.assertEqual(CustomerNameFixJob.objects.get().status,
CustomerNameFixJob.Status.COMPLETED)
def test_retry_is_bounded_and_surfaces_error(self):
calls = itertools.count(1)
attempts = []
def always_locked(self, objs, fields, batch_size=None):
attempts.append(next(calls))
raise OperationalError("database is locked")
with mock.patch.object(QuerySet, "bulk_update", always_locked), \
mock.patch("admin_panel.management.commands.fix_customer_names.time.sleep"):
with self.assertRaises(CommandError):
self.run_command(**self.APPLY)
self.assertEqual(len(attempts), MAX_DB_RETRIES) # retry نامحدود نیست
self.assertEqual(self.names(), [name for name, _ in CASES])
self.assertEqual(CustomerNameFixJob.objects.get().status,
CustomerNameFixJob.Status.FAILED)
def test_batches_cover_all_customers_across_sizes(self):
for batch_size in (1, 2, 5, 200):
with self.subTest(batch_size=batch_size):
CustomerNameFixJob.objects.all().delete()
for customer, (name, _) in zip(self.customers, CASES):
Customer.objects.filter(pk=customer.pk).update(name_family=name)
self.run_command(batch_size=batch_size, new_job=True, **self.APPLY)
self.assertEqual(self.names(), self.expected_names())