From 863ca89485d98ee6182642eaf70450727d6a7f88 Mon Sep 17 00:00:00 2001 From: javad Date: Wed, 16 Sep 2026 14:45:10 +0330 Subject: [PATCH] some fix sms retry --- .../commands/process_notifications.py | 20 ++- ...014_backfill_safe_0098_delivery_retries.py | 78 ++++++++++++ tests/test_domain.py | 116 +++++++++++++++++- 3 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 apps/notifications/migrations/0014_backfill_safe_0098_delivery_retries.py diff --git a/apps/notifications/management/commands/process_notifications.py b/apps/notifications/management/commands/process_notifications.py index 93aba2e..84bd0ce 100644 --- a/apps/notifications/management/commands/process_notifications.py +++ b/apps/notifications/management/commands/process_notifications.py @@ -20,9 +20,10 @@ from apps.tickets.models import Ticket logger = logging.getLogger(__name__) DEFINITIVE_DELIVERY_RETRY_EVENT = "STAFF_TICKET_CREATED" -DEFINITIVE_DELIVERY_RETRY_STATUSES = { - NotificationJob.DeliveryStatus.UNDELIVERED, - NotificationJob.DeliveryStatus.EXPIRED, +SAFE_0098_DELIVERY_RETRY_RESULTS = { + (NotificationJob.DeliveryStatus.UNDELIVERED, "6"), + (NotificationJob.DeliveryStatus.EXPIRED, "7"), + (NotificationJob.DeliveryStatus.ERROR, "-31"), } def _magic_link(job, reason): @@ -175,6 +176,10 @@ def _delivery_retry_dedupe_key(job): return f"{job.dedupe_key[:200-len(suffix)]}{suffix}" +def _is_safe_0098_delivery_retry_result(job): + return (job.delivery_status, (job.delivery_status_code or "").strip()) in SAFE_0098_DELIVERY_RETRY_RESULTS + + def _schedule_definitive_delivery_retry(job): """Schedule exactly one delayed resend for a definitive individual-ticket failure.""" if ( @@ -184,7 +189,7 @@ def _schedule_definitive_delivery_retry(job): or not job.ticket_id or job.ticket.group_dispatch_id or not job.delivery_is_final - or job.delivery_status not in DEFINITIVE_DELIVERY_RETRY_STATUSES + or not _is_safe_0098_delivery_retry_result(job) ): return None sent_at = job.sent_at or job.created_at @@ -268,6 +273,13 @@ def _process_delivery_checks(options): else: job.delivery_status = result.status if result.status in NotificationJob.DeliveryStatus.values else NotificationJob.DeliveryStatus.ERROR job.delivery_next_check_at = None + # 0098SMS code 6 remains provisional during polling. Once the + # complete monitoring window is exhausted it is safe to treat + # this exact result as terminal for the single delayed resend. + job.delivery_is_final = ( + job.delivery_status == NotificationJob.DeliveryStatus.UNDELIVERED + and job.delivery_status_code == "6" + ) job.delivery_description = f"{result.description} نتیجه پس از {job.delivery_check_attempts} نوبت استعلام در بازه پایش نهایی شد."[:500] job.save(update_fields=[ "delivery_status", "delivery_status_code", "delivery_description", diff --git a/apps/notifications/migrations/0014_backfill_safe_0098_delivery_retries.py b/apps/notifications/migrations/0014_backfill_safe_0098_delivery_retries.py new file mode 100644 index 0000000..4b904a3 --- /dev/null +++ b/apps/notifications/migrations/0014_backfill_safe_0098_delivery_retries.py @@ -0,0 +1,78 @@ +from datetime import timedelta + +from django.conf import settings +from django.db import migrations +from django.db.models import Q +from django.utils import timezone + + +SAFE_0098_RESULTS = ( + ("UNDELIVERED", "6"), + ("EXPIRED", "7"), + ("ERROR", "-31"), +) + + +def _retry_dedupe_key(job): + suffix = f":definitive-delivery-retry:{job.pk}" + return f"{job.dedupe_key[:200-len(suffix)]}{suffix}" + + +def queue_historical_safe_0098_delivery_retries(apps, schema_editor): + """Queue one delayed resend for old individual staff-ticket SMS jobs.""" + NotificationJob = apps.get_model("notifications", "NotificationJob") + delay = getattr(settings, "SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS", 2700) + stagger = getattr(settings, "SMS_HISTORICAL_DELIVERY_RETRY_STAGGER_SECONDS", 30) + max_checks = getattr(settings, "SMS_DELIVERY_MAX_CHECKS", 15) + now = timezone.now() + monitor_cutoff = now - timedelta(seconds=delay) + safe_result = ( + Q(delivery_status="EXPIRED", delivery_status_code="7") + | Q(delivery_status="ERROR", delivery_status_code="-31") + | Q(delivery_status="UNDELIVERED", delivery_status_code="6", sent_at__lte=monitor_cutoff) + | Q(delivery_status="UNDELIVERED", delivery_status_code="6", delivery_check_attempts__gte=max_checks) + ) + sources = NotificationJob.objects.filter( + safe_result, + channel="SMS", + event_type="STAFF_TICKET_CREATED", + status="SENT", + ticket__group_dispatch__isnull=True, + delivery_next_check_at__isnull=True, + delivery_retry_source__isnull=True, + ).order_by("id") + queued = 0 + for source in sources.iterator(): + if NotificationJob.objects.filter(delivery_retry_source_id=source.pk).exists(): + continue + sent_at = source.sent_at or source.created_at + due = max( + sent_at + timedelta(seconds=delay), + now + timedelta(seconds=(queued + 1) * stagger), + ) + NotificationJob.objects.create( + channel=source.channel, + event_type=source.event_type, + customer_id=source.customer_id, + staff_user_id=source.staff_user_id, + ticket_id=source.ticket_id, + payload=source.payload, + status="PENDING", + attempts=0, + next_attempt_at=due, + dedupe_key=_retry_dedupe_key(source), + delivery_retry_source_id=source.pk, + ) + NotificationJob.objects.filter(pk=source.pk).update(delivery_is_final=True) + queued += 1 + + +class Migration(migrations.Migration): + dependencies = [("notifications", "0013_notificationjob_definitive_delivery_retry")] + + operations = [ + migrations.RunPython( + queue_historical_safe_0098_delivery_retries, + migrations.RunPython.noop, + ), + ] diff --git a/tests/test_domain.py b/tests/test_domain.py index ff2f3a5..fb15990 100644 --- a/tests/test_domain.py +++ b/tests/test_domain.py @@ -701,7 +701,7 @@ def test_provisional_or_unrelated_delivery_states_never_schedule_automatic_resen from apps.integrations.sms.base import SMSDeliveryResult from apps.notifications.management.commands import process_notifications - settings.SMS_DELIVERY_MAX_CHECKS=1 + settings.SMS_DELIVERY_MAX_CHECKS=2 staff=make_staff("provisional-retry-staff") staff_ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="دلیوری موقت",body="متن") staff_job=NotificationJob.objects.get(ticket=staff_ticket,event_type="STAFF_TICKET_CREATED") @@ -729,6 +729,67 @@ def test_provisional_or_unrelated_delivery_states_never_schedule_automatic_resen assert process_notifications._schedule_definitive_delivery_retry(unrelated) is None +@pytest.mark.django_db +def test_0098_code_6_schedules_one_resend_only_after_monitoring_is_exhausted(category,customer,monkeypatch,settings): + from django.core.management import call_command + from apps.integrations.sms.base import SMSDeliveryResult + from apps.notifications.management.commands import process_notifications + + settings.SMS_DELIVERY_MAX_CHECKS=1 + settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS=2700 + staff=make_staff("terminal-code-6-staff") + ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="پایان پایش کد ۶",body="متن") + source=NotificationJob.objects.get(ticket=ticket,event_type="STAFF_TICKET_CREATED") + NotificationJob.objects.filter(pk=source.pk).update( + status=NotificationJob.Status.SENT,sent_at=timezone.now()-timedelta(minutes=45), + provider_message_id="123456789",delivery_status=NotificationJob.DeliveryStatus.PENDING, + delivery_next_check_at=timezone.now()-timedelta(seconds=1), + ) + class Provider: + def get_delivery_status(self,provider_id): + return SMSDeliveryResult("UNDELIVERED","6","تحویل هنوز تأیید نشده است.",is_final=False) + monkeypatch.setattr(process_notifications,"get_sms_provider",lambda:Provider()) + + call_command("process_notifications",once=True,job_id=source.pk) + source.refresh_from_db() + retry=NotificationJob.objects.get(delivery_retry_source=source) + assert source.delivery_status==NotificationJob.DeliveryStatus.UNDELIVERED + assert source.delivery_is_final is True and source.delivery_next_check_at is None + assert retry.next_attempt_at<=timezone.now()+timedelta(seconds=2) + process_notifications._schedule_definitive_delivery_retry(source) + assert NotificationJob.objects.filter(delivery_retry_source=source).count()==1 + + +@pytest.mark.django_db +def test_0098_minus_31_schedules_one_resend_at_45_minute_floor(category,customer,monkeypatch,settings): + from django.core.management import call_command + from apps.integrations.sms.base import SMSDeliveryResult + from apps.notifications.management.commands import process_notifications + + settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS=2700 + staff=make_staff("safe-minus-31-staff") + ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="شناسه یافت نشد",body="متن") + source=NotificationJob.objects.get(ticket=ticket,event_type="STAFF_TICKET_CREATED") + sent_at=timezone.now() + NotificationJob.objects.filter(pk=source.pk).update( + status=NotificationJob.Status.SENT,sent_at=sent_at, + provider_message_id="123456789",delivery_status=NotificationJob.DeliveryStatus.PENDING, + delivery_next_check_at=timezone.now()-timedelta(seconds=1), + ) + class Provider: + def get_delivery_status(self,provider_id): + return SMSDeliveryResult("ERROR","-31","شناسه پیامک یافت نشد.",is_final=True) + monkeypatch.setattr(process_notifications,"get_sms_provider",lambda:Provider()) + + call_command("process_notifications",once=True,job_id=source.pk) + source.refresh_from_db() + retry=NotificationJob.objects.get(delivery_retry_source=source) + assert source.delivery_is_final is True and source.delivery_status_code=="-31" + assert sent_at+timedelta(seconds=2699)<=retry.next_attempt_at<=sent_at+timedelta(seconds=2701) + process_notifications._schedule_definitive_delivery_retry(source) + assert NotificationJob.objects.filter(delivery_retry_source=source).count()==1 + + @pytest.mark.django_db def test_blocked_and_delivery_error_are_not_definitive_resend_candidates(category,customer): from apps.notifications.management.commands import process_notifications @@ -739,9 +800,13 @@ def test_blocked_and_delivery_error_are_not_definitive_resend_candidates(categor source.status=NotificationJob.Status.SENT source.sent_at=timezone.now()-timedelta(hours=1) source.delivery_is_final=True - for status in (NotificationJob.DeliveryStatus.BLOCKED,NotificationJob.DeliveryStatus.ERROR): + for status,code in ( + (NotificationJob.DeliveryStatus.BLOCKED,"4"), + (NotificationJob.DeliveryStatus.ERROR,"9"), + ): source.delivery_status=status - source.save(update_fields=["status","sent_at","delivery_is_final","delivery_status"]) + source.delivery_status_code=code + source.save(update_fields=["status","sent_at","delivery_is_final","delivery_status","delivery_status_code"]) assert process_notifications._schedule_definitive_delivery_retry(source) is None assert not NotificationJob.objects.filter(delivery_retry_source=source).exists() @@ -771,6 +836,49 @@ def test_historical_expired_staff_ticket_delivery_is_queued_once_by_data_migrati migration.queue_historical_definitive_delivery_retries(django_apps,None) assert NotificationJob.objects.filter(delivery_retry_source=source).count()==1 + +@pytest.mark.django_db +def test_historical_safe_0098_results_are_queued_once_by_followup_migration(category,customer,settings): + import importlib + from django.apps import apps as django_apps + + settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS=2700 + settings.SMS_HISTORICAL_DELIVERY_RETRY_STAGGER_SECONDS=30 + settings.SMS_DELIVERY_MAX_CHECKS=15 + staff=make_staff("historical-safe-0098-staff") + sources=[] + for index,(status,code) in enumerate((("UNDELIVERED","6"),("EXPIRED","7"),("ERROR","-31"))): + ticket=create_ticket( + customer=customer,created_by=staff,unit=category.unit,category=category, + subject=f"خطای تاریخی امن {index}",body="متن", + ) + source=NotificationJob.objects.get(ticket=ticket,event_type="STAFF_TICKET_CREATED") + NotificationJob.objects.filter(pk=source.pk).update( + status=NotificationJob.Status.SENT,sent_at=timezone.now()-timedelta(days=1), + provider_message_id=f"98765432{index}",delivery_status=status, + delivery_status_code=code,delivery_next_check_at=None,delivery_is_final=False, + ) + sources.append(source) + unsafe_ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="خطای تاریخی ناامن",body="متن") + unsafe=NotificationJob.objects.get(ticket=unsafe_ticket,event_type="STAFF_TICKET_CREATED") + NotificationJob.objects.filter(pk=unsafe.pk).update( + status=NotificationJob.Status.SENT,sent_at=timezone.now()-timedelta(days=1), + provider_message_id="987654329",delivery_status=NotificationJob.DeliveryStatus.ERROR, + delivery_status_code="9",delivery_next_check_at=None,delivery_is_final=True, + ) + migration=importlib.import_module("apps.notifications.migrations.0014_backfill_safe_0098_delivery_retries") + + before=timezone.now() + migration.queue_historical_safe_0098_delivery_retries(django_apps,None) + for position,source in enumerate(sources,start=1): + source.refresh_from_db() + retry=NotificationJob.objects.get(delivery_retry_source=source) + assert source.delivery_is_final is True + assert retry.next_attempt_at>=before+timedelta(seconds=position*30-1) + assert not NotificationJob.objects.filter(delivery_retry_source=unsafe).exists() + migration.queue_historical_safe_0098_delivery_retries(django_apps,None) + assert NotificationJob.objects.filter(delivery_retry_source_id__in=[source.pk for source in sources]).count()==3 + @pytest.mark.django_db def test_customer_ticket_created_sms_has_seven_day_magic_link(category,customer,settings,monkeypatch): from datetime import timedelta @@ -898,7 +1006,7 @@ def test_0098_provider_maps_every_documented_send_error(settings,code,expected): assert expected in result.error -@pytest.mark.parametrize("code,status,is_final", [("4","BLOCKED",True),("5","DELIVERED",True),("6","UNDELIVERED",False),("7","EXPIRED",True),("8","UNKNOWN",False),("9","ERROR",True),("12","ERROR",True),("13","ERROR",True)]) +@pytest.mark.parametrize("code,status,is_final", [("4","BLOCKED",True),("5","DELIVERED",True),("6","UNDELIVERED",False),("7","EXPIRED",True),("8","UNKNOWN",False),("9","ERROR",True),("12","ERROR",True),("13","ERROR",True),("-31","ERROR",True)]) def test_0098_provider_maps_documented_delivery_states(settings,code,status,is_final): from apps.integrations.sms.sms0098 import ZeroZeroNinetyEightSMSProvider settings.SMS0098_USERNAME="user"; settings.SMS0098_PASSWORD="pass"; settings.SMS0098_FROM="3000"