some fix sms retry

apps/notifications/management/commands/process_notifications.py

@@ -20,9 +20,10 @@ from apps.tickets.models import Ticket
 logger = logging.getLogger(__name__)
 
 DEFINITIVE_DELIVERY_RETRY_EVENT = "STAFF_TICKET_CREATED"
-DEFINITIVE_DELIVERY_RETRY_STATUSES = {
-    NotificationJob.DeliveryStatus.UNDELIVERED,
-    NotificationJob.DeliveryStatus.EXPIRED,
+SAFE_0098_DELIVERY_RETRY_RESULTS = {
+    (NotificationJob.DeliveryStatus.UNDELIVERED, "6"),
+    (NotificationJob.DeliveryStatus.EXPIRED, "7"),
+    (NotificationJob.DeliveryStatus.ERROR, "-31"),
 }
 
 def _magic_link(job, reason):
@@ -175,6 +176,10 @@ def _delivery_retry_dedupe_key(job):
     return f"{job.dedupe_key[:200-len(suffix)]}{suffix}"
 
 
+def _is_safe_0098_delivery_retry_result(job):
+    return (job.delivery_status, (job.delivery_status_code or "").strip()) in SAFE_0098_DELIVERY_RETRY_RESULTS
+
+
 def _schedule_definitive_delivery_retry(job):
     """Schedule exactly one delayed resend for a definitive individual-ticket failure."""
     if (
@@ -184,7 +189,7 @@ def _schedule_definitive_delivery_retry(job):
         or not job.ticket_id
         or job.ticket.group_dispatch_id
         or not job.delivery_is_final
-        or job.delivery_status not in DEFINITIVE_DELIVERY_RETRY_STATUSES
+        or not _is_safe_0098_delivery_retry_result(job)
     ):
         return None
     sent_at = job.sent_at or job.created_at
@@ -268,6 +273,13 @@ def _process_delivery_checks(options):
             else:
                 job.delivery_status = result.status if result.status in NotificationJob.DeliveryStatus.values else NotificationJob.DeliveryStatus.ERROR
                 job.delivery_next_check_at = None
+                # 0098SMS code 6 remains provisional during polling. Once the
+                # complete monitoring window is exhausted it is safe to treat
+                # this exact result as terminal for the single delayed resend.
+                job.delivery_is_final = (
+                    job.delivery_status == NotificationJob.DeliveryStatus.UNDELIVERED
+                    and job.delivery_status_code == "6"
+                )
                 job.delivery_description = f"{result.description} نتیجه پس از {job.delivery_check_attempts} نوبت استعلام در بازه پایش نهایی شد."[:500]
         job.save(update_fields=[
             "delivery_status", "delivery_status_code", "delivery_description",

apps/notifications/migrations/0014_backfill_safe_0098_delivery_retries.py

@@ -0,0 +1,78 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import migrations
+from django.db.models import Q
+from django.utils import timezone
+
+
+SAFE_0098_RESULTS = (
+    ("UNDELIVERED", "6"),
+    ("EXPIRED", "7"),
+    ("ERROR", "-31"),
+)
+
+
+def _retry_dedupe_key(job):
+    suffix = f":definitive-delivery-retry:{job.pk}"
+    return f"{job.dedupe_key[:200-len(suffix)]}{suffix}"
+
+
+def queue_historical_safe_0098_delivery_retries(apps, schema_editor):
+    """Queue one delayed resend for old individual staff-ticket SMS jobs."""
+    NotificationJob = apps.get_model("notifications", "NotificationJob")
+    delay = getattr(settings, "SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS", 2700)
+    stagger = getattr(settings, "SMS_HISTORICAL_DELIVERY_RETRY_STAGGER_SECONDS", 30)
+    max_checks = getattr(settings, "SMS_DELIVERY_MAX_CHECKS", 15)
+    now = timezone.now()
+    monitor_cutoff = now - timedelta(seconds=delay)
+    safe_result = (
+        Q(delivery_status="EXPIRED", delivery_status_code="7")
+        | Q(delivery_status="ERROR", delivery_status_code="-31")
+        | Q(delivery_status="UNDELIVERED", delivery_status_code="6", sent_at__lte=monitor_cutoff)
+        | Q(delivery_status="UNDELIVERED", delivery_status_code="6", delivery_check_attempts__gte=max_checks)
+    )
+    sources = NotificationJob.objects.filter(
+        safe_result,
+        channel="SMS",
+        event_type="STAFF_TICKET_CREATED",
+        status="SENT",
+        ticket__group_dispatch__isnull=True,
+        delivery_next_check_at__isnull=True,
+        delivery_retry_source__isnull=True,
+    ).order_by("id")
+    queued = 0
+    for source in sources.iterator():
+        if NotificationJob.objects.filter(delivery_retry_source_id=source.pk).exists():
+            continue
+        sent_at = source.sent_at or source.created_at
+        due = max(
+            sent_at + timedelta(seconds=delay),
+            now + timedelta(seconds=(queued + 1) * stagger),
+        )
+        NotificationJob.objects.create(
+            channel=source.channel,
+            event_type=source.event_type,
+            customer_id=source.customer_id,
+            staff_user_id=source.staff_user_id,
+            ticket_id=source.ticket_id,
+            payload=source.payload,
+            status="PENDING",
+            attempts=0,
+            next_attempt_at=due,
+            dedupe_key=_retry_dedupe_key(source),
+            delivery_retry_source_id=source.pk,
+        )
+        NotificationJob.objects.filter(pk=source.pk).update(delivery_is_final=True)
+        queued += 1
+
+
+class Migration(migrations.Migration):
+    dependencies = [("notifications", "0013_notificationjob_definitive_delivery_retry")]
+
+    operations = [
+        migrations.RunPython(
+            queue_historical_safe_0098_delivery_retries,
+            migrations.RunPython.noop,
+        ),
+    ]

tests/test_domain.py

@@ -701,7 +701,7 @@ def test_provisional_or_unrelated_delivery_states_never_schedule_automatic_resen
     from apps.integrations.sms.base import SMSDeliveryResult
     from apps.notifications.management.commands import process_notifications
 
-    settings.SMS_DELIVERY_MAX_CHECKS=1
+    settings.SMS_DELIVERY_MAX_CHECKS=2
     staff=make_staff("provisional-retry-staff")
     staff_ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="دلیوری موقت",body="متن")
     staff_job=NotificationJob.objects.get(ticket=staff_ticket,event_type="STAFF_TICKET_CREATED")
@@ -729,6 +729,67 @@ def test_provisional_or_unrelated_delivery_states_never_schedule_automatic_resen
     assert process_notifications._schedule_definitive_delivery_retry(unrelated) is None
 
 
+@pytest.mark.django_db
+def test_0098_code_6_schedules_one_resend_only_after_monitoring_is_exhausted(category,customer,monkeypatch,settings):
+    from django.core.management import call_command
+    from apps.integrations.sms.base import SMSDeliveryResult
+    from apps.notifications.management.commands import process_notifications
+
+    settings.SMS_DELIVERY_MAX_CHECKS=1
+    settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS=2700
+    staff=make_staff("terminal-code-6-staff")
+    ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="پایان پایش کد ۶",body="متن")
+    source=NotificationJob.objects.get(ticket=ticket,event_type="STAFF_TICKET_CREATED")
+    NotificationJob.objects.filter(pk=source.pk).update(
+        status=NotificationJob.Status.SENT,sent_at=timezone.now()-timedelta(minutes=45),
+        provider_message_id="123456789",delivery_status=NotificationJob.DeliveryStatus.PENDING,
+        delivery_next_check_at=timezone.now()-timedelta(seconds=1),
+    )
+    class Provider:
+        def get_delivery_status(self,provider_id):
+            return SMSDeliveryResult("UNDELIVERED","6","تحویل هنوز تأیید نشده است.",is_final=False)
+    monkeypatch.setattr(process_notifications,"get_sms_provider",lambda:Provider())
+
+    call_command("process_notifications",once=True,job_id=source.pk)
+    source.refresh_from_db()
+    retry=NotificationJob.objects.get(delivery_retry_source=source)
+    assert source.delivery_status==NotificationJob.DeliveryStatus.UNDELIVERED
+    assert source.delivery_is_final is True and source.delivery_next_check_at is None
+    assert retry.next_attempt_at<=timezone.now()+timedelta(seconds=2)
+    process_notifications._schedule_definitive_delivery_retry(source)
+    assert NotificationJob.objects.filter(delivery_retry_source=source).count()==1
+
+
+@pytest.mark.django_db
+def test_0098_minus_31_schedules_one_resend_at_45_minute_floor(category,customer,monkeypatch,settings):
+    from django.core.management import call_command
+    from apps.integrations.sms.base import SMSDeliveryResult
+    from apps.notifications.management.commands import process_notifications
+
+    settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS=2700
+    staff=make_staff("safe-minus-31-staff")
+    ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="شناسه یافت نشد",body="متن")
+    source=NotificationJob.objects.get(ticket=ticket,event_type="STAFF_TICKET_CREATED")
+    sent_at=timezone.now()
+    NotificationJob.objects.filter(pk=source.pk).update(
+        status=NotificationJob.Status.SENT,sent_at=sent_at,
+        provider_message_id="123456789",delivery_status=NotificationJob.DeliveryStatus.PENDING,
+        delivery_next_check_at=timezone.now()-timedelta(seconds=1),
+    )
+    class Provider:
+        def get_delivery_status(self,provider_id):
+            return SMSDeliveryResult("ERROR","-31","شناسه پیامک یافت نشد.",is_final=True)
+    monkeypatch.setattr(process_notifications,"get_sms_provider",lambda:Provider())
+
+    call_command("process_notifications",once=True,job_id=source.pk)
+    source.refresh_from_db()
+    retry=NotificationJob.objects.get(delivery_retry_source=source)
+    assert source.delivery_is_final is True and source.delivery_status_code=="-31"
+    assert sent_at+timedelta(seconds=2699)<=retry.next_attempt_at<=sent_at+timedelta(seconds=2701)
+    process_notifications._schedule_definitive_delivery_retry(source)
+    assert NotificationJob.objects.filter(delivery_retry_source=source).count()==1
+
+
 @pytest.mark.django_db
 def test_blocked_and_delivery_error_are_not_definitive_resend_candidates(category,customer):
     from apps.notifications.management.commands import process_notifications
@@ -739,9 +800,13 @@ def test_blocked_and_delivery_error_are_not_definitive_resend_candidates(categor
     source.status=NotificationJob.Status.SENT
     source.sent_at=timezone.now()-timedelta(hours=1)
     source.delivery_is_final=True
-    for status in (NotificationJob.DeliveryStatus.BLOCKED,NotificationJob.DeliveryStatus.ERROR):
+    for status,code in (
+        (NotificationJob.DeliveryStatus.BLOCKED,"4"),
+        (NotificationJob.DeliveryStatus.ERROR,"9"),
+    ):
         source.delivery_status=status
-        source.save(update_fields=["status","sent_at","delivery_is_final","delivery_status"])
+        source.delivery_status_code=code
+        source.save(update_fields=["status","sent_at","delivery_is_final","delivery_status","delivery_status_code"])
         assert process_notifications._schedule_definitive_delivery_retry(source) is None
     assert not NotificationJob.objects.filter(delivery_retry_source=source).exists()
 
@@ -771,6 +836,49 @@ def test_historical_expired_staff_ticket_delivery_is_queued_once_by_data_migrati
     migration.queue_historical_definitive_delivery_retries(django_apps,None)
     assert NotificationJob.objects.filter(delivery_retry_source=source).count()==1
 
+
+@pytest.mark.django_db
+def test_historical_safe_0098_results_are_queued_once_by_followup_migration(category,customer,settings):
+    import importlib
+    from django.apps import apps as django_apps
+
+    settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS=2700
+    settings.SMS_HISTORICAL_DELIVERY_RETRY_STAGGER_SECONDS=30
+    settings.SMS_DELIVERY_MAX_CHECKS=15
+    staff=make_staff("historical-safe-0098-staff")
+    sources=[]
+    for index,(status,code) in enumerate((("UNDELIVERED","6"),("EXPIRED","7"),("ERROR","-31"))):
+        ticket=create_ticket(
+            customer=customer,created_by=staff,unit=category.unit,category=category,
+            subject=f"خطای تاریخی امن {index}",body="متن",
+        )
+        source=NotificationJob.objects.get(ticket=ticket,event_type="STAFF_TICKET_CREATED")
+        NotificationJob.objects.filter(pk=source.pk).update(
+            status=NotificationJob.Status.SENT,sent_at=timezone.now()-timedelta(days=1),
+            provider_message_id=f"98765432{index}",delivery_status=status,
+            delivery_status_code=code,delivery_next_check_at=None,delivery_is_final=False,
+        )
+        sources.append(source)
+    unsafe_ticket=create_ticket(customer=customer,created_by=staff,unit=category.unit,category=category,subject="خطای تاریخی ناامن",body="متن")
+    unsafe=NotificationJob.objects.get(ticket=unsafe_ticket,event_type="STAFF_TICKET_CREATED")
+    NotificationJob.objects.filter(pk=unsafe.pk).update(
+        status=NotificationJob.Status.SENT,sent_at=timezone.now()-timedelta(days=1),
+        provider_message_id="987654329",delivery_status=NotificationJob.DeliveryStatus.ERROR,
+        delivery_status_code="9",delivery_next_check_at=None,delivery_is_final=True,
+    )
+    migration=importlib.import_module("apps.notifications.migrations.0014_backfill_safe_0098_delivery_retries")
+
+    before=timezone.now()
+    migration.queue_historical_safe_0098_delivery_retries(django_apps,None)
+    for position,source in enumerate(sources,start=1):
+        source.refresh_from_db()
+        retry=NotificationJob.objects.get(delivery_retry_source=source)
+        assert source.delivery_is_final is True
+        assert retry.next_attempt_at>=before+timedelta(seconds=position*30-1)
+    assert not NotificationJob.objects.filter(delivery_retry_source=unsafe).exists()
+    migration.queue_historical_safe_0098_delivery_retries(django_apps,None)
+    assert NotificationJob.objects.filter(delivery_retry_source_id__in=[source.pk for source in sources]).count()==3
+
 @pytest.mark.django_db
 def test_customer_ticket_created_sms_has_seven_day_magic_link(category,customer,settings,monkeypatch):
     from datetime import timedelta
@@ -898,7 +1006,7 @@ def test_0098_provider_maps_every_documented_send_error(settings,code,expected):
     assert expected in result.error
 
 
-@pytest.mark.parametrize("code,status,is_final", [("4","BLOCKED",True),("5","DELIVERED",True),("6","UNDELIVERED",False),("7","EXPIRED",True),("8","UNKNOWN",False),("9","ERROR",True),("12","ERROR",True),("13","ERROR",True)])
+@pytest.mark.parametrize("code,status,is_final", [("4","BLOCKED",True),("5","DELIVERED",True),("6","UNDELIVERED",False),("7","EXPIRED",True),("8","UNKNOWN",False),("9","ERROR",True),("12","ERROR",True),("13","ERROR",True),("-31","ERROR",True)])
 def test_0098_provider_maps_documented_delivery_states(settings,code,status,is_final):
     from apps.integrations.sms.sms0098 import ZeroZeroNinetyEightSMSProvider
     settings.SMS0098_USERNAME="user"; settings.SMS0098_PASSWORD="pass"; settings.SMS0098_FROM="3000"