import logging import time from datetime import timedelta import jdatetime from django.conf import settings from django.core.management.base import BaseCommand from django.db import transaction from django.db.models import Q from django.utils import timezone from apps.accounts.models import TicketMagicLink from apps.integrations.bale.providers import get_bale_provider from apps.integrations.sms.providers import get_sms_provider from apps.messaging.models import MessageReadReceipt from apps.notifications.models import NotificationJob,NotificationPolicy,SMSTemplate from apps.notifications.services import customer_reply_bale_policy,staff_reply_policy from apps.tickets.models import Ticket logger = logging.getLogger(__name__) DEFINITIVE_DELIVERY_RETRY_EVENT = "STAFF_TICKET_CREATED" DEFINITIVE_DELIVERY_RETRY_STATUSES = { NotificationJob.DeliveryStatus.UNDELIVERED, NotificationJob.DeliveryStatus.EXPIRED, } def _magic_link(job, reason): token,link = TicketMagicLink.issue_with_record(job.ticket, reason=reason, created_by=job.staff_user or job.ticket.owner or job.ticket.created_by) return link, f"{settings.PUBLIC_BASE_URL}/t/{token}/" def _sms_context(job,magic_link=""): recipient = job.customer or job.ticket.customer profile=getattr(recipient,"customer_profile",None) unit=job.ticket.owning_unit customer_profile=getattr(job.ticket.customer,"customer_profile",None) first_name=getattr(profile,"first_name","") or "" last_name=getattr(profile,"last_name","") or "" recipient_name=" ".join(filter(None,[first_name,last_name])) or "ثبت نشده" return {"magic_link":magic_link,"first_name":first_name,"last_name":last_name,"recipient_name":recipient_name,"unit_name":unit.name if unit else "","company_name":getattr(settings,"COMPANY_NAME","لوله گستر گلپایگان"),"ticket_number":job.ticket.ticket_number,"subject":job.ticket.subject,"province":job.ticket.province or getattr(customer_profile,"province","") or "","city":job.ticket.city or getattr(customer_profile,"city","") or "","expert_name":job.ticket.assigned_expert_name,"customer_mobile":job.ticket.customer.local_mobile or "ثبت نشده","category_name":job.ticket.category.title} def _render_sms(event_type,context,unit=None): from string import Formatter template=(SMSTemplate.objects.filter(event_type=event_type,is_active=True,unit=unit).first() if unit else None) or SMSTemplate.objects.get(event_type=event_type,is_active=True,unit__isnull=True) used={field for _,field,_,_ in Formatter().parse(template.body) if field} invalid=used-set(template.allowed_tags) if invalid: raise ValueError(f"SMS template contains unsupported tags: {', '.join(sorted(invalid))}") missing=used-set(context) if missing: raise ValueError(f"SMS template context is missing: {', '.join(sorted(missing))}") return template.body.format_map(context) def _bale_ticket_text(ticket, message=None, *, is_reply=False, message_count=1, reopened=False): profile = getattr(ticket.customer, "customer_profile", None) name = " ".join(x for x in [getattr(profile, "first_name", ""), getattr(profile, "last_name", "")] if x).strip() or "ثبت نشده" location = "، ".join(x for x in [getattr(profile, "city", "") or ticket.city, getattr(profile, "province", "") or ticket.province] if x) message = message or ticket.ticket_messages.filter(is_internal=False).order_by("created_at", "id").first() raw_excerpt = " ".join((message.body if message else "").split()) excerpt = f"{raw_excerpt[:277].rstrip()}…" if len(raw_excerpt)>280 else raw_excerpt or "بدون متن" local_created = timezone.localtime(message.created_at if message else ticket.created_at) jalali = jdatetime.datetime.fromgregorian(datetime=local_created, locale=jdatetime.FA_LOCALE).strftime("%d %B، %H:%M") lines=[ "♻️ بازگشایی تیکت توسط مشتری" if reopened else (f"💬 {message_count} پیام جدید مشتری" if is_reply and message_count>1 else ("💬 پاسخ جدید مشتری" if is_reply else "🆕 تیکت جدید مشتری")), "────────────", f"🎫 {ticket.ticket_number}", f"📌 {ticket.subject}", f"👤 {name}", f"📞 {ticket.customer.local_mobile or 'ثبت نشده'}", ] if location: lines.append(f"📍 {location}") lines.extend([f"🕒 {jalali}", "", f"📝 {excerpt}", "", "🔗 مشاهده و پاسخ", f"{settings.PUBLIC_BASE_URL}/staff/tickets/{ticket.public_id}/"]) return "\n".join(lines) def _send_sms(job): issued = None if job.event_type == "UNIT_SUPERVISOR_NEW_TICKET": recipient = job.payload.get("recipient_mobile", "") if not recipient: raise ValueError("Unit supervisor mobile is not configured") message = _render_sms(job.event_type, _sms_context(job), job.ticket.owning_unit) return get_sms_provider().send_message(recipient, message), issued if job.event_type == "CUSTOMER_TICKET_CREATED": issued, url = _magic_link(job, "customer_created_ticket") message = _render_sms(job.event_type,_sms_context(job,url),job.ticket.owning_unit) elif job.event_type == "STAFF_TICKET_CREATED": issued, url = _magic_link(job, "staff_created_ticket"); message = _render_sms(job.event_type,_sms_context(job,url),job.ticket.owning_unit) elif job.event_type == "STAFF_REPLY": issued, url = _magic_link(job, "staff_reply"); message = _render_sms(job.event_type,_sms_context(job,url),job.ticket.owning_unit) else: raise ValueError(f"Unsupported SMS event: {job.event_type}") return get_sms_provider().send_message(job.customer.mobile, message), issued def _send_bale(job): unit_chat_id = (getattr(job.ticket.owning_unit, "bale_chat_id", "") or "").strip() if job.event_type == "NEW_CUSTOMER_TICKET": return get_bale_provider().send_message(unit_chat_id, _bale_ticket_text(job.ticket)) if unit_chat_id else None if job.event_type in ["CUSTOMER_TICKET_REPLY","CUSTOMER_TICKET_REOPENED"]: message = job.ticket.ticket_messages.filter(pk=job.payload.get("message_id"), is_internal=False).first() if not message: raise ValueError("Customer reply message no longer exists") return get_bale_provider().send_message(unit_chat_id, _bale_ticket_text(job.ticket,message,is_reply=True,message_count=job.payload.get("message_count",1),reopened=job.event_type=="CUSTOMER_TICKET_REOPENED")) if unit_chat_id else None if not job.staff_user or not hasattr(job.staff_user, "staff_profile"): raise ValueError("Bale recipient is not configured") profile = job.staff_user.staff_profile if not profile.bale_notifications_enabled: return None return get_bale_provider().send_message(profile.bale_chat_id, f"رویداد جدید برای تیکت {job.ticket.ticket_number}\n{settings.PUBLIC_BASE_URL}/staff/tickets/{job.ticket.public_id}/") def _prepare_staff_reply_sms(job): """Final transactional guard immediately before the irreversible provider call.""" with transaction.atomic(): ticket=Ticket.objects.select_for_update().get(pk=job.ticket_id) locked=NotificationJob.objects.select_for_update().get(pk=job.pk) policy=staff_reply_policy() message_id=locked.payload.get("latest_message_id") message=ticket.ticket_messages.filter(pk=message_id,is_internal=False,author__account_type="STAFF").first() reason="" if not policy.is_active: reason="policy_disabled" elif not message: reason="staff_message_missing" elif MessageReadReceipt.objects.filter(message=message,user_id=locked.customer_id).exists(): reason="customer_read" elif ticket.ticket_messages.filter(is_internal=False,author_id=locked.customer_id,created_at__gt=message.created_at).exists(): reason="customer_replied" if reason: locked.status=NotificationJob.Status.CANCELLED; locked.last_error=reason locked.save(update_fields=["status","last_error"]) return False due=ticket.last_activity_at+timedelta(seconds=policy.quiet_period_seconds) if due>timezone.now(): locked.status=NotificationJob.Status.PENDING; locked.next_attempt_at=due; locked.last_error="conversation_active" locked.save(update_fields=["status","next_attempt_at","last_error"]) return False return True def _prepare_customer_reply_bale(job): with transaction.atomic(): ticket=Ticket.objects.select_for_update().get(pk=job.ticket_id) locked=NotificationJob.objects.select_for_update().get(pk=job.pk) policy=customer_reply_bale_policy() if not policy.is_active: locked.status=NotificationJob.Status.CANCELLED; locked.last_error="policy_disabled"; locked.save(update_fields=["status","last_error"]); return False due=ticket.last_customer_message_at+timedelta(seconds=policy.quiet_period_seconds) if due>timezone.now(): locked.status=NotificationJob.Status.PENDING; locked.next_attempt_at=due; locked.last_error="customer_chat_active" locked.save(update_fields=["status","next_attempt_at","last_error"]); return False return True def _record_failure(job, error, provider_code=""): """Persist a provider failure using the channel-specific retry policy.""" job.attempts += 1 job.last_error = str(error)[:1000] job.provider_status_code = str(provider_code or "")[:50] job.provider_message_id = "" if job.channel == NotificationJob.Channel.SMS: if job.attempts < 1 + settings.SMS_MAX_RETRIES: job.status = NotificationJob.Status.RETRYING job.next_attempt_at = timezone.now() + timedelta(seconds=settings.SMS_RETRY_DELAY_SECONDS) else: job.status = NotificationJob.Status.FAILED else: job.status = NotificationJob.Status.FAILED job.next_attempt_at = timezone.now() + timedelta(seconds=min(3600, 2**job.attempts * 10)) def _record_success(job, result): job.attempts += 1 job.status = NotificationJob.Status.SENT job.sent_at = timezone.now() job.last_error = "" job.provider_message_id = getattr(result, "provider_id", "") if result is not None else "" job.provider_status_code = getattr(result, "provider_code", "") if result is not None else "" job.delivery_is_final = False if job.channel == NotificationJob.Channel.SMS and job.provider_message_id: job.delivery_status = NotificationJob.DeliveryStatus.PENDING job.delivery_next_check_at = timezone.now() + timedelta(seconds=settings.SMS_DELIVERY_CHECK_DELAY_SECONDS) def _delivery_retry_dedupe_key(job): suffix = f":definitive-delivery-retry:{job.pk}" return f"{job.dedupe_key[:200-len(suffix)]}{suffix}" def _schedule_definitive_delivery_retry(job): """Schedule exactly one delayed resend for a definitive individual-ticket failure.""" if ( job.channel != NotificationJob.Channel.SMS or job.event_type != DEFINITIVE_DELIVERY_RETRY_EVENT or job.delivery_retry_source_id or not job.ticket_id or job.ticket.group_dispatch_id or not job.delivery_is_final or job.delivery_status not in DEFINITIVE_DELIVERY_RETRY_STATUSES ): return None sent_at = job.sent_at or job.created_at due = max( timezone.now(), sent_at + timedelta(seconds=settings.SMS_DEFINITIVE_DELIVERY_RETRY_AFTER_SECONDS), ) retry, _ = NotificationJob.objects.get_or_create( delivery_retry_source=job, defaults={ "channel": job.channel, "event_type": job.event_type, "customer_id": job.customer_id, "staff_user_id": job.staff_user_id, "ticket_id": job.ticket_id, "payload": job.payload, "next_attempt_at": due, "dedupe_key": _delivery_retry_dedupe_key(job), }, ) return retry def _process_delivery_checks(options): if options.get("channel") not in (None, NotificationJob.Channel.SMS): return now = timezone.now() with transaction.atomic(): pollable_statuses = [ NotificationJob.DeliveryStatus.PENDING, NotificationJob.DeliveryStatus.CHECK_FAILED, NotificationJob.DeliveryStatus.UNKNOWN, NotificationJob.DeliveryStatus.UNDELIVERED, ] # Resume provisional results created by older worker versions, but do # not revive messages beyond the configured 45-minute monitoring span. monitor_cutoff = now - timedelta(seconds=settings.SMS_DELIVERY_CHECK_DELAY_SECONDS * settings.SMS_DELIVERY_MAX_CHECKS) NotificationJob.objects.filter( channel=NotificationJob.Channel.SMS, status=NotificationJob.Status.SENT, delivery_status__in=[NotificationJob.DeliveryStatus.UNKNOWN, NotificationJob.DeliveryStatus.UNDELIVERED], delivery_next_check_at__isnull=True, delivery_check_attempts__lt=settings.SMS_DELIVERY_MAX_CHECKS, sent_at__gte=monitor_cutoff, ).update(delivery_next_check_at=now) queryset = NotificationJob.objects.select_for_update(skip_locked=True).filter( channel=NotificationJob.Channel.SMS, status=NotificationJob.Status.SENT, delivery_status__in=pollable_statuses, delivery_next_check_at__lte=now, ) if options.get("job_id"): queryset = queryset.filter(pk=options["job_id"]) elif options.get("min_id"): queryset = queryset.filter(pk__gte=options["min_id"]) jobs = list(queryset.order_by("delivery_next_check_at")[:20]) # Lease each status check so parallel workers cannot query it twice. NotificationJob.objects.filter(pk__in=[job.pk for job in jobs]).update( delivery_next_check_at=now + timedelta(minutes=10), ) provider = get_sms_provider() for job in jobs: result = provider.get_delivery_status(job.provider_message_id) if result is None: job.delivery_status = "" job.delivery_next_check_at = None job.delivery_description = "این ارائه‌دهنده از استعلام دلیوری پشتیبانی نمی‌کند." job.delivery_is_final = False else: job.delivery_check_attempts += 1 job.delivery_checked_at = timezone.now() job.delivery_status_code = result.provider_code[:50] job.delivery_description = result.description[:500] job.delivery_is_final = result.is_final if result.is_final: job.delivery_status = result.status job.delivery_next_check_at = None elif job.delivery_check_attempts < settings.SMS_DELIVERY_MAX_CHECKS: job.delivery_status = result.status job.delivery_next_check_at = timezone.now() + timedelta(seconds=settings.SMS_DELIVERY_CHECK_DELAY_SECONDS) else: job.delivery_status = result.status if result.status in NotificationJob.DeliveryStatus.values else NotificationJob.DeliveryStatus.ERROR job.delivery_next_check_at = None job.delivery_description = f"{result.description} نتیجه پس از {job.delivery_check_attempts} نوبت استعلام در بازه پایش نهایی شد."[:500] job.save(update_fields=[ "delivery_status", "delivery_status_code", "delivery_description", "delivery_checked_at", "delivery_check_attempts", "delivery_next_check_at", "delivery_is_final", ]) _schedule_definitive_delivery_retry(job) if job.ticket_id and job.ticket.group_dispatch_id: from apps.tickets.services import refresh_group_dispatch_status refresh_group_dispatch_status(job.ticket.group_dispatch_id) class Command(BaseCommand): help = "Process notification outbox" def add_arguments(self, parser): parser.add_argument("--once", action="store_true") parser.add_argument("--job-id", type=int, help="Process only one notification job") parser.add_argument("--min-id", type=int, help="Process this job id and newer jobs") parser.add_argument("--channel", choices=NotificationJob.Channel.values, help="Process only one notification channel") def handle(self, *args, **options): while True: stale = timezone.now() - timedelta(minutes=10) NotificationJob.objects.filter(status="PROCESSING", next_attempt_at__lt=stale, channel=NotificationJob.Channel.SMS).update(status=NotificationJob.Status.RETRYING, next_attempt_at=timezone.now(), last_error="بازیابی تلاش ارسال متوقف‌شده") NotificationJob.objects.filter(status="PROCESSING", next_attempt_at__lt=stale, channel=NotificationJob.Channel.BALE).update(status=NotificationJob.Status.FAILED, next_attempt_at=timezone.now(), last_error="Recovered stale processing job") with transaction.atomic(): sms_limit = 1 + settings.SMS_MAX_RETRIES queryset = NotificationJob.objects.select_for_update(skip_locked=True).filter( Q(channel=NotificationJob.Channel.SMS, attempts__lt=sms_limit, status__in=[NotificationJob.Status.PENDING, NotificationJob.Status.RETRYING]) | Q(channel=NotificationJob.Channel.BALE, attempts__lt=settings.MAX_NOTIFICATION_ATTEMPTS, status__in=[NotificationJob.Status.PENDING, NotificationJob.Status.FAILED]), next_attempt_at__lte=timezone.now(), ) if options.get("channel"): queryset = queryset.filter(channel=options["channel"]) if options.get("job_id"): queryset = queryset.filter(pk=options["job_id"]) elif options.get("min_id"): queryset = queryset.filter(pk__gte=options["min_id"]) jobs = list(queryset.order_by("created_at")[:20]) NotificationJob.objects.filter(pk__in=[x.pk for x in jobs]).update(status="PROCESSING", next_attempt_at=timezone.now()) for job in jobs: issued_link = None try: if job.channel == NotificationJob.Channel.SMS and job.event_type == "STAFF_REPLY" and not _prepare_staff_reply_sms(job): continue if job.channel == NotificationJob.Channel.BALE and job.event_type == "CUSTOMER_TICKET_REPLY" and not _prepare_customer_reply_bale(job): continue if job.channel == NotificationJob.Channel.SMS: result, issued_link = _send_sms(job) else: result = _send_bale(job) if result is not None and not result.success: _record_failure(job, result.error or "ارائه‌دهنده پیام را نپذیرفت.", getattr(result, "provider_code", "")) if issued_link: issued_link.revoked_at = timezone.now() issued_link.save(update_fields=["revoked_at"]) else: _record_success(job, result) except Exception as exc: if issued_link: issued_link.revoked_at = timezone.now(); issued_link.save(update_fields=["revoked_at"]) _record_failure(job, exc) logger.exception("notification failed id=%s", job.pk) job.save() if job.ticket_id and job.ticket.group_dispatch_id: from apps.tickets.services import refresh_group_dispatch_status refresh_group_dispatch_status(job.ticket.group_dispatch_id) _process_delivery_checks(options) if options["once"] or options.get("job_id"): break time.sleep(5)