seed script
This commit is contained in:
1 parent
fa9d154db9
commit
08c30aa47f
15 files changed
+496
-17
No files matched your search
+16
-1
@@ -1,6 +1,6 @@
|
||||
from django.contrib import admin
|
||||
|
||||
from .models import PageTitle, SiteBranding
|
||||
from .models import PageTitle, ProductionSeedState, SiteBranding
|
||||
|
||||
|
||||
@admin.register(SiteBranding)
|
||||
@@ -35,3 +35,18 @@ class PageTitleAdmin(admin.ModelAdmin):
|
||||
list_editable = ("title", "is_active")
|
||||
search_fields = ("route_name", "title")
|
||||
list_filter = ("is_active",)
|
||||
|
||||
|
||||
@admin.register(ProductionSeedState)
|
||||
class ProductionSeedStateAdmin(admin.ModelAdmin):
|
||||
list_display = ("bundle_sha256", "applied_at")
|
||||
readonly_fields = ("bundle_sha256", "manifest", "applied_at")
|
||||
|
||||
def has_add_permission(self, request):
|
||||
return False
|
||||
|
||||
def has_change_permission(self, request, obj=None):
|
||||
return False
|
||||
|
||||
def has_delete_permission(self, request, obj=None):
|
||||
return False
|
||||
@@ -0,0 +1,98 @@
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import tarfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from django.apps import apps
|
||||
from django.conf import settings
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import connection, transaction
|
||||
|
||||
from apps.core.production_seed import EXCLUDED_VOLATILE_MODELS, SEED_MODEL_LABELS, sha256_file
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Export durable SQLite data and media as a PostgreSQL-safe production seed bundle."
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("--output", default="production-seed")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if connection.vendor != "sqlite":
|
||||
raise CommandError("Export must run against the source SQLite database.")
|
||||
|
||||
output_dir = Path(options["output"]).resolve()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
targets = [output_dir / name for name in ("data.json.gz", "media.tar.gz", "manifest.json")]
|
||||
if any(path.exists() for path in targets) and not options["force"]:
|
||||
raise CommandError(f"Seed files already exist in {output_dir}; pass --force to replace them.")
|
||||
|
||||
temporary_dir = output_dir / ".building"
|
||||
if temporary_dir.exists():
|
||||
shutil.rmtree(temporary_dir)
|
||||
temporary_dir.mkdir()
|
||||
try:
|
||||
data_path = temporary_dir / "data.json.gz"
|
||||
buffer = io.StringIO()
|
||||
with transaction.atomic():
|
||||
call_command(
|
||||
"dumpdata",
|
||||
*SEED_MODEL_LABELS,
|
||||
format="json",
|
||||
indent=2,
|
||||
use_natural_foreign_keys=True,
|
||||
use_natural_primary_keys=True,
|
||||
stdout=buffer,
|
||||
verbosity=0,
|
||||
)
|
||||
with gzip.open(data_path, "wt", encoding="utf-8") as stream:
|
||||
stream.write(buffer.getvalue())
|
||||
|
||||
media_path = temporary_dir / "media.tar.gz"
|
||||
media_root = Path(settings.MEDIA_ROOT)
|
||||
media_files = []
|
||||
with tarfile.open(media_path, "w:gz") as archive:
|
||||
if media_root.exists():
|
||||
for source in sorted(media_root.rglob("*")):
|
||||
if source.is_file() and not source.is_symlink():
|
||||
relative = source.relative_to(media_root).as_posix()
|
||||
archive.add(source, arcname=relative, recursive=False)
|
||||
media_files.append(relative)
|
||||
|
||||
counts = {
|
||||
label: apps.get_model(label).objects.count()
|
||||
for label in SEED_MODEL_LABELS
|
||||
}
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"source_database": "sqlite",
|
||||
"models": counts,
|
||||
"media_file_count": len(media_files),
|
||||
"excluded_volatile_models": list(EXCLUDED_VOLATILE_MODELS),
|
||||
"files": {
|
||||
"data.json.gz": sha256_file(data_path),
|
||||
"media.tar.gz": sha256_file(media_path),
|
||||
},
|
||||
}
|
||||
manifest_path = temporary_dir / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
for target in targets:
|
||||
source = temporary_dir / target.name
|
||||
source.replace(target)
|
||||
finally:
|
||||
shutil.rmtree(temporary_dir, ignore_errors=True)
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(
|
||||
f"Production seed exported to {output_dir} "
|
||||
f"({sum(counts.values())} rows, {len(media_files)} media files)."
|
||||
))
|
||||
self.stdout.write(self.style.WARNING(
|
||||
"This bundle contains personal data and password hashes. Keep it out of Git and transfer it securely."
|
||||
))
|
||||
@@ -0,0 +1,105 @@
|
||||
import json
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import connection, transaction
|
||||
|
||||
from apps.accounts.models import User
|
||||
from apps.core.models import ProductionSeedState
|
||||
from apps.core.production_seed import sha256_file
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Import a verified production seed bundle into a pristine PostgreSQL database."
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("bundle_dir", nargs="?", default="/app/production-seed")
|
||||
|
||||
def _load_manifest(self, bundle_dir):
|
||||
manifest_path = bundle_dir / "manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
raise CommandError(f"Missing seed manifest: {manifest_path}")
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest.get("schema_version") != 1:
|
||||
raise CommandError("Unsupported production seed schema version.")
|
||||
for filename, expected in manifest.get("files", {}).items():
|
||||
path = bundle_dir / filename
|
||||
if not path.is_file() or sha256_file(path) != expected:
|
||||
raise CommandError(f"Seed integrity check failed for {filename}.")
|
||||
return manifest, sha256_file(manifest_path)
|
||||
|
||||
def _extract_media(self, archive_path, destination):
|
||||
destination = destination.resolve()
|
||||
with tarfile.open(archive_path, "r:gz") as archive:
|
||||
for member in archive.getmembers():
|
||||
member_path = Path(member.name)
|
||||
if member.issym() or member.islnk() or member_path.is_absolute() or ".." in member_path.parts:
|
||||
raise CommandError(f"Unsafe media archive member: {member.name}")
|
||||
if member.isdir():
|
||||
continue
|
||||
if not member.isfile():
|
||||
raise CommandError(f"Unsupported media archive member: {member.name}")
|
||||
target = (destination / member_path).resolve()
|
||||
if destination not in target.parents:
|
||||
raise CommandError(f"Unsafe media archive path: {member.name}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = archive.extractfile(member)
|
||||
if source is None:
|
||||
raise CommandError(f"Could not read media archive member: {member.name}")
|
||||
with source, target.open("wb") as stream:
|
||||
shutil.copyfileobj(source, stream)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if connection.vendor != "postgresql":
|
||||
raise CommandError("Production seed import is only allowed on PostgreSQL.")
|
||||
|
||||
bundle_dir = Path(options["bundle_dir"]).resolve()
|
||||
manifest, bundle_sha256 = self._load_manifest(bundle_dir)
|
||||
if ProductionSeedState.objects.filter(bundle_sha256=bundle_sha256).exists():
|
||||
self.stdout.write(self.style.SUCCESS("Production seed is already applied; skipping."))
|
||||
return
|
||||
if ProductionSeedState.objects.exists() or User.objects.exists():
|
||||
raise CommandError("Target database is not pristine; refusing to replace production data.")
|
||||
|
||||
media_root = Path(settings.MEDIA_ROOT).resolve()
|
||||
if media_root.exists() and any(path.is_file() for path in media_root.rglob("*")):
|
||||
raise CommandError("MEDIA_ROOT is not empty; refusing to overwrite production files.")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="customer-ticket-seed-") as temporary:
|
||||
staged_media = Path(temporary) / "media"
|
||||
staged_media.mkdir()
|
||||
self._extract_media(bundle_dir / "media.tar.gz", staged_media)
|
||||
|
||||
copied_files = []
|
||||
try:
|
||||
with transaction.atomic():
|
||||
# Migrations contain defaults. Flush those rows before loading
|
||||
# the authoritative SQLite snapshot; schema/migration history remains.
|
||||
call_command("flush", interactive=False, verbosity=0)
|
||||
call_command("loaddata", str(bundle_dir / "data.json.gz"), verbosity=1)
|
||||
media_root.mkdir(parents=True, exist_ok=True)
|
||||
for source in sorted(staged_media.rglob("*")):
|
||||
if not source.is_file():
|
||||
continue
|
||||
target = media_root / source.relative_to(staged_media)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, target)
|
||||
copied_files.append(target)
|
||||
ProductionSeedState.objects.create(
|
||||
bundle_sha256=bundle_sha256,
|
||||
manifest=manifest,
|
||||
)
|
||||
except Exception:
|
||||
for path in reversed(copied_files):
|
||||
path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(
|
||||
f"Production seed applied: {sum(manifest['models'].values())} rows and "
|
||||
f"{manifest['media_file_count']} media files."
|
||||
))
|
||||
@@ -0,0 +1,22 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("core", "0005_sitebranding_company_logo_and_more")]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="ProductionSeedState",
|
||||
fields=[
|
||||
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
|
||||
("bundle_sha256", models.CharField(max_length=64, unique=True)),
|
||||
("manifest", models.JSONField(default=dict)),
|
||||
("applied_at", models.DateTimeField(auto_now_add=True)),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "وضعیت بذر Production",
|
||||
"verbose_name_plural": "وضعیت بذرهای Production",
|
||||
"ordering": ("-applied_at",),
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -125,3 +125,19 @@ class PageTitle(models.Model):
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.route_name}: {self.title}"
|
||||
|
||||
|
||||
class ProductionSeedState(models.Model):
|
||||
"""Audit marker that makes production seed imports idempotent."""
|
||||
|
||||
bundle_sha256 = models.CharField(max_length=64, unique=True)
|
||||
manifest = models.JSONField(default=dict)
|
||||
applied_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-applied_at",)
|
||||
verbose_name = "وضعیت بذر Production"
|
||||
verbose_name_plural = "وضعیت بذرهای Production"
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.bundle_sha256[:12]} @ {self.applied_at}"
|
||||
@@ -0,0 +1,48 @@
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SEED_MODEL_LABELS = (
|
||||
"core.SiteBranding",
|
||||
"core.PageTitle",
|
||||
"auth.Group",
|
||||
"accounts.User",
|
||||
"staff.OrganizationalUnit",
|
||||
"customers.ContactGroup",
|
||||
"customers.CustomerProfile",
|
||||
"staff.UnitMembership",
|
||||
"staff.StaffProfile",
|
||||
"tickets.TicketCategory",
|
||||
"tickets.SLAConfig",
|
||||
"tickets.TicketTag",
|
||||
"tickets.GroupDispatch",
|
||||
"tickets.Ticket",
|
||||
"tickets.TicketAccessGrant",
|
||||
"tickets.TicketEvent",
|
||||
"messaging.Message",
|
||||
"messaging.MessageRevision",
|
||||
"messaging.MessageMention",
|
||||
"messaging.MessageReadReceipt",
|
||||
"tickets.TicketReadState",
|
||||
"attachments.AttachmentPolicy",
|
||||
"attachments.Attachment",
|
||||
"notifications.SMSTemplate",
|
||||
"notifications.NotificationPolicy",
|
||||
)
|
||||
|
||||
EXCLUDED_VOLATILE_MODELS = (
|
||||
"admin.LogEntry",
|
||||
"sessions.Session",
|
||||
"accounts.OTPChallenge",
|
||||
"accounts.TicketMagicLink",
|
||||
"accounts.MagicLinkAccessLog",
|
||||
"notifications.NotificationJob",
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
Reference in new issue
Block a user