chore: import existing codebase as baseline

finance/management/commands/bootstrap.py

@@ -0,0 +1,29 @@
+import getpass
+import os
+
+from django.contrib.auth.models import User
+from django.contrib.auth.password_validation import validate_password
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from finance.models import Membership, Organization
+
+
+class Command(BaseCommand):
+    help = "Create the initial organization administrator without a default password"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--username", required=True)
+        parser.add_argument("--organization", required=True)
+
+    def handle(self, *args, **options):
+        username = options["username"]
+        if User.objects.filter(username=username).exists():
+            raise CommandError("Username already exists; no changes made")
+        password = os.environ.get("BOOTSTRAP_PASSWORD") or getpass.getpass("Administrator password: ")
+        validate_password(password, User(username=username))
+        with transaction.atomic():
+            org = Organization.objects.create(name=options["organization"])
+            user = User.objects.create_user(username=username, password=password)
+            Membership.objects.create(user=user, organization=org, role="admin")
+        self.stdout.write("Administrator created")

finance/management/commands/cleanup_formula_exports.py

@@ -0,0 +1,65 @@
+import time
+from datetime import timedelta
+from pathlib import Path
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.utils import timezone
+
+from finance.models import FormulaExport
+
+
+class Command(BaseCommand):
+    help = "Remove expired formula workbooks and stale temporary files"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--loop", action="store_true")
+        parser.add_argument("--interval", type=int, default=3600)
+
+    def handle(self, *args, **options):
+        while True:
+            jobs, files = self.clean_expired()
+            self.stdout.write(f"Expired {jobs} jobs; removed {files} files")
+            if not options["loop"]:
+                return
+            time.sleep(max(60, options["interval"]))
+
+    def clean_expired(self):
+        export_root = Path(settings.MEDIA_ROOT) / "formula-exports"
+        if export_root.is_symlink():
+            raise ValueError("公式导出目录不能是符号链接")
+        root = export_root.resolve()
+        now = timezone.now()
+        jobs = files = 0
+
+        for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator():
+            path = Path(job.path) if job.path else None
+            resolved = path.resolve() if path and path.is_absolute() and not path.is_symlink() else None
+            in_root = bool(resolved and resolved.is_relative_to(root))
+            try:
+                with transaction.atomic():
+                    if in_root and resolved.is_file():
+                        resolved.unlink()
+                        files += 1
+                    job.status, job.path = "expired", ""
+                    job.save(update_fields=["status", "path"])
+                    jobs += 1
+            except OSError as exc:
+                self.stderr.write(f"Could not remove expired export {job.pk}: {exc}")
+
+        cutoff = (now - timedelta(hours=24)).timestamp()
+        if root.is_dir() and not root.is_symlink():
+            for user_dir in root.iterdir():
+                if user_dir.is_symlink() or not user_dir.is_dir():
+                    continue
+                for path in user_dir.glob("*.tmp"):
+                    try:
+                        if path.is_symlink() or not path.is_file() or path.stat().st_mtime > cutoff:
+                            continue
+                        path.unlink()
+                        files += 1
+                    except OSError as exc:
+                        self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}")
+
+        return jobs, files

finance/management/commands/import_file.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+
+from django.contrib.auth.models import User
+from django.core.files import File
+from django.core.management.base import BaseCommand, CommandError
+
+from finance.importing import file_digest, profile_file, suggest_mapping, validate_mapping
+from finance.models import Audit, Dataset
+
+
+class Command(BaseCommand):
+    help = "Import a local source using reviewed field aliases and lookup rates"
+
+    def add_arguments(self, parser):
+        parser.add_argument("path")
+        parser.add_argument("--username", required=True)
+        parser.add_argument("--confirm-mapping", action="store_true")
+
+    def handle(self, *args, **options):
+        user = User.objects.get(username=options["username"])
+        if user.membership.role != "admin" or user.membership.departments:
+            raise CommandError("Full administrator required")
+        path = Path(options["path"])
+        profile = profile_file(path)
+        mapping = validate_mapping(suggest_mapping(profile), profile)
+        if not options["confirm_mapping"]:
+            self.stdout.write(str(mapping))
+            self.stdout.write("Review mapping, then rerun with --confirm-mapping")
+            return
+        with path.open("rb") as source:
+            file = File(source, name=path.name)
+            digest = file_digest(file)
+            ds = Dataset.objects.filter(organization=user.membership.organization, sha256=digest).first()
+            if ds:
+                self.stdout.write(f"Existing version: {ds.pk} {ds.status}")
+                return
+            ds = Dataset.objects.create(
+                organization=user.membership.organization,
+                created_by=user,
+                name=path.name,
+                file=file,
+                sha256=digest,
+                mapping=mapping,
+                profile=profile,
+                status="queued",
+            )
+        Audit.objects.create(
+            organization=user.membership.organization,
+            user=user,
+            action="dataset.cli_import",
+            resource=str(ds.pk),
+            details={"mapping": mapping},
+        )
+        self.stdout.write(f"{ds.pk}: queued; import_worker will process this version")

finance/management/commands/import_worker.py

@@ -0,0 +1,74 @@
+import time
+
+from django.core.management.base import BaseCommand
+from django.db import connection, transaction
+from django.utils import timezone
+
+from finance.exporting import process_formula_export
+from finance.importing import process_dataset
+from finance.models import Dataset, FormulaExport
+
+
+class Command(BaseCommand):
+    help = "Process durable import and formula-export jobs"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--once", action="store_true")
+
+    def handle(self, *args, **options):
+        postgres = connection.vendor == "postgresql"
+        if postgres:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT pg_try_advisory_lock(782146291)")
+                if not cursor.fetchone()[0]:
+                    self.stdout.write("Another worker owns the database lock")
+                    return
+        try:
+            self.run_jobs(options, postgres)
+        finally:
+            if postgres:
+                with connection.cursor() as cursor:
+                    cursor.execute("SELECT pg_advisory_unlock(782146291)")
+
+    def run_jobs(self, options, recover):
+        while True:
+            processed = self._next_import(recover)
+            if not processed:
+                processed = self._next_export(recover)
+            if options["once"]:
+                return
+            if not processed:
+                time.sleep(2)
+
+    def _claim(self, model, states):
+        with transaction.atomic():
+            qs = model.objects.filter(status__in=states).order_by("created_at")
+            if connection.features.has_select_for_update_skip_locked:
+                qs = qs.select_for_update(skip_locked=True)
+            item = qs.first()
+            if not item:
+                return None
+            updated = model.objects.filter(pk=item.pk, status__in=states).update(status="processing")
+            return item if updated else None
+
+    def _next_import(self, recover):
+        states = ["queued", "processing"] if recover else ["queued"]
+        item = self._claim(Dataset, states)
+        if not item:
+            return False
+        Dataset.objects.filter(pk=item.pk).update(started_at=timezone.now())
+        result = process_dataset(item.pk)
+        self.stdout.write(f"{result.pk}: {result.status}, {result.row_count} rows")
+        return True
+
+    def _next_export(self, recover):
+        states = ["queued", "processing"] if recover else ["queued"]
+        item = self._claim(FormulaExport, states)
+        if not item:
+            return False
+        try:
+            stats = process_formula_export(item.pk)
+            self.stdout.write(f"{item.pk}: formula export ready, {stats['rows']} rows")
+        except Exception as exc:
+            self.stderr.write(f"Formula export {item.pk} failed: {exc}")
+        return True