Jayway Liu
54dc58d2a5
- process_formula_export 失败清理受阻时记录不含敏感数据的 WARNING（任务编号+文件名）， 并保留 job.path 使残留文件仍可追踪，而不是清空后失联 - cleanup_formula_exports 新增孤儿 .xlsx 回收：超过 24h 保留期且不属于任何 未过期成功任务的文件统一回收（覆盖失败任务残留、任务行被级联删除两种来源）， 不触碰仍处于活动任务的文件 - 测试：注入 unlink 权限错误→任务失败状态仍保存+告警可见；恢复权限后回收成功 且其他任务文件不受影响；失败任务残留/任务行已删/未过期成功/近期文件/非 UUID 命名
107 lines
4.3 KiB
Python
107 lines
4.3 KiB
Python
import time
|
|
import uuid
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
|
|
from django.conf import settings
|
|
from django.core.management.base import BaseCommand
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
|
|
from finance.models import FormulaExport
|
|
|
|
ORPHAN_RETENTION_HOURS = 24
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Remove expired formula workbooks and reclaim orphan files"
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("--loop", action="store_true")
|
|
parser.add_argument("--interval", type=int, default=3600)
|
|
|
|
def handle(self, *args, **options):
|
|
while True:
|
|
jobs, files, orphans = self.clean_expired()
|
|
self.stdout.write(
|
|
f"Expired {jobs} jobs; removed {files} files; reclaimed {orphans} orphan files"
|
|
)
|
|
if not options["loop"]:
|
|
return
|
|
time.sleep(max(60, options["interval"]))
|
|
|
|
def clean_expired(self):
|
|
export_root = Path(settings.MEDIA_ROOT) / "formula-exports"
|
|
if export_root.is_symlink():
|
|
raise ValueError("公式导出目录不能是符号链接")
|
|
root = export_root.resolve()
|
|
now = timezone.now()
|
|
jobs = files = orphans = 0
|
|
|
|
for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator():
|
|
path = Path(job.path) if job.path else None
|
|
resolved = path.resolve() if path and path.is_absolute() and not path.is_symlink() else None
|
|
in_root = bool(resolved and resolved.is_relative_to(root))
|
|
try:
|
|
with transaction.atomic():
|
|
if in_root and resolved.is_file():
|
|
resolved.unlink()
|
|
files += 1
|
|
job.status, job.path = "expired", ""
|
|
job.save(update_fields=["status", "path"])
|
|
jobs += 1
|
|
except OSError as exc:
|
|
self.stderr.write(f"Could not remove expired export {job.pk}: {exc}")
|
|
|
|
cutoff = (now - timedelta(hours=24)).timestamp()
|
|
if root.is_dir() and not root.is_symlink():
|
|
for user_dir in root.iterdir():
|
|
if user_dir.is_symlink() or not user_dir.is_dir():
|
|
continue
|
|
for path in user_dir.glob("*.tmp"):
|
|
try:
|
|
if path.is_symlink() or not path.is_file() or path.stat().st_mtime > cutoff:
|
|
continue
|
|
path.unlink()
|
|
files += 1
|
|
except OSError as exc:
|
|
self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}")
|
|
orphans += self._reclaim_orphan_workbooks(user_dir, now)
|
|
|
|
return jobs, files, orphans
|
|
|
|
def _reclaim_orphan_workbooks(self, user_dir, now):
|
|
"""回收超过保留期且不再属于任何有效导出任务的 .xlsx 孤儿文件。
|
|
|
|
有效任务指状态 succeeded、路径一致且尚未过期的任务；失败任务残留的文件和任务行
|
|
已被删除（如数据集级联删除）的文件在保留期后回收，绝不触碰活动任务的文件。
|
|
"""
|
|
retention = (now - timedelta(hours=ORPHAN_RETENTION_HOURS)).timestamp()
|
|
removed = 0
|
|
for path in user_dir.glob("*.xlsx"):
|
|
try:
|
|
if path.is_symlink() or not path.is_file() or path.stat().st_mtime > retention:
|
|
continue
|
|
job_id = uuid.UUID(path.stem)
|
|
except (OSError, ValueError):
|
|
continue
|
|
job = FormulaExport.objects.filter(pk=job_id).first()
|
|
if (
|
|
job is not None
|
|
and job.status == "succeeded"
|
|
and job.path == str(path)
|
|
and job.expires_at is not None
|
|
and job.expires_at > now
|
|
):
|
|
continue
|
|
try:
|
|
path.unlink()
|
|
removed += 1
|
|
if job is not None and job.path == str(path):
|
|
job.path = ""
|
|
job.save(update_fields=["path"])
|
|
self.stdout.write(f"Reclaimed leftover workbook of job {job.pk}")
|
|
except OSError as exc:
|
|
self.stderr.write(f"Could not reclaim orphan export {path.stem}: {exc}")
|
|
return removed
|