fix(cleanup): F02 清理失败记录含任务编号的告警,孤儿 xlsx 按保留期回收

- process_formula_export 失败清理受阻时记录不含敏感数据的 WARNING(任务编号+文件名),
  并保留 job.path 使残留文件仍可追踪,而不是清空后失联
- cleanup_formula_exports 新增孤儿 .xlsx 回收:超过 24h 保留期且不属于任何
  未过期成功任务的文件统一回收(覆盖失败任务残留、任务行被级联删除两种来源),
  不触碰仍处于活动任务的文件
- 测试:注入 unlink 权限错误→任务失败状态仍保存+告警可见;恢复权限后回收成功
  且其他任务文件不受影响;失败任务残留/任务行已删/未过期成功/近期文件/非 UUID 命名
This commit is contained in:
Jayway Liu
2026-09-26 01:20:50 +08:00
parent 36fd5f3461
commit 54dc58d2a5
4 changed files with 186 additions and 7 deletions
+7
View File
@@ -675,6 +675,13 @@ def process_formula_export(export_id):
job.status, job.error, job.finished_at, job.expires_at = "failed", str(exc)[:500], timezone.now(), None
# 只清理本次登记且确认位于受控目录内的文件;目录被换成符号链接时拒绝删除。
leftover = [path for path in created if _unlink_created_file(path) == "blocked"]
if leftover:
# 日志只含任务编号与文件名,不包含敏感数据;残留文件由清理命令按保留期回收。
logger.warning(
"公式导出任务 %s 失败后清理文件受阻,待清理命令回收:%s",
job.pk,
"、".join(path.name for path in leftover),
)
job.path = str(target) if target in leftover else ""
job.save(update_fields=["status", "error", "path", "finished_at", "expires_at"])
raise
@@ -1,4 +1,5 @@
import time
import uuid
from datetime import timedelta
from pathlib import Path
@@ -9,9 +10,11 @@ from django.utils import timezone
from finance.models import FormulaExport
ORPHAN_RETENTION_HOURS = 24
class Command(BaseCommand):
help = "Remove expired formula workbooks and stale temporary files"
help = "Remove expired formula workbooks and reclaim orphan files"
def add_arguments(self, parser):
parser.add_argument("--loop", action="store_true")
@@ -19,8 +22,10 @@ class Command(BaseCommand):
def handle(self, *args, **options):
while True:
jobs, files = self.clean_expired()
self.stdout.write(f"Expired {jobs} jobs; removed {files} files")
jobs, files, orphans = self.clean_expired()
self.stdout.write(
f"Expired {jobs} jobs; removed {files} files; reclaimed {orphans} orphan files"
)
if not options["loop"]:
return
time.sleep(max(60, options["interval"]))
@@ -31,7 +36,7 @@ class Command(BaseCommand):
raise ValueError("公式导出目录不能是符号链接")
root = export_root.resolve()
now = timezone.now()
jobs = files = 0
jobs = files = orphans = 0
for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator():
path = Path(job.path) if job.path else None
@@ -61,5 +66,41 @@ class Command(BaseCommand):
files += 1
except OSError as exc:
self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}")
orphans += self._reclaim_orphan_workbooks(user_dir, now)
return jobs, files
return jobs, files, orphans
def _reclaim_orphan_workbooks(self, user_dir, now):
"""回收超过保留期且不再属于任何有效导出任务的 .xlsx 孤儿文件。
有效任务指状态 succeeded、路径一致且尚未过期的任务;失败任务残留的文件和任务行
已被删除(如数据集级联删除)的文件在保留期后回收,绝不触碰活动任务的文件。
"""
retention = (now - timedelta(hours=ORPHAN_RETENTION_HOURS)).timestamp()
removed = 0
for path in user_dir.glob("*.xlsx"):
try:
if path.is_symlink() or not path.is_file() or path.stat().st_mtime > retention:
continue
job_id = uuid.UUID(path.stem)
except (OSError, ValueError):
continue
job = FormulaExport.objects.filter(pk=job_id).first()
if (
job is not None
and job.status == "succeeded"
and job.path == str(path)
and job.expires_at is not None
and job.expires_at > now
):
continue
try:
path.unlink()
removed += 1
if job is not None and job.path == str(path):
job.path = ""
job.save(update_fields=["path"])
self.stdout.write(f"Reclaimed leftover workbook of job {job.pk}")
except OSError as exc:
self.stderr.write(f"Could not reclaim orphan export {path.stem}: {exc}")
return removed