diff --git a/finance/exporting.py b/finance/exporting.py index b08ecf2..3051a5e 100644 --- a/finance/exporting.py +++ b/finance/exporting.py @@ -675,6 +675,13 @@ def process_formula_export(export_id): job.status, job.error, job.finished_at, job.expires_at = "failed", str(exc)[:500], timezone.now(), None # 只清理本次登记且确认位于受控目录内的文件；目录被换成符号链接时拒绝删除。 leftover = [path for path in created if _unlink_created_file(path) == "blocked"] + if leftover: + # 日志只含任务编号与文件名，不包含敏感数据；残留文件由清理命令按保留期回收。 + logger.warning( + "公式导出任务 %s 失败后清理文件受阻，待清理命令回收：%s", + job.pk, + "、".join(path.name for path in leftover), + ) job.path = str(target) if target in leftover else "" job.save(update_fields=["status", "error", "path", "finished_at", "expires_at"]) raise diff --git a/finance/management/commands/cleanup_formula_exports.py b/finance/management/commands/cleanup_formula_exports.py index 692e1aa..20cd91c 100644 --- a/finance/management/commands/cleanup_formula_exports.py +++ b/finance/management/commands/cleanup_formula_exports.py @@ -1,4 +1,5 @@ import time +import uuid from datetime import timedelta from pathlib import Path @@ -9,9 +10,11 @@ from django.utils import timezone from finance.models import FormulaExport +ORPHAN_RETENTION_HOURS = 24 + class Command(BaseCommand): - help = "Remove expired formula workbooks and stale temporary files" + help = "Remove expired formula workbooks and reclaim orphan files" def add_arguments(self, parser): parser.add_argument("--loop", action="store_true") @@ -19,8 +22,10 @@ class Command(BaseCommand): def handle(self, *args, **options): while True: - jobs, files = self.clean_expired() - self.stdout.write(f"Expired {jobs} jobs; removed {files} files") + jobs, files, orphans = self.clean_expired() + self.stdout.write( + f"Expired {jobs} jobs; removed {files} files; reclaimed {orphans} orphan files" + ) if not options["loop"]: return time.sleep(max(60, options["interval"])) @@ -31,7 +36,7 @@ class Command(BaseCommand): raise ValueError("公式导出目录不能是符号链接") root = export_root.resolve() now = timezone.now() - jobs = files = 0 + jobs = files = orphans = 0 for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator(): path = Path(job.path) if job.path else None @@ -61,5 +66,41 @@ class Command(BaseCommand): files += 1 except OSError as exc: self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}") + orphans += self._reclaim_orphan_workbooks(user_dir, now) - return jobs, files + return jobs, files, orphans + + def _reclaim_orphan_workbooks(self, user_dir, now): + """回收超过保留期且不再属于任何有效导出任务的 .xlsx 孤儿文件。 + + 有效任务指状态 succeeded、路径一致且尚未过期的任务；失败任务残留的文件和任务行 + 已被删除（如数据集级联删除）的文件在保留期后回收，绝不触碰活动任务的文件。 + """ + retention = (now - timedelta(hours=ORPHAN_RETENTION_HOURS)).timestamp() + removed = 0 + for path in user_dir.glob("*.xlsx"): + try: + if path.is_symlink() or not path.is_file() or path.stat().st_mtime > retention: + continue + job_id = uuid.UUID(path.stem) + except (OSError, ValueError): + continue + job = FormulaExport.objects.filter(pk=job_id).first() + if ( + job is not None + and job.status == "succeeded" + and job.path == str(path) + and job.expires_at is not None + and job.expires_at > now + ): + continue + try: + path.unlink() + removed += 1 + if job is not None and job.path == str(path): + job.path = "" + job.save(update_fields=["path"]) + self.stdout.write(f"Reclaimed leftover workbook of job {job.pk}") + except OSError as exc: + self.stderr.write(f"Could not reclaim orphan export {path.stem}: {exc}") + return removed diff --git a/tests/test_formula_cleanup.py b/tests/test_formula_cleanup.py index 19d5475..33acc9d 100644 --- a/tests/test_formula_cleanup.py +++ b/tests/test_formula_cleanup.py @@ -1,4 +1,5 @@ import os +import uuid from datetime import timedelta import pytest @@ -27,6 +28,12 @@ def cleanup_root(settings, tmp_path): return tmp_path / "formula-exports", dataset, user +def _aged(path, hours=25): + stamp = (timezone.now() - timedelta(hours=hours)).timestamp() + os.utime(path, (stamp, stamp)) + return path + + def test_cleanup_expires_jobs_and_removes_files(cleanup_root): root, dataset, user = cleanup_root path = root / str(user.pk) / "expired.xlsx" @@ -55,8 +62,7 @@ def test_cleanup_removes_only_stale_temporary_files(cleanup_root): stale, recent = user_dir / "stale.tmp", user_dir / "recent.tmp" stale.touch() recent.touch() - old = timezone.now().timestamp() - 25 * 3600 - os.utime(stale, (old, old)) + _aged(stale) call_command("cleanup_formula_exports") @@ -81,3 +87,71 @@ def test_cleanup_does_not_delete_files_outside_export_root(cleanup_root, tmp_pat job.refresh_from_db() assert job.status == "expired" assert outside.read_bytes() == b"keep" + + +def test_cleanup_reclaims_failed_job_leftover_after_retention(cleanup_root): + root, dataset, user = cleanup_root + job = FormulaExport.objects.create(dataset=dataset, user=user, status="failed") + path = root / str(user.pk) / f"{job.pk}.xlsx" + path.parent.mkdir(parents=True) + path.write_bytes(b"leftover") + job.path = str(path) + job.save(update_fields=["path"]) + _aged(path) + + call_command("cleanup_formula_exports") + + job.refresh_from_db() + assert not path.exists() + assert job.path == "" + + +def test_cleanup_keeps_workbook_of_unexpired_succeeded_job(cleanup_root): + root, dataset, user = cleanup_root + job = FormulaExport.objects.create( + dataset=dataset, + user=user, + status="succeeded", + expires_at=timezone.now() + timedelta(hours=1), + ) + path = root / str(user.pk) / f"{job.pk}.xlsx" + path.parent.mkdir(parents=True) + path.write_bytes(b"active") + job.path = str(path) + job.save(update_fields=["path"]) + _aged(path) + + call_command("cleanup_formula_exports") + + assert path.read_bytes() == b"active" + job.refresh_from_db() + assert job.path == str(path) + + +def test_cleanup_reclaims_orphan_whose_job_row_was_deleted(cleanup_root): + root, _, user = cleanup_root + user_dir = root / str(user.pk) + user_dir.mkdir(parents=True) + orphan = user_dir / f"{uuid.uuid4()}.xlsx" + orphan.write_bytes(b"orphan") + recent = user_dir / f"{uuid.uuid4()}.xlsx" + recent.write_bytes(b"recent orphan") + _aged(orphan) + + call_command("cleanup_formula_exports") + + assert not orphan.exists() + assert recent.exists() + + +def test_cleanup_ignores_non_uuid_workbook_names(cleanup_root): + root, _, user = cleanup_root + user_dir = root / str(user.pk) + user_dir.mkdir(parents=True) + odd = user_dir / "notes.xlsx" + odd.write_bytes(b"keep") + _aged(odd) + + call_command("cleanup_formula_exports") + + assert odd.exists() diff --git a/tests/test_formula_export_reliability.py b/tests/test_formula_export_reliability.py index 9e15b8d..9376041 100644 --- a/tests/test_formula_export_reliability.py +++ b/tests/test_formula_export_reliability.py @@ -1,3 +1,4 @@ +import os import stat from decimal import Decimal from io import BytesIO @@ -306,6 +307,62 @@ def test_formula_export_retry_after_failure_succeeds(workspace, monkeypatch): assert not list(Path(job.path).parent.glob(f"{job.pk}.*.tmp")) +def test_cleanup_failure_is_logged_and_reclaimed_after_permissions_restored( + workspace, settings, monkeypatch, caplog +): + import logging as logging_module + from datetime import timedelta + + from django.core.management import call_command + from django.utils import timezone + + from finance.exporting import process_formula_export + + ds, *_ = workspace + job = _queued_job(ds) + user_dir = Path(settings.MEDIA_ROOT) / "formula-exports" / str(job.user_id) + user_dir.mkdir(parents=True) + target = user_dir / f"{job.pk}.xlsx" + target.write_bytes(b"stale workbook") + other = FormulaExport.objects.create( + dataset=ds, + user=ds.created_by, + context={"start": "2026-08", "end": "2026-08", "comparison": "yoy", "group": "department"}, + status="succeeded", + expires_at=timezone.now() + timedelta(hours=1), + ) + kept = user_dir / f"{other.pk}.xlsx" + other.path = str(kept) + other.save(update_fields=["path"]) + kept.write_bytes(b"active workbook") + + def fail(*args, **kwargs): + args[-1].write(b"partial") + os.chmod(user_dir, 0o500) + raise ValueError("controlled export failure") + + monkeypatch.setattr("finance.exporting.build_formula_workbook", fail) + with pytest.raises(ValueError, match="controlled export failure"): + process_formula_export(job.pk) + job.refresh_from_db() + assert job.status == "failed" + assert job.path == str(target) + warnings = [r for r in caplog.records if r.levelno == logging_module.WARNING] + assert warnings and all(str(job.pk) in record.getMessage() for record in warnings) + + os.chmod(user_dir, 0o700) + old = (timezone.now() - timedelta(hours=25)).timestamp() + for leftover in [target, *user_dir.glob(f"{job.pk}.*.tmp"), kept]: + os.utime(leftover, (old, old)) + call_command("cleanup_formula_exports") + + job.refresh_from_db() + assert not target.exists() + assert not list(user_dir.glob(f"{job.pk}.*.tmp")) + assert job.path == "" + assert kept.read_bytes() == b"active workbook" + + def test_formula_workbook_rejects_unclassified_label_collision(workspace): from decimal import Decimal