fix(cleanup): F02 清理失败记录含任务编号的告警,孤儿 xlsx 按保留期回收
- process_formula_export 失败清理受阻时记录不含敏感数据的 WARNING(任务编号+文件名), 并保留 job.path 使残留文件仍可追踪,而不是清空后失联 - cleanup_formula_exports 新增孤儿 .xlsx 回收:超过 24h 保留期且不属于任何 未过期成功任务的文件统一回收(覆盖失败任务残留、任务行被级联删除两种来源), 不触碰仍处于活动任务的文件 - 测试:注入 unlink 权限错误→任务失败状态仍保存+告警可见;恢复权限后回收成功 且其他任务文件不受影响;失败任务残留/任务行已删/未过期成功/近期文件/非 UUID 命名
This commit is contained in:
@@ -675,6 +675,13 @@ def process_formula_export(export_id):
|
|||||||
job.status, job.error, job.finished_at, job.expires_at = "failed", str(exc)[:500], timezone.now(), None
|
job.status, job.error, job.finished_at, job.expires_at = "failed", str(exc)[:500], timezone.now(), None
|
||||||
# 只清理本次登记且确认位于受控目录内的文件;目录被换成符号链接时拒绝删除。
|
# 只清理本次登记且确认位于受控目录内的文件;目录被换成符号链接时拒绝删除。
|
||||||
leftover = [path for path in created if _unlink_created_file(path) == "blocked"]
|
leftover = [path for path in created if _unlink_created_file(path) == "blocked"]
|
||||||
|
if leftover:
|
||||||
|
# 日志只含任务编号与文件名,不包含敏感数据;残留文件由清理命令按保留期回收。
|
||||||
|
logger.warning(
|
||||||
|
"公式导出任务 %s 失败后清理文件受阻,待清理命令回收:%s",
|
||||||
|
job.pk,
|
||||||
|
"、".join(path.name for path in leftover),
|
||||||
|
)
|
||||||
job.path = str(target) if target in leftover else ""
|
job.path = str(target) if target in leftover else ""
|
||||||
job.save(update_fields=["status", "error", "path", "finished_at", "expires_at"])
|
job.save(update_fields=["status", "error", "path", "finished_at", "expires_at"])
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import time
|
import time
|
||||||
|
import uuid
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -9,9 +10,11 @@ from django.utils import timezone
|
|||||||
|
|
||||||
from finance.models import FormulaExport
|
from finance.models import FormulaExport
|
||||||
|
|
||||||
|
ORPHAN_RETENTION_HOURS = 24
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = "Remove expired formula workbooks and stale temporary files"
|
help = "Remove expired formula workbooks and reclaim orphan files"
|
||||||
|
|
||||||
def add_arguments(self, parser):
|
def add_arguments(self, parser):
|
||||||
parser.add_argument("--loop", action="store_true")
|
parser.add_argument("--loop", action="store_true")
|
||||||
@@ -19,8 +22,10 @@ class Command(BaseCommand):
|
|||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
while True:
|
while True:
|
||||||
jobs, files = self.clean_expired()
|
jobs, files, orphans = self.clean_expired()
|
||||||
self.stdout.write(f"Expired {jobs} jobs; removed {files} files")
|
self.stdout.write(
|
||||||
|
f"Expired {jobs} jobs; removed {files} files; reclaimed {orphans} orphan files"
|
||||||
|
)
|
||||||
if not options["loop"]:
|
if not options["loop"]:
|
||||||
return
|
return
|
||||||
time.sleep(max(60, options["interval"]))
|
time.sleep(max(60, options["interval"]))
|
||||||
@@ -31,7 +36,7 @@ class Command(BaseCommand):
|
|||||||
raise ValueError("公式导出目录不能是符号链接")
|
raise ValueError("公式导出目录不能是符号链接")
|
||||||
root = export_root.resolve()
|
root = export_root.resolve()
|
||||||
now = timezone.now()
|
now = timezone.now()
|
||||||
jobs = files = 0
|
jobs = files = orphans = 0
|
||||||
|
|
||||||
for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator():
|
for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator():
|
||||||
path = Path(job.path) if job.path else None
|
path = Path(job.path) if job.path else None
|
||||||
@@ -61,5 +66,41 @@ class Command(BaseCommand):
|
|||||||
files += 1
|
files += 1
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}")
|
self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}")
|
||||||
|
orphans += self._reclaim_orphan_workbooks(user_dir, now)
|
||||||
|
|
||||||
return jobs, files
|
return jobs, files, orphans
|
||||||
|
|
||||||
|
def _reclaim_orphan_workbooks(self, user_dir, now):
|
||||||
|
"""回收超过保留期且不再属于任何有效导出任务的 .xlsx 孤儿文件。
|
||||||
|
|
||||||
|
有效任务指状态 succeeded、路径一致且尚未过期的任务;失败任务残留的文件和任务行
|
||||||
|
已被删除(如数据集级联删除)的文件在保留期后回收,绝不触碰活动任务的文件。
|
||||||
|
"""
|
||||||
|
retention = (now - timedelta(hours=ORPHAN_RETENTION_HOURS)).timestamp()
|
||||||
|
removed = 0
|
||||||
|
for path in user_dir.glob("*.xlsx"):
|
||||||
|
try:
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_mtime > retention:
|
||||||
|
continue
|
||||||
|
job_id = uuid.UUID(path.stem)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
continue
|
||||||
|
job = FormulaExport.objects.filter(pk=job_id).first()
|
||||||
|
if (
|
||||||
|
job is not None
|
||||||
|
and job.status == "succeeded"
|
||||||
|
and job.path == str(path)
|
||||||
|
and job.expires_at is not None
|
||||||
|
and job.expires_at > now
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
path.unlink()
|
||||||
|
removed += 1
|
||||||
|
if job is not None and job.path == str(path):
|
||||||
|
job.path = ""
|
||||||
|
job.save(update_fields=["path"])
|
||||||
|
self.stdout.write(f"Reclaimed leftover workbook of job {job.pk}")
|
||||||
|
except OSError as exc:
|
||||||
|
self.stderr.write(f"Could not reclaim orphan export {path.stem}: {exc}")
|
||||||
|
return removed
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
|
import uuid
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -27,6 +28,12 @@ def cleanup_root(settings, tmp_path):
|
|||||||
return tmp_path / "formula-exports", dataset, user
|
return tmp_path / "formula-exports", dataset, user
|
||||||
|
|
||||||
|
|
||||||
|
def _aged(path, hours=25):
|
||||||
|
stamp = (timezone.now() - timedelta(hours=hours)).timestamp()
|
||||||
|
os.utime(path, (stamp, stamp))
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
def test_cleanup_expires_jobs_and_removes_files(cleanup_root):
|
def test_cleanup_expires_jobs_and_removes_files(cleanup_root):
|
||||||
root, dataset, user = cleanup_root
|
root, dataset, user = cleanup_root
|
||||||
path = root / str(user.pk) / "expired.xlsx"
|
path = root / str(user.pk) / "expired.xlsx"
|
||||||
@@ -55,8 +62,7 @@ def test_cleanup_removes_only_stale_temporary_files(cleanup_root):
|
|||||||
stale, recent = user_dir / "stale.tmp", user_dir / "recent.tmp"
|
stale, recent = user_dir / "stale.tmp", user_dir / "recent.tmp"
|
||||||
stale.touch()
|
stale.touch()
|
||||||
recent.touch()
|
recent.touch()
|
||||||
old = timezone.now().timestamp() - 25 * 3600
|
_aged(stale)
|
||||||
os.utime(stale, (old, old))
|
|
||||||
|
|
||||||
call_command("cleanup_formula_exports")
|
call_command("cleanup_formula_exports")
|
||||||
|
|
||||||
@@ -81,3 +87,71 @@ def test_cleanup_does_not_delete_files_outside_export_root(cleanup_root, tmp_pat
|
|||||||
job.refresh_from_db()
|
job.refresh_from_db()
|
||||||
assert job.status == "expired"
|
assert job.status == "expired"
|
||||||
assert outside.read_bytes() == b"keep"
|
assert outside.read_bytes() == b"keep"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_reclaims_failed_job_leftover_after_retention(cleanup_root):
|
||||||
|
root, dataset, user = cleanup_root
|
||||||
|
job = FormulaExport.objects.create(dataset=dataset, user=user, status="failed")
|
||||||
|
path = root / str(user.pk) / f"{job.pk}.xlsx"
|
||||||
|
path.parent.mkdir(parents=True)
|
||||||
|
path.write_bytes(b"leftover")
|
||||||
|
job.path = str(path)
|
||||||
|
job.save(update_fields=["path"])
|
||||||
|
_aged(path)
|
||||||
|
|
||||||
|
call_command("cleanup_formula_exports")
|
||||||
|
|
||||||
|
job.refresh_from_db()
|
||||||
|
assert not path.exists()
|
||||||
|
assert job.path == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_keeps_workbook_of_unexpired_succeeded_job(cleanup_root):
|
||||||
|
root, dataset, user = cleanup_root
|
||||||
|
job = FormulaExport.objects.create(
|
||||||
|
dataset=dataset,
|
||||||
|
user=user,
|
||||||
|
status="succeeded",
|
||||||
|
expires_at=timezone.now() + timedelta(hours=1),
|
||||||
|
)
|
||||||
|
path = root / str(user.pk) / f"{job.pk}.xlsx"
|
||||||
|
path.parent.mkdir(parents=True)
|
||||||
|
path.write_bytes(b"active")
|
||||||
|
job.path = str(path)
|
||||||
|
job.save(update_fields=["path"])
|
||||||
|
_aged(path)
|
||||||
|
|
||||||
|
call_command("cleanup_formula_exports")
|
||||||
|
|
||||||
|
assert path.read_bytes() == b"active"
|
||||||
|
job.refresh_from_db()
|
||||||
|
assert job.path == str(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_reclaims_orphan_whose_job_row_was_deleted(cleanup_root):
|
||||||
|
root, _, user = cleanup_root
|
||||||
|
user_dir = root / str(user.pk)
|
||||||
|
user_dir.mkdir(parents=True)
|
||||||
|
orphan = user_dir / f"{uuid.uuid4()}.xlsx"
|
||||||
|
orphan.write_bytes(b"orphan")
|
||||||
|
recent = user_dir / f"{uuid.uuid4()}.xlsx"
|
||||||
|
recent.write_bytes(b"recent orphan")
|
||||||
|
_aged(orphan)
|
||||||
|
|
||||||
|
call_command("cleanup_formula_exports")
|
||||||
|
|
||||||
|
assert not orphan.exists()
|
||||||
|
assert recent.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_ignores_non_uuid_workbook_names(cleanup_root):
|
||||||
|
root, _, user = cleanup_root
|
||||||
|
user_dir = root / str(user.pk)
|
||||||
|
user_dir.mkdir(parents=True)
|
||||||
|
odd = user_dir / "notes.xlsx"
|
||||||
|
odd.write_bytes(b"keep")
|
||||||
|
_aged(odd)
|
||||||
|
|
||||||
|
call_command("cleanup_formula_exports")
|
||||||
|
|
||||||
|
assert odd.exists()
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import os
|
||||||
import stat
|
import stat
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
@@ -306,6 +307,62 @@ def test_formula_export_retry_after_failure_succeeds(workspace, monkeypatch):
|
|||||||
assert not list(Path(job.path).parent.glob(f"{job.pk}.*.tmp"))
|
assert not list(Path(job.path).parent.glob(f"{job.pk}.*.tmp"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_failure_is_logged_and_reclaimed_after_permissions_restored(
|
||||||
|
workspace, settings, monkeypatch, caplog
|
||||||
|
):
|
||||||
|
import logging as logging_module
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from django.core.management import call_command
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from finance.exporting import process_formula_export
|
||||||
|
|
||||||
|
ds, *_ = workspace
|
||||||
|
job = _queued_job(ds)
|
||||||
|
user_dir = Path(settings.MEDIA_ROOT) / "formula-exports" / str(job.user_id)
|
||||||
|
user_dir.mkdir(parents=True)
|
||||||
|
target = user_dir / f"{job.pk}.xlsx"
|
||||||
|
target.write_bytes(b"stale workbook")
|
||||||
|
other = FormulaExport.objects.create(
|
||||||
|
dataset=ds,
|
||||||
|
user=ds.created_by,
|
||||||
|
context={"start": "2026-08", "end": "2026-08", "comparison": "yoy", "group": "department"},
|
||||||
|
status="succeeded",
|
||||||
|
expires_at=timezone.now() + timedelta(hours=1),
|
||||||
|
)
|
||||||
|
kept = user_dir / f"{other.pk}.xlsx"
|
||||||
|
other.path = str(kept)
|
||||||
|
other.save(update_fields=["path"])
|
||||||
|
kept.write_bytes(b"active workbook")
|
||||||
|
|
||||||
|
def fail(*args, **kwargs):
|
||||||
|
args[-1].write(b"partial")
|
||||||
|
os.chmod(user_dir, 0o500)
|
||||||
|
raise ValueError("controlled export failure")
|
||||||
|
|
||||||
|
monkeypatch.setattr("finance.exporting.build_formula_workbook", fail)
|
||||||
|
with pytest.raises(ValueError, match="controlled export failure"):
|
||||||
|
process_formula_export(job.pk)
|
||||||
|
job.refresh_from_db()
|
||||||
|
assert job.status == "failed"
|
||||||
|
assert job.path == str(target)
|
||||||
|
warnings = [r for r in caplog.records if r.levelno == logging_module.WARNING]
|
||||||
|
assert warnings and all(str(job.pk) in record.getMessage() for record in warnings)
|
||||||
|
|
||||||
|
os.chmod(user_dir, 0o700)
|
||||||
|
old = (timezone.now() - timedelta(hours=25)).timestamp()
|
||||||
|
for leftover in [target, *user_dir.glob(f"{job.pk}.*.tmp"), kept]:
|
||||||
|
os.utime(leftover, (old, old))
|
||||||
|
call_command("cleanup_formula_exports")
|
||||||
|
|
||||||
|
job.refresh_from_db()
|
||||||
|
assert not target.exists()
|
||||||
|
assert not list(user_dir.glob(f"{job.pk}.*.tmp"))
|
||||||
|
assert job.path == ""
|
||||||
|
assert kept.read_bytes() == b"active workbook"
|
||||||
|
|
||||||
|
|
||||||
def test_formula_workbook_rejects_unclassified_label_collision(workspace):
|
def test_formula_workbook_rejects_unclassified_label_collision(workspace):
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user