diff --git a/docs/fix-record-2026-09-26.md b/docs/fix-record-2026-09-26.md new file mode 100644 index 0000000..3a59e13 --- /dev/null +++ b/docs/fix-record-2026-09-26.md @@ -0,0 +1,30 @@ +# 剩余问题修复记录 · 2026-09-26 + +对应 `docs/remaining-issues-2026-09-26.html` 第三节(F01–F08)的修复交付记录。 +分支:`fix/remaining-issues-2026-09-26`(基线 `0ac0c4e` 起逐项小步提交)。 + +## 修复状态 + +| 编号 | 状态 | 提交 | 说明 | +| --- | --- | --- | --- | +| F01 · P0 · 符号链接清理 | 已修复 | `36fd5f3` | 根目录/用户目录/目标文件先 lstat 完整校验(拒绝符号链接与非目录/非普通文件),校验失败不删除任何磁盘文件;失败清理只处理本次登记的受控文件并通过 `O_NOFOLLOW` 目录描述符 unlink,目录被换成符号链接时拒绝删除。测试覆盖用户目录链接、目标链接、运行中目录被换成链接(受保护文件字节不变)、失败后重试成功。 | +| F02 · P1 · 清理失败与孤儿 | 已修复 | `54dc58d` | 清理受阻时记录不含敏感数据的 WARNING(任务编号+文件名)并保留 `job.path` 使残留可追踪;`cleanup_formula_exports` 新增孤儿 `.xlsx` 回收:超过 24h 保留期且不属于未过期成功任务的文件统一回收,不触碰活动任务。测试含 unlink 权限错误注入→告警→恢复权限后回收且其他任务文件不受影响。 | +| F03 · P1 · 证据弹窗卡加载 | 已修复 | `6cf236a` | 同期间同页点击直接返回;第二页点击当前期间回第一页刷新;失败显示错误+重试按钮;空结果显示空态行。E2E 覆盖重复点击、快速切换、第二页切期间、失败重试、空列表。 | +| F04 · P1 · 载入定义丢上下文 | 已修复 | `9a5ab26` | 载入定义时以“用户消息+助手确认”把名称/含义/范围/缺失处理/表达式写入可见对话上下文(需勾选同意才发送),修改后仍需重新确认;选择空选项同步清理定义与上下文。服务端单条消息上限 1000→2000 容纳完整定义。E2E 验证模型请求包含原定义且“乘以 100”修改生效。 | +| F05 · P1 · 定义库并发与审计 | 已修复 | `0aaa87e` | 迁移 0005 增加 `revision`;保存在单事务内锁定用户所属行:配额(50/数据集)原子化、同名更新必须携带匹配 revision(409 冲突)、审计与变更同事务回滚;保存限流 30/h。前端保存同名定义携带 revision 并明示覆盖行为。PostgreSQL 并发配额用例作为独立门禁(本机 SQLite 自动跳过)。 | +| F06 · P1 · 工作簿可解释性 | 已修复 | `9962573` | 参数页“产品毛利率”改为只读快照(可编辑列全部为“否”),写明调整位置是计算明细 MarginRate 列;参数页新增维度键-显示名完整对照;使用说明新增输入字典(Revenue/Cost/GrossInput/MarginRate/ListAmount 的单位、来源字段、修改位置、缺失含义)与固定结构边界行。 | +| F07 · P1 · 对账容差 | 已修复 | `649fec4` | 对账检查页新增容差列:人数严格相等、金额 0.01、比率 0.0001,状态公式按 `ABS(差异)<=容差` 判定;自定义列用 `to_decimal` 白名单表达式解释器(不执行代码)计算 Decimal 服务器参考,不再标注“需业务验收”;参数页写明容差规则与“调整需财务负责人批准”。 | +| F08 · P1 · 撤权与生命周期 | 已修复 | `5ebe83a` | worker 执行前重新核验账号 `is_active` 与 admin/analyst 角色(撤权/停用后排队任务失败、不发布);导出状态/下载接口补齐角色白名单(降为 viewer 后 403);`import_worker` 对超过 15 分钟的 processing 任务重新认领(PG 依赖咨询锁立即恢复),崩溃重启不再无限停留。 | + +## 本轮完整测试记录(2026-09-26,SQLite 隔离环境) + +- **后端:87 passed,7 skipped,90.3 秒;finance 覆盖率 88.50%(85% 门槛通过)。** + 跳过项:4 项 LibreOffice 重算(本机未安装 soffice,环境门禁,CI 中执行)、1 项 PostgreSQL 并发配额(依赖 PG 行锁,生产库单独验证)、1 项原附件全量 UI(未设置 `REFERENCE_DATASET_NAME`)。 +- **浏览器 E2E:3 passed,1 skipped,21.0 秒。** 新隔离数据库(`var/e2e-test.sqlite3` + 独立 MEDIA_ROOT),覆盖追问、保存定义、关闭重开、载入定义修改后生成(乘以 100)、下载,含 F03 全部交互回归(重复点击/快速切换/失败重试/空列表/翻页)、桌面与手机。 +- Ruff check / ruff format、TypeScript(tsc --noEmit)、Vite 构建、Django check、`makemigrations --check`(迁移一致)、collectstatic:全部通过。 +- `pip-audit`:无已知漏洞。`npm audit --omit=dev`:本机 registry 访问失败(网络受限,CI 中执行),记录为环境限制项。 + +## 仍开放的事项(与报告一致,非本轮代码范围) + +- 第四节功能缺口(网页主汇总表新列、草稿续聊、原附件 332,830 行导出、高基数与可访问性专项等)与第五节外部门禁(目标 Excel 实机、财务口径签字、AI 真实联调、生产基础设施、PostgreSQL/SLA、安全运营)仍按原报告执行,责任人与关闭证据不变。 +- F07 容差数值(金额 0.01 / 比率 0.0001)为工程默认,仍需财务负责人确认或调整后才视为财务口径关闭。 diff --git a/finance/exporting.py b/finance/exporting.py index 28e5e18..e24fcab 100644 --- a/finance/exporting.py +++ b/finance/exporting.py @@ -1,7 +1,9 @@ import hashlib import hmac +import logging import os import secrets +import stat import warnings from datetime import timedelta from decimal import Decimal, InvalidOperation @@ -19,6 +21,7 @@ from .formula_columns import ( RESERVED_DATA_HEADERS, ensure_available_metrics, metrics_used, + to_decimal, to_excel, validate_column, ) @@ -28,6 +31,75 @@ FORMULA_VERSION = "3" MAX_ROWS = 100_000 MAX_GROUPS = 200 +# 对账比较规则:人数严格相等;金额 0.01;比率 0.0001(一个基点)。 +# 容差属财务口径,调整需财务负责人批准后方可修改。 +RECONCILE_TOLERANCES = { + "customers": Decimal("0"), + "revenue": Decimal("0.01"), + "cost": Decimal("0.01"), + "gross": Decimal("0.01"), + "self_revenue": Decimal("0.01"), + "basket": Decimal("0.0001"), + "margin": Decimal("0.0001"), + "discount_rate": Decimal("0.0001"), + "upgrade_rate": Decimal("0.0001"), + "list_basket": Decimal("0.0001"), + "roi": Decimal("0.0001"), +} +CUSTOM_COLUMN_TOLERANCE = Decimal("0.01") + +logger = logging.getLogger(__name__) + + +def _validated_export_directory(path, description): + """导出目录必须不存在或为真实目录;符号链接一律拒绝并保持文件原样。""" + try: + info = os.lstat(path) + except FileNotFoundError: + path.mkdir(parents=True, mode=0o700) + info = os.lstat(path) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + raise ValueError(f"{description}必须是真实目录,不能是符号链接") + os.chmod(path, 0o700) + return path + + +def _validated_export_target(user_dir, job): + """目标文件若已存在必须是普通文件;符号链接或其他类型一律拒绝覆盖。""" + target = user_dir / f"{job.pk}.xlsx" + try: + info = os.lstat(target) + except FileNotFoundError: + return target + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + raise ValueError("公式导出目标路径已存在且不是普通文件,已拒绝生成") + return target + + +def _unlink_created_file(path): + """只通过不跟随链接的目录句柄删除受控目录内的普通文件。 + + 返回 "removed"(已删除)、"missing"(本就不存在)或 "blocked"(无法安全删除, + 例如目录已被换成符号链接或权限不足),调用方据此保留失败任务与磁盘的一致性。 + """ + try: + dir_fd = os.open(path.parent, os.O_RDONLY | os.O_NOFOLLOW) + except OSError: + return "blocked" + try: + try: + info = os.lstat(path.name, dir_fd=dir_fd) + except FileNotFoundError: + return "missing" + if not stat.S_ISREG(info.st_mode): + return "blocked" + os.unlink(path.name, dir_fd=dir_fd) + return "removed" + except OSError: + return "blocked" + finally: + os.close(dir_fd) + def _excel_number(value): if value is None: @@ -408,7 +480,12 @@ def build_formula_workbook(ds, member, export_context, output): ], ["ROI 口径", mapping["roi_formula"], "按导入映射保存的公式分支", "否"], ["自营产品集合", _text_cell(parameters, ", ".join(products)), "升单率和自营收入口径", "否"], - ["产品毛利率", str(rates), "直接毛利缺失时的可编辑计算输入", "是"], + [ + "产品毛利率(快照)", + _text_cell(parameters, str(rates)), + "只读快照;导出时已按行写入计算明细 MarginRate 列,调整毛利率请在计算明细页按行修改", + "否", + ], [ "Excel 数值精度", "IEEE-754 双精度", @@ -417,8 +494,23 @@ def build_formula_workbook(ds, member, export_context, output): ], ["重算范围", f"最多 {MAX_ROWS:,} 行、{MAX_GROUPS} 组", "超限需缩小导出范围", "否"], ["兼容性", "SUMIFS/COUNTIFS", "移除动态数组人数公式;仍需目标 Excel 桌面版本验收", "否"], + [ + "对账容差", + "人数严格相等;金额 0.01;比率 0.0001", + "对账检查页按指标类型使用独立容差;自定义列按金额容差 0.01。容差调整需财务负责人批准", + "否", + ], ]: parameters.append(row) + for d in dims: + parameters.append( + [ + _text_cell(parameters, f"维度 {d['key']}"), + _text_cell(parameters, d["name"]), + "维度键与显示名对照(只读);计算明细与原始证据的列名即维度键", + "否", + ] + ) for definition in custom_columns: parameters.append( [ @@ -429,10 +521,33 @@ def build_formula_workbook(ds, member, export_context, output): ] ) instructions.append(_styled_row(instructions, ["主题", "说明"], green, white)) + + def source_name(role): + return next((c["name"] for c in mapping["columns"] if c["role"] == role), "未映射") + + editable_inputs = [ + ("Revenue", "实际收入", "金额", source_name("revenue"), "空值行不参与求和,汇总按现有行计算"), + ("Cost", "成本", "金额", source_name("cost"), "任一行缺失时本组成本及 ROI 等依赖指标不可用"), + ("GrossInput", "直接毛利", "金额", source_name("gross"), "空值行改按产品毛利率(lookup)计算毛利"), + ( + "MarginRate", + "产品毛利率", + "小数比率", + "导入确认的产品毛利率表", + "仅 GrossSource=lookup 的行参与毛利计算;0.2 表示 20%", + ), + ( + "ListAmount", + "原价金额", + "金额", + source_name("list_amount"), + "用于扣率与原价客单价;缺失则相关指标不可用", + ), + ] for row in [ ["用途", "公式分析工作簿;本地修改不回写系统。"], ["行粒度", "计算明细每行对应一个 Fact,不等同患者就诊数。"], - ["维度列", "列名为数据导入映射中的维度键;显示名称见本页分组维度。"], + ["维度列", "列名为数据导入映射中的维度键;键与显示名对照见“参数与口径”页的维度行。"], ["毛利", "直接毛利优先,否则收入×产品毛利率;无法计算则不可用。毛利贡献不代表净利润。"], ["ROI", "按选定口径;缺成本、成本非正或所需指标缺失时不可用。"], ["人数", "客户 token 跨行去重;缺客户编码时不可用。"], @@ -443,6 +558,25 @@ def build_formula_workbook(ds, member, export_context, output): ["变化率", "差额/ABS(基期);基期为0或期间缺失时不可用。"], ["精度", "原始 Decimal 以文本留存;Excel 双精度存在细微误差可能。"], ["隐私", "客户标识替换为本文件随机 token,文件仍按敏感财务信息保护。"], + [ + "输入字典", + "以下输入可在“计算明细”页直接修改,全部汇总公式与对账实时引用;字段、单位、来源与缺失含义逐项列出。", + ], + *[ + [ + _text_cell(instructions, f"输入 {header}({label})"), + _text_cell( + instructions, + f"单位:{unit};来源:{source};修改位置:计算明细页 {header} 列;缺失含义:{missing}。", + ), + ] + for header, label, unit, source, missing in editable_inputs + ], + [ + "固定结构", + "Period、维度列、CustomerToken、GrossSource、SelfOperated、CustomerPresent、RowCount 与“人数去重”页均为固定快照;" + "修改或增删这些列/行会使汇总、去重与对账失真,需要改变结构时请重新导出。", + ], ]: instructions.append(row) for definition in custom_columns: @@ -546,11 +680,18 @@ def build_formula_workbook(ds, member, export_context, output): reconcile.append( _styled_row( reconcile, - ["指标", "分组", "期间侧", "服务器 Decimal 参考", "公式结果", "差异", "状态"], + ["指标", "分组", "期间侧", "服务器 Decimal 参考", "公式结果", "差异", "容差", "状态"], green, white, ) ) + + def _reconcile_status(row_number): + return ( + f'=IF(D{row_number}="","无参考",IF(ISNUMBER(F{row_number}),' + f'IF(ABS(F{row_number})<=G{row_number},"通过","待复核"),"待复核"))' + ) + reconcile_row = 2 for label in labels: for side_name, periods, col in (("本期", ctx["current"], "C"), ("基期", ctx["baseline"], "D")): @@ -567,21 +708,25 @@ def build_formula_workbook(ds, member, export_context, output): "" if value is None else str(value), f"='分析汇总'!{col}{formula_row}", f'=IFERROR(E{reconcile_row}-D{reconcile_row},"")', - f'=IF(D{reconcile_row}="","无参考",IF(ISNUMBER(F{reconcile_row}),IF(ABS(F{reconcile_row})<0.01,"通过","待复核"),"待复核"))', + float(RECONCILE_TOLERANCES[key]), + _reconcile_status(reconcile_row), ] ) reconcile_row += 1 for index, definition in enumerate(custom_columns): formula_row = metric_rows[(label, f"custom_{index}")] + # 自定义列用 Decimal 白名单解释器计算服务器参考,不再依赖业务人工验收 + reference_value = to_decimal(definition["expression"], reference) reconcile.append( [ _text_cell(reconcile, definition["name"]), _text_cell(reconcile, label), side_name, - "自定义公式无服务器参考", + "" if reference_value is None else str(reference_value), f"='分析汇总'!{col}{formula_row}", - "", - "需业务验收", + f'=IFERROR(E{reconcile_row}-D{reconcile_row},"")', + float(CUSTOM_COLUMN_TOLERANCE), + _reconcile_status(reconcile_row), ] ) reconcile_row += 1 @@ -594,41 +739,51 @@ def build_formula_workbook(ds, member, export_context, output): def process_formula_export(export_id): job = FormulaExport.objects.select_related("dataset", "user").get(pk=export_id) - temporary = None target = None + created = [] try: - root = Path(settings.MEDIA_ROOT) / "formula-exports" - if root.is_symlink(): - raise ValueError("公式导出目录不能是符号链接") - root.mkdir(parents=True, exist_ok=True, mode=0o700) - root.chmod(0o700) - target = root / str(job.user_id) / f"{job.pk}.xlsx" - target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - if target.parent.is_symlink() or target.is_symlink(): - raise ValueError("公式导出路径不能是符号链接") - target.parent.chmod(0o700) - temporary = target.with_name(f"{job.pk}.{secrets.token_hex(8)}.tmp") member = Membership.objects.get(user=job.user, organization=job.dataset.organization) + # 执行阶段重新核验角色与账号状态:撤权或停用后排队中的任务不得继续发布敏感工作簿。 + if not job.user.is_active: + raise PermissionError("账号已停用,公式导出已拒绝执行") + if member.role not in ("admin", "analyst"): + raise PermissionError("账号角色已变更,公式导出已拒绝执行") if set(member.departments) != set(job.departments): raise PermissionError("科室授权已变更,请重新提交导出") + root = _validated_export_directory(Path(settings.MEDIA_ROOT) / "formula-exports", "公式导出目录") + user_dir = _validated_export_directory(root / str(job.user_id), "公式导出用户目录") + target = _validated_export_target(user_dir, job) + # 路径完整验证通过后才把目标登记为可清理文件;此前任何失败都不会删除磁盘文件。 + created.append(target) + temporary = target.with_name(f"{job.pk}.{secrets.token_hex(8)}.tmp") descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + created.append(temporary) with os.fdopen(descriptor, "wb") as file: stats = build_formula_workbook(job.dataset, member, job.context, file) - temporary.replace(target) + os.replace(temporary, target) + created = [target] job.path, job.status, job.error = str(target), "succeeded", "" job.finished_at = timezone.now() job.expires_at = timezone.now() + timedelta(hours=24) job.save(update_fields=["path", "status", "error", "finished_at", "expires_at"]) return stats except Exception as exc: - for path in (temporary, target): - try: - if path is not None and not path.is_symlink(): - path.unlink(missing_ok=True) - except OSError: - pass - job.status, job.error, job.path = "failed", str(exc)[:500], "" - job.finished_at, job.expires_at = timezone.now(), None + job.status, job.error, job.finished_at, job.expires_at = ( + "failed", + str(exc)[:500], + timezone.now(), + None, + ) + # 只清理本次登记且确认位于受控目录内的文件;目录被换成符号链接时拒绝删除。 + leftover = [path for path in created if _unlink_created_file(path) == "blocked"] + if leftover: + # 日志只含任务编号与文件名,不包含敏感数据;残留文件由清理命令按保留期回收。 + logger.warning( + "公式导出任务 %s 失败后清理文件受阻,待清理命令回收:%s", + job.pk, + "、".join(path.name for path in leftover), + ) + job.path = str(target) if target in leftover else "" job.save(update_fields=["status", "error", "path", "finished_at", "expires_at"]) raise diff --git a/finance/formula_columns.py b/finance/formula_columns.py index 9981ceb..72e22b2 100644 --- a/finance/formula_columns.py +++ b/finance/formula_columns.py @@ -1,4 +1,5 @@ import math +from decimal import Decimal from .analysis import METRICS @@ -107,3 +108,27 @@ def to_excel(node, references): if "number" in node: return str(node["number"]) return f"({to_excel(node['left'], references)}{OPERATORS[node['operator']]}{to_excel(node['right'], references)})" + + +def to_decimal(node, values): + """按已校验的白名单表达式树用 Decimal 计算服务器参考值。 + + 只遍历 metric/number/四则运算节点,不执行任何代码;依赖指标缺失或除零时 + 返回 None,与 Excel 侧 #N/A 的语义一致。 + """ + if "metric" in node: + return values[node["metric"]] + if "number" in node: + return Decimal(str(node["number"])) + left, right = to_decimal(node["left"], values), to_decimal(node["right"], values) + if left is None or right is None: + return None + if node["operator"] == "add": + return left + right + if node["operator"] == "subtract": + return left - right + if node["operator"] == "multiply": + return left * right + if right == 0: + return None + return left / right diff --git a/finance/formula_views.py b/finance/formula_views.py index 898b741..402a09b 100644 --- a/finance/formula_views.py +++ b/finance/formula_views.py @@ -3,14 +3,17 @@ from pathlib import Path import httpx from django.conf import settings +from django.db import IntegrityError, transaction from django.http import FileResponse, JsonResponse from django.utils import timezone from .analysis import METRICS, context, dimensions from .formula_columns import available_metrics, ensure_available_metrics, validate_column -from .models import FormulaDefinition, FormulaExport +from .models import FormulaDefinition, FormulaExport, Membership from .views import APIError, audit, body, dataset, endpoint, limit +MAX_DEFINITIONS_PER_DATASET = 50 + def design_column(messages, ds, ctx): if not (settings.AI_ENABLED and settings.OPENAI_API_KEY and settings.OPENAI_MODEL): @@ -22,7 +25,8 @@ def design_column(messages, ds, ctx): or set(item) != {"role", "content"} or item["role"] not in ("user", "assistant") or not isinstance(item["content"], str) - or len(item["content"]) > 1000 + # 2000 字符容纳载入已保存定义后的完整对话上下文 + or len(item["content"]) > 2000 for item in messages ): raise APIError("对话消息内容无效") @@ -197,6 +201,7 @@ def formula_definitions(request, dataset_id): { "id": str(item.pk), "definition": item.definition, + "revision": item.revision, "updated_at": item.updated_at.isoformat(), } for item in items @@ -204,28 +209,54 @@ def formula_definitions(request, dataset_id): } ) - definition = validate_column(body(request).get("definition")) + data = body(request) + definition = validate_column(data.get("definition")) try: ensure_available_metrics(ds, definition) except ValueError as exc: raise APIError(str(exc)) from exc normalized_name = definition["name"].strip().casefold() - existing = FormulaDefinition.objects.filter( - dataset=ds, user=request.user, normalized_name=normalized_name - ).first() - if not existing and FormulaDefinition.objects.filter(dataset=ds, user=request.user).count() >= 50: - raise APIError("每个数据集最多保存 50 个公式定义") - item, created = FormulaDefinition.objects.update_or_create( - dataset=ds, - user=request.user, - normalized_name=normalized_name, - defaults={"definition": definition}, - ) - audit(request, "formula_definition.save", item.pk, {"dataset": str(ds.pk), "name": definition["name"]}) + limit(f"formula-definition:{request.user.pk}", 30, 3600) + with transaction.atomic(): + # 锁定稳定的用户所属行序列化同一用户的并发保存,使配额检查与写入原子化。 + Membership.objects.select_for_update().get(pk=request.member.pk) + existing = ( + FormulaDefinition.objects.select_for_update() + .filter(dataset=ds, user=request.user, normalized_name=normalized_name) + .first() + ) + if existing: + # 同名更新按 revision 做乐观并发控制,避免两个窗口相互静默覆盖。 + if data.get("revision") != existing.revision: + raise APIError("定义已在其他窗口更新,请刷新后重试", 409) + existing.definition, existing.revision = definition, existing.revision + 1 + existing.save(update_fields=["definition", "revision", "updated_at"]) + item, created = existing, False + else: + if ( + FormulaDefinition.objects.filter(dataset=ds, user=request.user).count() + >= MAX_DEFINITIONS_PER_DATASET + ): + raise APIError(f"每个数据集最多保存 {MAX_DEFINITIONS_PER_DATASET} 个公式定义") + try: + item = FormulaDefinition.objects.create( + dataset=ds, + user=request.user, + normalized_name=normalized_name, + definition=definition, + ) + except IntegrityError as exc: + raise APIError("定义保存发生并发冲突,请重试", 409) from exc + created = True + # 审计与定义变更在同一事务内提交;审计失败时变更一并回滚。 + audit( + request, "formula_definition.save", item.pk, {"dataset": str(ds.pk), "name": definition["name"]} + ) return JsonResponse( { "id": str(item.pk), "definition": item.definition, + "revision": item.revision, "updated_at": item.updated_at.isoformat(), }, status=201 if created else 200, @@ -241,8 +272,9 @@ def formula_definition_delete(request, dataset_id, definition_id): if not item: raise APIError("公式定义不存在", 404) name = item.definition.get("name", "") - audit(request, "formula_definition.delete", item.pk, {"dataset": str(ds.pk), "name": name}) - item.delete() + with transaction.atomic(): + item.delete() + audit(request, "formula_definition.delete", item.pk, {"dataset": str(ds.pk), "name": name}) return JsonResponse({"deleted": True}) @@ -292,7 +324,7 @@ def _owned_job(request, dataset_id, export_id): return job -@endpoint(("GET",)) +@endpoint(("GET",), roles=("admin", "analyst")) def formula_export_status(request, dataset_id, export_id): job = _owned_job(request, dataset_id, export_id) return JsonResponse( @@ -305,7 +337,7 @@ def formula_export_status(request, dataset_id, export_id): ) -@endpoint(("GET",)) +@endpoint(("GET",), roles=("admin", "analyst")) def formula_export_download(request, dataset_id, export_id): job = _owned_job(request, dataset_id, export_id) if job.status != "succeeded" or not job.path or not job.expires_at or job.expires_at <= timezone.now(): diff --git a/finance/management/commands/cleanup_formula_exports.py b/finance/management/commands/cleanup_formula_exports.py index 692e1aa..61cb83f 100644 --- a/finance/management/commands/cleanup_formula_exports.py +++ b/finance/management/commands/cleanup_formula_exports.py @@ -1,4 +1,5 @@ import time +import uuid from datetime import timedelta from pathlib import Path @@ -9,9 +10,11 @@ from django.utils import timezone from finance.models import FormulaExport +ORPHAN_RETENTION_HOURS = 24 + class Command(BaseCommand): - help = "Remove expired formula workbooks and stale temporary files" + help = "Remove expired formula workbooks and reclaim orphan files" def add_arguments(self, parser): parser.add_argument("--loop", action="store_true") @@ -19,8 +22,8 @@ class Command(BaseCommand): def handle(self, *args, **options): while True: - jobs, files = self.clean_expired() - self.stdout.write(f"Expired {jobs} jobs; removed {files} files") + jobs, files, orphans = self.clean_expired() + self.stdout.write(f"Expired {jobs} jobs; removed {files} files; reclaimed {orphans} orphan files") if not options["loop"]: return time.sleep(max(60, options["interval"])) @@ -31,7 +34,7 @@ class Command(BaseCommand): raise ValueError("公式导出目录不能是符号链接") root = export_root.resolve() now = timezone.now() - jobs = files = 0 + jobs = files = orphans = 0 for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator(): path = Path(job.path) if job.path else None @@ -61,5 +64,41 @@ class Command(BaseCommand): files += 1 except OSError as exc: self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}") + orphans += self._reclaim_orphan_workbooks(user_dir, now) - return jobs, files + return jobs, files, orphans + + def _reclaim_orphan_workbooks(self, user_dir, now): + """回收超过保留期且不再属于任何有效导出任务的 .xlsx 孤儿文件。 + + 有效任务指状态 succeeded、路径一致且尚未过期的任务;失败任务残留的文件和任务行 + 已被删除(如数据集级联删除)的文件在保留期后回收,绝不触碰活动任务的文件。 + """ + retention = (now - timedelta(hours=ORPHAN_RETENTION_HOURS)).timestamp() + removed = 0 + for path in user_dir.glob("*.xlsx"): + try: + if path.is_symlink() or not path.is_file() or path.stat().st_mtime > retention: + continue + job_id = uuid.UUID(path.stem) + except (OSError, ValueError): + continue + job = FormulaExport.objects.filter(pk=job_id).first() + if ( + job is not None + and job.status == "succeeded" + and job.path == str(path) + and job.expires_at is not None + and job.expires_at > now + ): + continue + try: + path.unlink() + removed += 1 + if job is not None and job.path == str(path): + job.path = "" + job.save(update_fields=["path"]) + self.stdout.write(f"Reclaimed leftover workbook of job {job.pk}") + except OSError as exc: + self.stderr.write(f"Could not reclaim orphan export {path.stem}: {exc}") + return removed diff --git a/finance/management/commands/import_worker.py b/finance/management/commands/import_worker.py index 469b76b..f11ada7 100644 --- a/finance/management/commands/import_worker.py +++ b/finance/management/commands/import_worker.py @@ -1,13 +1,19 @@ import time +from datetime import timedelta from django.core.management.base import BaseCommand from django.db import connection, transaction +from django.db.models import Q from django.utils import timezone from finance.exporting import process_formula_export from finance.importing import process_dataset from finance.models import Dataset, FormulaExport +# 无咨询锁的数据库(如 SQLite)上等待该时限后才重新认领 processing 任务, +# 避免与仍在运行的 worker 抢任务;PostgreSQL 由咨询锁保证单 worker,可立即恢复。 +STALE_PROCESSING_SECONDS = 15 * 60 + class Command(BaseCommand): help = "Process durable import and formula-export jobs" @@ -30,30 +36,35 @@ class Command(BaseCommand): with connection.cursor() as cursor: cursor.execute("SELECT pg_advisory_unlock(782146291)") - def run_jobs(self, options, recover): + def run_jobs(self, options, recover_immediately): while True: - processed = self._next_import(recover) + processed = self._next_import(recover_immediately) if not processed: - processed = self._next_export(recover) + processed = self._next_export(recover_immediately) if options["once"]: return if not processed: time.sleep(2) - def _claim(self, model, states): + def _claim(self, model, immediate_recovery): + stale_before = timezone.now() - timedelta(seconds=STALE_PROCESSING_SECONDS) + claimable = Q(status="queued") | Q(status="processing", created_at__lt=stale_before) + if immediate_recovery: + claimable = Q(status="queued") | Q(status="processing") with transaction.atomic(): - qs = model.objects.filter(status__in=states).order_by("created_at") + qs = model.objects.filter(claimable).order_by("created_at") if connection.features.has_select_for_update_skip_locked: qs = qs.select_for_update(skip_locked=True) item = qs.first() if not item: return None - updated = model.objects.filter(pk=item.pk, status__in=states).update(status="processing") - return item if updated else None + # CAS 认领 queued 任务互斥;processing 仅在超时或咨询锁保护下重认领, + # 重复发布对同一目标路径写入相同内容,不会产生重复文件。 + claimed = model.objects.filter(pk=item.pk, status=item.status).update(status="processing") + return item if claimed else None - def _next_import(self, recover): - states = ["queued", "processing"] if recover else ["queued"] - item = self._claim(Dataset, states) + def _next_import(self, recover_immediately): + item = self._claim(Dataset, recover_immediately) if not item: return False Dataset.objects.filter(pk=item.pk).update(started_at=timezone.now()) @@ -61,9 +72,8 @@ class Command(BaseCommand): self.stdout.write(f"{result.pk}: {result.status}, {result.row_count} rows") return True - def _next_export(self, recover): - states = ["queued", "processing"] if recover else ["queued"] - item = self._claim(FormulaExport, states) + def _next_export(self, recover_immediately): + item = self._claim(FormulaExport, recover_immediately) if not item: return False try: diff --git a/finance/migrations/0005_formuladefinition_revision.py b/finance/migrations/0005_formuladefinition_revision.py new file mode 100644 index 0000000..0695b28 --- /dev/null +++ b/finance/migrations/0005_formuladefinition_revision.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.17 on 2026-09-25 17:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("finance", "0004_formuladefinition"), + ] + + operations = [ + migrations.AddField( + model_name="formuladefinition", + name="revision", + field=models.PositiveIntegerField(default=1), + ), + ] diff --git a/finance/models.py b/finance/models.py index 1de5920..e84caf7 100644 --- a/finance/models.py +++ b/finance/models.py @@ -121,6 +121,7 @@ class FormulaDefinition(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) normalized_name = models.CharField(max_length=320) definition = models.JSONField() + revision = models.PositiveIntegerField(default=1) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) diff --git a/frontend/src/dialogs.tsx b/frontend/src/dialogs.tsx index 0e52997..6e5e96b 100644 --- a/frontend/src/dialogs.tsx +++ b/frontend/src/dialogs.tsx @@ -1,5 +1,12 @@ import { useEffect, useRef, useState } from "react"; -import { ChevronLeft, ChevronRight, Download, Save, Trash2, X } from "lucide-react"; +import { + ChevronLeft, + ChevronRight, + Download, + Save, + Trash2, + X, +} from "lucide-react"; import { api } from "./api"; import { labels, @@ -24,6 +31,7 @@ type FormulaExpression = type SavedFormulaDefinition = { id: string; definition: Record; + revision: number; updated_at: string; }; @@ -34,6 +42,19 @@ function describeExpression(expression: FormulaExpression): string { return `(${describeExpression(expression.left)} ${operators[expression.operator]} ${describeExpression(expression.right)})`; } +function definitionSummary(definition: Record): string { + const text = [ + `已载入已保存定义「${String(definition.name)}」`, + `含义:${String(definition.meaning)}`, + `适用范围:${String(definition.scope)}`, + `缺失/零分母处理:${String(definition.missing)}`, + `计算表达式:${JSON.stringify(definition.expression)}`, + ].join(";"); + return text.length <= 2000 + ? text + : `${text.slice(0, 2000)}…(内容过长已截断)`; +} + export function Modal({ title, onClose, @@ -349,7 +370,8 @@ export function EvidenceDialog({ }>(), [side, setSide] = useState<"current" | "baseline">("current"), [page, setPage] = useState(0), - [error, setError] = useState(""); + [error, setError] = useState(""), + [reloadKey, setReloadKey] = useState(0); useEffect(() => { let active = true; setResult(undefined); @@ -364,8 +386,14 @@ export function EvidenceDialog({ return () => { active = false; }; - }, [ds.id, ctx, side, page]); + }, [ds.id, ctx, side, page, reloadKey]); function selectSide(value: "current" | "baseline") { + if (value === side) { + // 重复点击当前期间:仅在不在第一页时回到第一页触发刷新, + // 同期间同页直接返回,避免清空列表后停在“加载中”。 + if (page !== 0) setPage(0); + return; + } setResult(undefined); setPage(0); setSide(value); @@ -401,7 +429,8 @@ export function EvidenceDialog({ {user.role !== "admin" &&

客户原始标识不包含在明细中。

} {error && (

- {error} + {error}{" "} +

)}
@@ -435,6 +464,11 @@ export function EvidenceDialog({ {String(r.gross ?? "")} ))} + {result && result.rows.length === 0 && ( + + 本期范围内没有数据 + + )}
@@ -769,14 +803,25 @@ export function FormulaExportDialog({ setBusy(true); setError(""); try { + // 同名保存按服务端匹配规则覆盖旧版本;带上已知版本的 revision, + // 其他窗口先更新过则服务端返回 409,避免静默覆盖。 + const normalizedName = (value: string) => value.trim().toLowerCase(); + const existing = definitions.find( + (item) => + normalizedName(String(item.definition.name)) === + normalizedName(String(column.name)), + ); const saved = await api( `/datasets/${ds.id}/formula-definitions`, "POST", - { definition: column }, + { definition: column, revision: existing?.revision }, ); setDefinitions((items) => [...items.filter((item) => item.id !== saved.id), saved].sort((a, b) => - String(a.definition.name).localeCompare(String(b.definition.name), "zh-CN"), + String(a.definition.name).localeCompare( + String(b.definition.name), + "zh-CN", + ), ), ); setSelectedDefinition(saved.id); @@ -789,9 +834,25 @@ export function FormulaExportDialog({ function loadDefinition(id: string) { setSelectedDefinition(id); const saved = definitions.find((item) => item.id === id); - if (!saved) return; + if (!saved) { + // 选择空选项时同步清理当前定义,避免下拉选择与预览不一致 + setColumn(null); + setMessages([]); + setQuestions([]); + return; + } setColumn(saved.definition); - setMessages([]); + // 载入定义后把原定义显式放进可见、经同意的对话上下文, + // 使“把刚才公式改成乘以 100”这类指代能关联到已载入表达式; + // 修改结果仍需重新确认,不会静默沿用旧定义。 + setMessages([ + { role: "user", content: definitionSummary(saved.definition) }, + { + role: "assistant", + content: + "已载入该定义作为当前基准。请描述要调整的内容,我会在该定义基础上生成新的列定义,仍需你确认后再生成。", + }, + ]); setQuestions([]); setDraft(""); } @@ -862,6 +923,11 @@ export function FormulaExportDialog({ )} + {definitions.length > 0 && !task && ( +

+ 保存同名定义会覆盖原版本;若已在其他窗口被修改,保存将提示刷新后重试。 +

+ )}