commit 0ac0c4eb56c471072d5f7947c40deef4241a0809 Author: Jayway Liu <14022498+bluejayway@user.noreply.gitee.com> Date: Sat Sep 26 01:12:10 2026 +0800 chore: import existing codebase as baseline diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f53c63c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.venv +node_modules +.env +var +staticfiles +frontend/dist +test-results +playwright-report +.coverage +__pycache__ +*.pyc diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cebb790 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +APP_ENV=production +SECRET_KEY=REPLACE_WITH_AT_LEAST_50_RANDOM_CHARACTERS +POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_HEX_PASSWORD +DATABASE_URL=postgresql://traceiq:REPLACE_WITH_RANDOM_HEX_PASSWORD@db:5432/traceiq +DOMAIN=finance.example.com +ALLOWED_HOSTS=finance.example.com,localhost +CSRF_TRUSTED_ORIGINS=https://finance.example.com +TRUST_PROXY=1 +DB_SSLMODE=disable +MAX_UPLOAD_BYTES=67108864 +MAX_IMPORT_ROWS=1000000 +AI_ENABLED=0 +OPENAI_API_KEY= +OPENAI_MODEL= +VITE_AG_GRID_LICENSE_KEY= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..14f3cb7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: Verify +on: [push, pull_request] +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: ci-only-password + POSTGRES_DB: traceiq + ports: ['5432:5432'] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + env: + DATABASE_URL: postgresql://postgres:ci-only-password@localhost:5432/traceiq + E2E_PASSWORD: CI-Test-Only-SecurePassword882! + PYTHONPATH: . + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + version: '0.10.12' + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: uv sync --frozen --python 3.12 + - run: npm ci --registry=https://registry.npmjs.org + - run: uv run ruff check server finance tests scripts + - run: uv run ruff format --check server finance tests scripts + - run: uv run python manage.py makemigrations --check --dry-run + - run: uv run pytest --cov=finance --cov-report=term-missing --cov-fail-under=85 + - run: npm run build + - run: npm audit --omit=dev --registry=https://registry.npmjs.org + - run: uv run pip-audit + - run: uv run python manage.py migrate --noinput + - run: uv run python manage.py collectstatic --noinput + - run: uv run python manage.py bootstrap --username admin --organization CI + env: + BOOTSTRAP_PASSWORD: ${{ env.E2E_PASSWORD }} + - run: npx playwright install --with-deps chromium + - name: Browser regression + shell: bash + run: | + uv run python manage.py runserver 127.0.0.1:8000 --noreload > /tmp/web.log 2>&1 & + web=$! + uv run python manage.py import_worker > /tmp/worker.log 2>&1 & + worker=$! + trap 'kill "$web" "$worker"' EXIT + for i in {1..30}; do curl -fsS http://127.0.0.1:8000/api/health && break; sleep 1; done + npm run test:e2e + - uses: actions/upload-artifact@v4 + if: always() + with: + name: browser-results + path: | + playwright-report + test-results diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8b11da4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +.venv/ +node_modules/ +__pycache__/ +*.pyc +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ +.env +*.sqlite3 +var/ +staticfiles/ +frontend/dist/ +test-results/ +playwright-report/ +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..383ee09 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM node:22-bookworm-slim AS frontend +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci --registry=https://registry.npmjs.org +COPY tsconfig.json vite.config.ts ./ +COPY frontend ./frontend +ARG VITE_AG_GRID_LICENSE_KEY="" +ENV VITE_AG_GRID_LICENSE_KEY=$VITE_AG_GRID_LICENSE_KEY +RUN npm run build + +FROM ghcr.io/astral-sh/uv:0.10.12 AS uv +FROM python:3.12-slim-bookworm +COPY --from=uv /uv /usr/local/bin/uv +WORKDIR /app +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 UV_COMPILE_BYTECODE=1 PATH="/app/.venv/bin:$PATH" +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-dev +COPY manage.py ./ +COPY server ./server +COPY finance ./finance +COPY --from=frontend /app/frontend/dist ./frontend/dist +RUN mkdir -p /app/var/uploads && python manage.py collectstatic --noinput && useradd --uid 10001 --create-home app && chown -R app:app /app/var +USER app +EXPOSE 8000 +CMD ["gunicorn", "server.wsgi:application", "--bind", "0.0.0.0:8000", "--workers", "3", "--threads", "2", "--timeout", "120", "--access-logfile", "-", "--error-logfile", "-"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..7975490 --- /dev/null +++ b/README.md @@ -0,0 +1,118 @@ +# TraceIQ 财务分析工作台 + +响应式 Web 前端 + Django API + PostgreSQL。基于分享会话、医院财务 Excel 原始附件及 AG Grid 开发计划实现，当前交付是可运行、经过本地回归的核心业务版本，不等于全部需求完成或已获生产上线认证。缺口与上线门禁见 [验收状态](docs/acceptance.md)。 + +## 已实现 + +- XLSX / UTF-8 CSV 导入、双行表头、原始解释保留、字段语义确认、产品毛利率及自营产品范围配置。 +- 原始文件私有存储、组织内文件 SHA-256 去重、导入事务、持久队列、PostgreSQL worker 崩溃恢复。 +- 实际收入、成本、毛利、两种明确 ROI 公式、去重人数、客单价、毛利率、产品扣率、自营收入、升单率、原价客单价。 +- 同比、环比、自定义等长月份范围；缺基期禁止比较，缺成本显示不可用，缺月份趋势断开。 +- 总览、异常追踪、贡献分解、自定义下钻层级、面包屑、原始行证据分页、CSV / XLSX 证据导出。 +- 个人设置保存/取消/预览与乐观锁；调查备注与状态持久化；账号启停、角色及科室范围、操作审计。 +- 本地规则助手；可选 OpenAI Responses 结构化意图接口，用户同意后仅发送问题与维度/当前筛选，查询仍受后端权限约束。 + +当前原附件没有医生、成本和连续月度数据，不会生成虚构的医生排名、ROI 或环比结论。毛利是根据产品分类毛利率计算的贡献值，不等同于已审计净利润。 + +## 技术结构 + +| 目录 | 职责 | +| --- | --- | +| `frontend/src` | React 19 / TypeScript、AG Grid、ECharts、响应式 UI | +| `finance/importing.py` | 解析、映射、Decimal 校验、批量导入 | +| `finance/analysis.py` | 权限内聚合、比较、异常与贡献 | +| `finance/views.py` | 身份认证、受控 API、导出、设置、账号、审计 | +| `finance/assistant.py` | 规则助手及外部模型意图校验 | +| `server` | Django 配置、路由、WSGI | +| `tests` | PostgreSQL 集成测试、Playwright 浏览器回归 | +| `scripts` | 本地账号初始化、原始附件独立对账 | + +后端金额使用 `NUMERIC(44,24)`，Python Decimal 精度 60；JSON 金额为字符串。浏览器 Number 仅用于绘图与显示，不作为账务计算或持久化来源。Excel 精确金额以文本单元格导出，避免 Excel 的 15 位有效数字上限。客户端不会一次加载 33 万条明细，只加载聚合结果，证据查询每页最多 200 行。 + +## 本地运行 + +需要 Node.js 22、uv、Python 3.12、PostgreSQL 17。所有示例从仓库根目录执行。 + +```bash +uv sync --frozen --python 3.12 +npm ci --registry=https://registry.npmjs.org +export DATABASE_URL='postgresql://USER:PASSWORD@127.0.0.1:5432/traceiq' +uv run python manage.py migrate +PYTHONPATH=. uv run python scripts/init_local.py +npm run build +uv run python manage.py collectstatic --noinput +uv run python manage.py runserver 127.0.0.1:8000 --noreload +``` + +另一个终端启动导入 worker： + +```bash +export DATABASE_URL='postgresql://USER:PASSWORD@127.0.0.1:5432/traceiq' +uv run python manage.py import_worker +``` + +访问 `http://127.0.0.1:8000`，本地随机账号保存在 `var/local-access.txt`，文件权限 0600。该脚本拒绝生产模式。开发热更新可在第三个终端运行 `npm run dev`，Vite 将 `/api` 代理到 8000。 + +本次本机验证使用专属集群 `var/postgres`、端口 55432、库 traceiq： + +```bash +/opt/homebrew/opt/postgresql@17/bin/pg_ctl -D var/postgres -l var/postgres.log -o '-h 127.0.0.1 -p 55432' start +export DATABASE_URL='postgresql://jianwel@127.0.0.1:55432/traceiq' +``` + +该集群只监听本机，使用开发认证策略，不可公开端口或复制到生产。没有设置开机自启。SQLite 仅作轻量开发后备，不保证高精度金额聚合，禁止财务验收和生产使用。 + +## 生产部署 + +1. 在独立服务器配置域名 DNS、防火墙及 TLS，创建 `.env`，参考 `.env.example`。生成至少 50 字符随机 SECRET_KEY、数据库强密码，两处数据库密码需一致；密码有特殊字符时 DATABASE_URL 中必须 URL 编码。 +2. 设置准确的 `DOMAIN`、`ALLOWED_HOSTS`、`CSRF_TRUSTED_ORIGINS`。仅开放 Caddy 80/443；数据库与 Gunicorn 不发布主机端口。 +3. 执行 `docker compose --env-file .env config --quiet`，再执行 `docker compose up --build -d`。migrate 服务成功后才启动 web/worker，Caddy 自动申请 HTTPS 证书。 +4. 执行 `docker compose exec web python manage.py bootstrap --username admin --organization '医院财务中心'`，交互输入管理员密码，无默认生产密码。 +5. 执行 `docker compose exec web python manage.py check --deploy`，在隔离预生产环境运行验收，完成下面运维门禁再上线。 + +默认单个导入 worker 用 PostgreSQL advisory lock 防止重复处理；其他 worker 获取不到锁会退出。worker 异常退出，数据库回滚未完成的 Fact 事务并释放锁，下次启动会重新处理 processing。需要受控停机时先等任务 ready 再停，强制停止会导致整个文件重新导入。通过 `import_file PATH --username admin --confirm-mapping` 的 CLI 导入也入队，不绕过 worker。 + +`TRUST_PROXY=1` 仅用于 web 无公网直连、反向代理覆盖 X-Forwarded-Proto 和 X-Real-IP 的部署。登录 IP 限流在此模式使用经过校验的 X-Real-IP，未启用时忽略该头。`.env.example` 的 DB_SSLMODE=disable 仅适用同宿主私有容器网络；跨主机数据库用 verify-full 与受信证书。代理上传上限 66 MB、应用文件上限 64 MiB。生产使用环境变量/密钥服务注入密码，不提交 `.env`。 + +## 备份与运维 + +- 同时备份 PostgreSQL 与 uploads 卷，不能只备份其中一个。uploads 包含原始财务/客户标识，需加密磁盘、受控访问及留存期限。 +- 低频窗口暂停新上传，等待当前导入完成并停止 worker；用 `pg_dump -Fc` 创建数据库备份，再快照 uploads，记录同一备份批次。将备份放在加密、异地主机或受控对象存储。 +- 恢复到**隔离新库和新卷**，使用 `pg_restore --exit-on-error` 后检查迁移、账号范围、导入状态，执行金额/人数对账，抽查原始行与导出，再切换流量。未完成恢复演练不能上线。 +- 定时执行 `python manage.py clearsessions`；监控 `/api/health`、HTTP 5xx、查询 P95、导入队列等待时间、processing 持续时间、磁盘/数据库容量、失败审计与备份新鲜度。当前未配置外部告警渠道。 +- 管理员忘记密码由运维执行 `python manage.py changepassword USERNAME`，现阶段没有邮件找回、自助密码修改、SSO 或 MFA。 +- 审计记录对网页用户只读，但数据库管理员仍可修改；需要不可抵赖审计时，应外送到独立追加写日志平台。 +- 发布前备份并固定镜像 digest；失败时回滚应用镜像。涉及不可逆数据迁移不得直接反向迁移，按批准的数据库恢复方案操作。 + +## 测试 + +测试数据库需要 CREATE DATABASE 权限。测试账号和 fixture 均为虚构数据。 + +```bash +uv run ruff check server finance tests scripts +uv run ruff format --check server finance tests scripts +uv run python manage.py makemigrations --check --dry-run +uv run pytest --cov=finance --cov-report=term-missing --cov-fail-under=85 +npm run build +npm run test:e2e +npm audit --omit=dev --registry=https://registry.npmjs.org +uv run pip-audit +``` + +Playwright 需要正在运行的本地 API 和 worker，可用 E2E_PASSWORD 指定测试 admin 密码，默认读取本地凭据文件。首次执行 `npx playwright install chromium`。浏览器 fixture 测试会新建 `e2e-finance.csv` 数据集，不可指向正式生产环境。`.github/workflows/ci.yml` 在独立 PostgreSQL 上执行同样流程，尚未推送到远端触发。 + +原附件只在本机使用，不在仓库中： + +```bash +uv run python scripts/inspect_source.py '/Users/jianwel/Downloads/分析数据_原稿(1).xlsx' +PYTHONPATH=. uv run python scripts/reconcile_reference.py +REFERENCE_DATASET_NAME='分析数据_原稿(1).xlsx' npm run test:e2e +``` + +## AI 与跨平台 + +默认 AI_ENABLED=0。启用外部模型需配置 `AI_ENABLED=1`、`OPENAI_API_KEY`、`OPENAI_MODEL`，并完成机构数据处理审批。密钥只在后端。用户自由输入的问题也可能含敏感信息，不能仅凭不发送原始行就认定无隐私风险。模型不能提交 SQL、修改金额或修改账号；输出只允许受控意图，异常/拒答关闭执行。 + +前端为标准 Web，API 无平台耦合，可由后续 PWA、Tauri 或 Capacitor 客户端复用。当前没有打包桌面/移动客户端，也不在浏览器离线缓存财务数据。 + +AG Grid Community 默认可运行；设置 `VITE_AG_GRID_LICENSE_KEY` 后注册 Enterprise 模块并启用已接入的侧栏/状态栏。必须自行取得合适的商用许可证。SSRM / Pivot / Master-Detail 尚未接入，不能因存在 Enterprise 依赖就声称已完成这些功能。 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..17918d3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,78 @@ +name: traceiq +x-app: &app + build: + context: . + args: + VITE_AG_GRID_LICENSE_KEY: ${VITE_AG_GRID_LICENSE_KEY:-} + image: traceiq:local + env_file: .env + volumes: + - uploads:/app/var/uploads + restart: unless-stopped + init: true + read_only: true + tmpfs: + - /tmp:size=1g,mode=1777 + cap_drop: [ALL] + security_opt: [no-new-privileges:true] +services: + db: + image: postgres:17-bookworm + environment: + POSTGRES_DB: traceiq + POSTGRES_USER: traceiq + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + volumes: + - postgres:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: [CMD-SHELL, 'pg_isready -U traceiq -d traceiq'] + interval: 5s + retries: 20 + migrate: + <<: *app + command: [python, manage.py, migrate, --noinput] + restart: "no" + depends_on: + db: + condition: service_healthy + web: + <<: *app + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen(urllib.request.Request('http://localhost:8000/api/health', headers={'X-Forwarded-Proto':'https'}), timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 + worker: + <<: *app + command: [python, manage.py, import_worker] + depends_on: + migrate: + condition: service_completed_successfully + cleanup: + <<: *app + command: [python, manage.py, cleanup_formula_exports, --loop] + depends_on: + migrate: + condition: service_completed_successfully + proxy: + image: caddy:2.10-alpine + environment: + DOMAIN: ${DOMAIN:?Set DOMAIN} + ports: ['80:80', '443:443'] + volumes: + - ./deploy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + restart: unless-stopped + depends_on: + web: + condition: service_healthy +volumes: + postgres: + uploads: + caddy_data: + caddy_config: diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 0000000..b1afc77 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,10 @@ +{$DOMAIN} { + request_body { + max_size 66MB + } + reverse_proxy web:8000 { + header_up X-Forwarded-Proto {scheme} + header_up X-Real-IP {remote_host} + } + log +} diff --git a/docs/acceptance.md b/docs/acceptance.md new file mode 100644 index 0000000..6a961c5 --- /dev/null +++ b/docs/acceptance.md @@ -0,0 +1,113 @@ +# 验收状态与上线门禁 + +日期：2026-09-24，Asia/Singapore。本文区分实现状态和验证状态，不把编译成功等同于生产商用验收。 + +## 公式工作簿与对话新增列 + +本轮新增校验：对话模型、导出 API 及最终工作簿生成器均只允许引用当前数据映射可计算的指标；模型误报完整或客户端绕过对话直接提交时，服务端仍会拒绝缺少来源字段的自定义列。 + +> **当前发布判断：** 本轮后端 61 passed、finance 覆盖率 87.66%；公式工作簿版本 3 已改为流式写入，100,000 行产品上限实测峰值 RSS 约 136.4 MB。新增列流程覆盖追问、补充、定义确认、数据映射可用性防护及公式工作簿下载。LibreOffice 重算测试覆盖特定黄金样例；Microsoft Excel、财务口径签字、真实 AI 合规、原附件 332,830 行范围方案及目标环境部署仍未验收。 + +| 需求 | 状态 | 实现 / 尚缺内容 | +| --- | --- | --- | +| 对话定义自定义列 | 已实现；mock 端到端通过 | 用户描述列名、含义、范围及计算；缺项时模型主动追问，补充后结合全对话重审；只接受既有指标/常数和白名单运算树。编辑补充时保留当前定义供核对，未发送的草稿会禁用确认；最多 6 轮，限 10 列、8 层表达式。真实外部模型联调未做。 | +| 自定义列公式导出 | 已实现；小样本验证通过 | 异步任务生成 XLSX；自定义列引用汇总指标公式，并包含使用说明、参数口径、计算明细、原始证据和对账页；另保留原有证据快照导出。 | +| Excel 公式引擎验收 | 尚未通过发布门禁 | 消费人数与升单率改用随机去标识 CustomerRoster 固定人数表 + COUNTIFS，不依赖 UNIQUE/FILTER；4 项 LibreOfficeDev 重算测试已通过。Microsoft Excel 仅打开过样本并在公式栏看到 COUNTIFS/SUMIFS；Excel 为只读无法保存重算副本，且所见分组与预期不匹配，故 Excel 实机重算结果仍未验收。身份、分组和期间是固定快照，编辑金额可重算，不支持通过改客户身份改变人数。 | +| 生产上线资格 | 未完成 | 公式导出最大 100,000 行/200 组，真实附件 332,830 行尚不能一次导出；真实 AI 服务联调、数据出境审批、目标 Excel 兼容、性能及安全审查均未完成。 | + +## 最新验证（2026-09-24） + +- 对话新增列及公式导出回归：后端全量 `61 passed`，finance 覆盖率 `87.66%`；新增用例覆盖流式工作簿、精确文本条件、未分类对账、Excel 长度限制、公式注入、保留列冲突、任务文件权限/路径、授权快照顺序和路径准备失败恢复。 + +- 当前源码自动化基线（2026-09-24）：61 passed，finance 覆盖率 87.66%（85% 门槛）；包含列定义追问、数据映射可用性防护、流式导出和 LibreOffice 公式重算回归。Ruff lint/format、TypeScript/Vite 构建、Django check、迁移检查通过。 +- 本轮隔离 SQLite Playwright：3 passed、1 skipped；新增列追问与下载、桌面综合工作流、手机布局通过；原附件 UI 用例因未设置 `REFERENCE_DATASET_NAME` 跳过。隔离数据库和媒体目录位于 `/tmp/caiwu-roadmap-e2e*`；登录成功的正式用例使用临时口令环境变量。 +- 完整临时生产配置下 `check --deploy` 无告警；pip-audit 和 npm audit 均为 0 已知漏洞。Docker 镜像构建未运行（Docker daemon 不可连接），Compose 校验未运行（真实 `.env` 不存在）。 +- 真实附件 PostgreSQL 17 对账通过：332,830 行、5 个分组，收入/毛利/人数与历史独立对账一致，缺成本 332,830 行、ROI 不可用。完整公式工作簿生成基准 96.94 秒、51,532,457 字节、峰值 RSS 3,308,617,728 字节；单机单次生成不等于资源/SLA 通过。 +- 本轮以安装的 LibreOfficeDev 执行 `tests/test_formula_recalculation.py`（3 项）与 `tests/test_formula_recalculation_gross.py`（1 项）：共 4 项通过，覆盖自定义汇总/变化率、编辑明细金额后的联动、零成本 ROI、缺失基期，以及 direct/lookup 毛利来源。该结果不是 Microsoft Excel 实机验收。 +- 流式容量基准：公式版本 3 的 20,000 行合成工作簿生成 5.26 秒、3,265,273 字节、峰值 RSS 77,922,304 字节；100,000 行生成 25.96 秒、16,217,767 字节、峰值 RSS 136,429,568 字节。100,000 行产物 ZIP/OOXML 完整性、7 个工作表、表格关系、版本及汇总公式读取通过；这是本机隔离 SQLite 单次基准，不是目标环境并发 SLA。 +- Microsoft Excel 桌面检查记录：在只读 Excel 中打开 `/tmp/caiwu-excel-validation.xlsx`，公式栏可见 COUNTIFS/SUMIFS；无法保存重算副本，且所选样例分组为 `Test`、与预期不匹配。结果仅证明样本中存在公式，不证明 Excel 计算正确，发布门禁维持未通过。 +- `compose.yaml` 已配置 `cleanup_formula_exports --loop` 常驻清理服务，清理逻辑有 3 项测试覆盖；目标环境仍需确认运行与告警。 + +## 剩余功能与发布阻断 +| P0 | Microsoft Excel 公式引擎验收 | 已有 LibreOfficeDev 小样本公式缓存证据，仍需目标 Excel Windows/Mac 重算和兼容性验收。 | +| P0 | 真实附件范围与业务口径 | 100,000 行/200 组产品上限已用流式版本实测；原附件 332,830 行仍超范围，需选择可审计分卷或正式提高上限后再验。财务签字仍未完成。 | +| P0 | 真实 AI 合规与商用环境 | 真实模型联调、数据出境/隐私审批和目标环境安全验证尚缺。 | + +| 优先级 | 未完成项 | 发布影响 / 下一步 | +| --- | --- | --- | +| P0 | Microsoft Excel 公式引擎验收 | 人数公式使用静态去重辅助表 + COUNTIFS；仍需目标 Excel Windows/Mac 打开、重算、保存，覆盖人数、升单率、空集、缺期及金额编辑后的计算。 | +| P0 | 真实附件全量公式导出 | 当前上限 100,000 明细行/200 分组，原附件 332,830 行；100,000 行流式资源压测已完成，仍需定义切分导出并保证跨文件口径，或批准扩大产品范围后做 332,830 行目标引擎验收。 | +| P0 | 财务业务口径签字 | 确认 N 列实际收入与 I 列实收、退款、成本归属、毛利、ROI、升单、自营产品集和舍入规则；真实附件成本缺失且 ROI 不可用。 | +| P0 | 外部 AI 合规与真实联调 | 完成真实模型超时/限额/故障演练、数据出境和隐私审批；未批准前保持 AI 关闭。 | +| P1 | 对话定义可扩展性 | 当前仅支持既有指标/常数与加减乘除；不支持任意 Excel 函数、跨明细新聚合、按零替代/跳过缺失、生成维度列或保存复用定义。新能力需逐项安全与黄金样例验证。 | +| P1 | 工作簿操作完整性 | 当前固定导出范围，不支持在 Excel 增加明细行后自动扩展所有报表；结构变化需重新导出。验证参数编辑、结构变化提示、模板版本和列字典。 | +| P1 | 定义体验差距 | 证据弹窗已支持本期/基期切换并通过 E2E；公式定义对话仍没有持久化草稿/定义库。确认复用范围后实现并补 E2E。 | +| P1 | 商业部署验收 | Docker 构建、HTTPS/证书、PostgreSQL/TLS、备份恢复、并发/SLA、审计监控、渗透测试、AG Grid Enterprise 商用许可需在目标环境验证/审批。 | + + + + +## 需求覆盖 + +| 需求 | 状态 | 实现 / 尚缺内容 | +| --- | --- | --- | +| 网页前端、手机适配 | 已实现，本地浏览器回归 | React + 响应式 CSS，无原生客户端 | +| XLSX/CSV、双行表头、解释型映射 | 已实现 | 标准别名建议 + 人工确认，不宣称通用 AI 自动识别全部表格 | +| 原始文件版本、重复上传、失败回滚 | 已实现并测试 | SHA-256 组织内唯一，源文件私有，成功版本不可修改 | +| 收入/毛利/成本/ROI/人数/客单价 | 已实现并测试 | 缺字段返回不可用；ROI 在导入时明确公式 | +| 自营收入/升单率/原价客单价 | 已实现并测试 | 新导入配置 self_operated_products；旧版本未配置时不可用，不偷偷改历史口径 | +| 同比/环比/自定义多月 | 已实现并测试 | 基期等长且不能重叠，缺失期间不比较 | +| 趋势/贡献/矩阵/异常列表 | 已实现 | ECharts 折线/横条图，非完整瀑布/Pareto/散点套件 | +| 动态下钻/面包屑/证据 | 已实现并测试 | 来源、区域、科室、学科、产品等映射维度；医生仅有字段才出现 | +| 设置保存/取消/预览、调查状态 | 已实现并测试 | 个人设置版本锁，调查按上下文持久化；没有团队工单分派 | +| CSV/Excel 导出 | 双模式已实现 | 既有 CSV/XLSX 为证据快照；公式 XLSX 为独立异步工作簿，含自定义公式列、说明页和固定人数去重辅助表。Microsoft Excel 引擎和全量附件门禁未通过，不可视为生产验收 | +| 账号/组织/科室权限/CSRF/审计 | 已实现并测试 | 应用层隔离；无数据库 RLS、SSO、MFA、自助密码找回 | +| Copilot | 本地规则助手可用；外部接口已实现 | 外部模型成功/拒答/非法意图以 mock 测试，尚未真实 API 联调 | +| AG Grid Enterprise | 部分接入 | 授权后注册模块；SSRM/Pivot/Tree Data/Master-Detail 未实现；现在使用服务端聚合 + 分页证据 | +| 全部过渡表口径重建 | 未完成 | 消费比率、实收/实际收入多口径、跨表 lookup 编辑、生效期毛利率、退款口径切换仍缺 | +| 异常规则 | 基础实现 | 不利变化率 + 影响金额 + 两期样本数；结构占比、历史分位数、告警投递尚未实现 | +| 原始附件 | 全量已导入并对账 | 不含医生/成本；只有两个单月，不允许虚构连续期间 | +| Docker/HTTPS/CI | 配置已交付，验证不完整 | Compose 使用示例环境文件替换后的配置校验通过；基础镜像 metadata 下载长时间无响应，已取消构建；无正式域名证书与远端 CI 运行记录 | + +原始附件“过渡表2”同时使用“实收金额”“收入”“实际收入”字样，产品扣率分子存在歧义。当前产品扣率与客单价明确使用选定的实际收入字段，原附件采用 N 列；不能直接当作 I 列实收口径通过财务签字。旧导入版本也不会因为新代码增加自营产品范围而静默更改映射。 + +## 真实数据对账 + +来源 SHA-256：`98ef41e898c24f0929a9da6ec185b7331f69f0478b52056cbc7834e1ef5877ec`。 + +| 指标 | 2025-08 | 2026-08 | +| --- | ---: | ---: | +| 明细行数 | 157512 | 175318 | +| N 列实际收入 | 225550255.4187610638720 | 152419115.348455502666069 | +| 产品毛利贡献 | 24790335.405853982758000 | 17868580.85186759073400145 | +| 去重消费人数 | 33147 | 38361 | + +总计 332830 行；收入同比 -32.4234%，毛利贡献同比 -27.9212%；ROI 不可用。独立逐行读取与数据库精确聚合一致，科室贡献变化合计等于总体变化。原数据 11990 行收入超过 4 位小数，未在落库时四舍五入到 4 位。 + +质量核查：负收入 12338 行、G/L 应收差异 14154 行、I/N 实收与实际收入差异 11991 行、成本缺失 332830 行、毛利率未匹配 0 行。本机单次完整查询约 3 秒，仅为一次开发机测量，不是并发性能 SLA。 + +## 测试记录 +- 本轮源码测试：44 passed，finance 覆盖率 86.95%；常规 pytest 在本机 SQLite 测试配置运行，独立 PostgreSQL CI 未启动；另在本机 PostgreSQL 17 隔离库执行原附件对账和完整公式工作簿生成基准。 +- 本轮浏览器回归：完整隔离 SQLite Playwright 3 passed、1 skipped；跳过未配置 `REFERENCE_DATASET_NAME` 的原附件 UI 专测。 + +- PostgreSQL 17：67 项后端测试通过（22.32 秒），finance 总覆盖率 89.17%，analysis 98%、assistant 90%、importing 86%、views 84%；CI 覆盖率门槛 85%。 +- 检查内容：Decimal 金额、人数去重、加权比率、零分母、退款、缺期、新增/消失分组贡献、XLSX 表头与 lookup、导入原子性、重复上传、非法映射、跨组织/科室隔离、CSRF、会话、登录限流、设置冲突、调查记录、导出公式注入、模型意图白名单与敏感字段最小化、worker 恢复、CLI 初始化。 +- TypeScript / Vite 生产构建通过；Django `check`、迁移检查及完整临时生产变量下的 `check --deploy` 均通过。目标部署环境仍未验收。 +- Python pip-audit 与前端 npm audit（生产依赖）均报告 0 个已知漏洞；这不是渗透测试或未来无漏洞保证。 +- 既有 Playwright 历史记录：3 项用例通过；当前分支 E2E 覆盖见上方 2026-09-24 最新验证，不能用历史结果替代本轮重跑。 +- 回归发现并修复调查控件标签定位与图表网格最小宽度问题。原附件截图为 `reference-desktop.png` / `reference-mobile.png`；截图文件已生成，但工具未可靠展示图片供逐像素人工复核，视觉精修仍应由产品人员验收。 +- 新增可信代理登录限流测试：未启用信任时忽略伪造客户端头，启用后仅接受合法 IP。生产 Caddy 覆盖 X-Real-IP，web 不得直接暴露公网。 +- 产物位于本机 `test-results/`、`playwright-report/`，未提交真实财务数据和凭据。 + +## 必须完成的上线门禁 + +1. 财务负责人确认收入/实收/原价、退款、升单、自营、成本归属与 ROI 定义；补齐缺失的业务口径和字段。 +2. 若要求原开发计划全部功能，补齐上述未完成项并逐项签字。当前版本不能标记为“所有前后端需求已完成”。 +3. 在目标基础设施完成 Docker 镜像构建、HTTPS、数据库认证/TLS、上传限制、非 root 容器及重启恢复验收。 +4. 用目标并发用户数、最大文件/维度基数做导入与查询压力测试，确定 SLA 和资源配额。当前聚合会返回所有分组，高基数维度需 SSRM 或服务端结果窗口。 +5. 完成真实 OpenAI 接口联调、超时/限额演练和数据出境/隐私审批；没有审批时保持关闭。 +6. 外部安全审查与渗透测试、最小权限数据库账号、密钥轮换、加密存储、访问日志脱敏、不可变审计与告警接入。 +7. 执行数据库 + 源文件一致性备份及恢复演练，记录 RPO/RTO，确定留存和删除流程。 +8. AG Grid Enterprise 若启用须有适用商业许可；完成所有依赖许可证盘点。 + +以上商用门禁不因本地自动化测试通过而自动满足；较早的 44、54 项测试及旧非流式 332,830 行峰值 RSS 约 3.31 GB 均为历史基线，不能替代当前版本的目标环境验收。 diff --git a/docs/excel-formula-export-plan.html b/docs/excel-formula-export-plan.html new file mode 100644 index 0000000..524d466 --- /dev/null +++ b/docs/excel-formula-export-plan.html @@ -0,0 +1,45 @@ + + + + + +财务 Agent · Excel 公式导出实施计划 + + + + + +
TRACEIQ / 财务 Agent / 工程实施与验收方案

Excel 公式导出实施计划

从只读证据快照，升级到可解释、可重算、可对账的财务工作簿。

状态：待实施 · 本文不是功能已完成或生产验收通过的证明

核查日期：2026-09-23 ｜ 交付范围：计划文档，不修改现有导出逻辑
+
+ +

01 / 核查结论：目前不含公式

当前 XLSX 导出不满足本次需求。

现有实现将每个单元格转换为字符串，并明确设置为文本类型。导出的收入、成本和毛利贡献是结果快照，没有可重算公式，也没有完整的行列字典和操作说明。

核查对象已存在的行为与目标的差距
finance/views.py · export()一张“证据明细”；前四行含版本 SHA、筛选上下文、ROI 口径、导出时间；所有值写为文本。没有公式计算层、汇总报表或说明页。
tests/test_finance.py · test_excel_export_precision_and_formula_safety明确断言所有单元格都不是公式，验证高精度金额及注入保护。只能证明证据快照行为正确，不能证明公式导出正确。
frontend/src/dialogs.tsx · EvidenceDialog选择 XLSX 下载证据明细。没有公式版选择、兼容性说明、精度提示或异步状态。
规模与参考附件现有单次导出上限 100,000 行；附件共有 332,830 行。不能通过现有接口一次导出原附件全部明细。

已运行的两项针对性测试：test_excel_export_precision_and_formula_safety、test_xlsx_double_header_and_lookup，结果 2 passed（1.59 秒）。后者验证导入双表头与毛利率匹配，不是 Excel 公式重算测试。

保留金额文本并非无意义限制：系统使用高精度 Decimal，而 Excel 普通数值只有约 15 位有效数字。新方案必须保留证据精度，不能直接把所有字符串改成浮点数。

+

02 / 设计选择与推荐

决策选项与代价推荐
导出模式A. 替换现有证据导出，破坏旧语义。
B. 保留证据快照，新增公式工作簿。		B。界面清楚区分“证据快照”和“公式分析工作簿”；CSV 始终是快照，不承诺公式。
精度A. 全部按 Excel 浮点数，丢失高精度证据。
B. 精确文本证据 + 数值计算层 + Decimal 对账。
C. 宏或自定义高精度插件，兼容、部署及安全成本高。		B。明确公式值遵守批准后的精度政策，不声称原生公式保持 24 位小数精确运算。
公式兼容性A. Microsoft 365，支持动态数组去重。
B. 旧版 Excel / WPS / LibreOffice，需另做兼容实现及验证。		先以 Microsoft 365 桌面 Windows / Mac 为明确验收目标；网页版另测。其他引擎通过验证后再声明支持，不能以应用跨平台替代工作簿兼容性。
人数计算A. UNIQUE/FILTER 真正按范围去重。
B. 服务端预制去重辅助表，修改客户或分组后可能失效。
C. 每行 COUNTIFS 扫全表，规模大时开销很高。		公式版选 A，限定报表分组数量并压测；B 仅作明确只读的兼容快照，不能冒充完全可重算。避免 C。
生成与重算A. 现有 openpyxl 写公式，打开 Excel 时重算。
B. 导出后由经验证的受控引擎重算并保存缓存。
C. 添加写缓存库，但缓存仍是服务器结果。		第一阶段 A + 真实 Excel 验收；首次打开前预览可能空白，必须告知。若业务要求下载即带可信缓存，则 B 是上线前置条件；不把 C 当作重算证明。
编辑能力A. 任意插行、改维度和期间并自动重建全部报表。
B. 固定导出范围，允许指定输入修改，结构性变化重新导出。		首版 B，边界容易验证；完整可变结构作为后续独立需求，不暗示已经支持。
大文件A. 同步放宽现有上限。
B. 数据库任务 + 独立 worker + 私有文件下载。		全量附件采用 B；复用项目已有任务处理模式，不预先引入新的消息队列。
+

03 / 工作簿结构与行列说明

输出放在前面，输入和证据放在后面。仅创建实际需要的页，不按每个指标拆页。所有名称均为规划，尚未实现。

工作表行与列的含义内容与用途
分析汇总每行一个指标或已选维度分组；列为本期、基期、差额、变化率、单位、可用状态。汇总金额、去重人数、比率由明细或计算结果引用产生；非加法指标不合计子组值。页首注明范围、期间、版本及模拟状态。
使用说明按主题分行：范围、粒度、字段、公式、操作、限制。字典列包括名称、业务含义、数据类型、单位、来源字段、可否修改、空值含义、公式与示例；另附版本、时区和支持的 Excel 版本。
参数与口径每行一个参数或产品毛利率；列为当前值、含义、来源、是否可改。收入映射、ROI 分支、自营产品集合、毛利率、精度政策；首版期间及筛选锁定。允许的模拟参数必须明确标识。
计算明细一行对应一个导入后的 Fact，保留来源定位键；不是一位客户或一次就诊。期间、维度、去标识客户 token、基础数值、原价、毛利来源、适用毛利率、毛利公式、缺失标志等；Excel Table 管理引用范围。
原始证据与计算明细一一对应；金额和标识保留精确文本。授权范围内的来源行、字段映射及原值；不可把完整患者原始信息无差别复制出去。工作表保护只防误操作，不是加密或权限控制。
对账检查每行一个指标及范围；列为服务器精确文本、批准精度的参考数值、公式结果、差异、阈值和检查状态。包含行数、缺失数、总分一致性、公式覆盖与精度警告；此页仅观察结果，不反向驱动业务公式。

说明页必须回答的问题

+

04 / 公式契约：每个衍生结果可追溯

基础事实可以是输入常量；新增衍生金额、计数、比率和差异必须来自真正的公式依赖。禁止用 =12345 或引用隐藏的静态答案来伪装计算。建立“指标 → 输入 → 公式 → 单位 → 缺失策略 → 测试”的显式清单。

指标计算定义边界与验收重点
收入 / 成本 / 原价在指定范围汇总对应基础字段。有数据但全为 0 与无数据不同；成本或原价部分缺失时依赖指标不可用，不能只合计非空项。
毛利贡献原始毛利有值时使用该输入；否则收入 × 已匹配产品毛利率；汇总逐行结果。必须恢复并记录 direct / lookup / missing 来源。Fact.gross 已是结果，不能直接把它当作原始毛利来伪造推导。必要时从保留的 raw 与映射提取，先做一致性测试。
ROIprofit_over_cost：(收入 − 成本) / 成本；gross_less_cost_over_cost：(毛利贡献 − 成本) / 成本。遵从数据版本映射；成本 ≤ 0 或任何必要字段缺失时不可用。
消费人数 / 客单价范围内客户 token 去重计数；收入 / 消费人数。跨月、跨产品、跨科室重复客户只计一次；有任一缺失客户时与现有后端口径一致地不可用。
毛利率 / 产品扣率毛利贡献 / 收入；收入 / 原价金额。先汇总再相除，不能平均行级比率；分母必须大于 0；不把扣率解释为“1 − 比值”。
自营收入 / 升单率配置自营产品范围内收入；该范围客户去重数 / 全部客户去重数。产品集合未配置时不可用；不能以自营收入占比替代升单率。
原价客单价原价金额 / 去重人数。原价或客户缺失时不可用。
差额 / 变化率本期 − 基期；差额 / ABS(基期)。基期 0 不计算变化率；缺期不比较；比率指标同时注明差额单位。
贡献率子组金额差额 / 父组金额差额。仅收入、成本、毛利、自营收入等可加指标；父级差额 0 不计算，不为人数或比率伪造加法贡献。

公式示例与生成规则

下列为设计样例。Data 表名、英文列名和命名范围须由模板定义，不能让用户字符串直接拼入公式；状态判断在拥有该指标的计算层生成。公式使用英文函数名和 OOXML 语法，实际写入动态数组需要验证序列化及兼容前缀。

收入（已确认该期间有记录）：
+=SUMIFS(Data[Revenue],Data[Period],CurrentPeriod)
+
+成本完整性检查后计算 ROI：
+=IF(CostStatus<>"OK",NA(),IF(Cost<=0,NA(),(Revenue-Cost)/Cost))
+
+变化率（先确认本期与基期可比较）：
+=IF(Comparable=0,NA(),IF(BaselineValue=0,NA(),
+ (CurrentValue-BaselineValue)/ABS(BaselineValue)))
+
+365 去重人数（先确认范围有记录，且该范围客户字段完整）：
+=IF(CustomerStatus<>"OK",NA(),
+ ROWS(UNIQUE(FILTER(Data[CustomerToken],Data[Period]=CurrentPeriod))))
+
+毛利计算分支：
+=IF([@GrossSource]="direct",[@GrossInput],
+ IF([@RateStatus]="OK",[@Revenue]*[@MarginRate],NA()))

多月、多维度条件需生成对应掩码；上述单月例子不能直接替代完整上下文。全量、大量分组不允许每个明细行扫描全表。公式构建须限制报告维度及单元格数量。

缺失表达推荐：计算层用预期的 NA() 阻止错误结果传播；输出页用配套状态显示“不可用：缺成本”等原因。只对已知状态做展示转换，不用 IFERROR(...,0) 吞掉真实公式错误。

+

05 / 精度与财务对账

推荐：原始精确文本不可变，计算数值有明确精度契约。

“显示两位小数”不是“计算只用两位小数”。原附件有 11,990 行收入超过四位小数，不能在导出时静默按行截断。

  1. 证据层：保留 Decimal 原始字符串及来源映射；不经 JavaScript Number 中转。完整金额不可因导出而发生舍入。
  2. 计算层：先检测有效数字、数量级和累计误差风险，再转换 Excel 数值。把转换策略、舍入位数、舍入模式和发生时点写入文件。
  3. 推荐默认：分析公式不逐行强制 ROUND；展示金额两位，最终财务比较按批准的规则舍入。若业务要求“每行先舍入再汇总”，另设明确版本口径，不能混用。
  4. 两套参照：Decimal 对原始业务口径的精确结果用于证据核验；按相同批准数值政策得到的参考结果用于 Excel 计算对账。区别“政策导致的差异”与“实现错误”。
  5. 误差门槛：逐项规定金额绝对误差、比率误差及人数严格相等。建议金额以最终批准币种最小单位一致为目标；具体阈值必须经真实样本和财务确认，不能先放宽到让测试通过。
  6. 边界处理：超过可接受数值范围、舍入临界点或差异超阈值时阻止宣称“精确一致”，给出可定位的警告或拒绝公式版，证据版仍保留。

不推荐启用 Excel“以显示精度为准”，因为它会改变存储值。原生数值公式不能承诺任意 24 位小数精确运算；若确有此硬要求，应保留服务端 Decimal 为权威计算引擎，并另行评估高精度插件，不在首版隐式引入宏。

+

06 / 修改后究竟会重算什么

操作首版承诺限制
修改允许编辑的收入、成本、原价、直接毛利行级毛利、汇总、比率、差额及对账随公式重算。原始证据不变；明确标记为本地模拟，不回写服务器。
修改已有产品的毛利率仅 lookup 来源行重算；direct 来源行不受影响。不自动新增产品，不静默替换原始毛利。
筛选或隐藏明细行默认仅改变可见行，不改变汇总范围。明确 SUMIFS 不随 AutoFilter 改变；若未来要“仅可见行汇总”，作为独立模式实现及测试。
改变期间、维度、客户、产品集合，追加或删除行首版不承诺；从系统重新选择范围并导出。Table 能扩展不等于分组报表、去重及状态范围全部正确扩展；锁定结构以防误解。
修改公式默认保护防误操作；允许授权副本解除保护后自行分析。解除保护后的结果不属于系统验证范围；工作表保护不是安全边界。

源事实 ID 或稳定行键只用于追溯；本地模拟状态可通过可编辑输入与导出初值比较生成。任何数据版本 SHA 都只标识服务器来源，不能认证已被本地修改的文件。

+

07 / 安全、权限与 33 万行性能

可信公式与不可信内容严格分离

任务化导出

保留旧同步证据入口的行为；新增公式版任务状态：queued → running → succeeded / failed / expired。请求校验后冻结数据版本、映射、筛选、权限范围、参数和公式版本。worker 用分块迭代处理，临时文件原子完成后才发布；支持超时、失败重试、去重请求、崩溃恢复和过期清理。

真实附件 332,830 行未超过 Excel 单表 1,048,576 行的限制，但当前接口的 100,000 行限制仍然存在。完整公式版必须另做容量评估，超过支持范围应显式拒绝或提示缩小范围，不能悄悄截断。

性能基准至少覆盖 1,000 / 100,000 / 332,830 行，测导出耗时、峰值内存、文件大小、首次打开、改值重算、并发排队。记录设备、Excel 版本和数据分组基数；本轮不虚构 SLA。高基数分组达到预算时提示缩小报表范围，不降级为静态结果却仍标“公式版”。

+

08 / 文件级实施步骤

阶段工作与目标文件完成条件 / 依赖
P0 · 口径冻结
估计 1–2 人日		复核 finance/analysis.py、finance/importing.py 与原附件；定义精度政策、公式清单、编辑边界及兼容矩阵；确认毛利来源恢复。财务确认收入 N / 实收 I、原价、退款、自营产品、成本与 ROI；否则不可冻结黄金结果。
P1 · 最小公式工作簿
估计 2–4 人日		新增 finance/exporting.py 集中生成模板、状态和公式；复用现有分析口径，不引入通用公式 DSL。先实现小样本汇总、明细、参数、说明和对账。每个衍生单元格可追溯；输入/公式/文本写入类型分离；小样本真实 Excel 重算通过。
P2 · 精度与隐私
估计 1–2 人日		同模块实现精确文本、数值转换预检、毛利来源、客户 token、来源定位和注入防护；增加针对性单元测试。Decimal 对账、去重、缺失、恶意文本及越权用例通过；不破坏证据版。
P3 · 任务与 API
估计 2–3 人日		在 finance/models.py 增加必要的导出任务与迁移；在 finance/views.py 及现有 URL 配置增加提交/查询/下载入口；新增导出 worker 命令，复用导入 worker 的部署模式。完整附件能异步生成；崩溃恢复、重试、权限撤销、留存清理可验证。schema_version、mode、context、compatibility 等字段由服务端白名单校验。
P4 · 网页交互
估计 1–2 人日		frontend/src/dialogs.tsx、types.ts、api.ts 按需调整：模式选择、兼容性与精度说明、任务状态、错误反馈、下载和重试。桌面/手机可用；不显示无依据百分比；错误不丢失筛选上下文；CSV 与旧 XLSX 行为清晰保留。
P5 · 引擎与全量验收
估计 3–5 人日		新增 tests/test_exporting.py、扩展 tests/e2e/workspace.spec.ts；编写受控重算与差异报告脚本；更新 docs/acceptance.md。下节所有适用门禁有可复现记录；33 万行性能验证及财务签字完成。

合计粗估 10–18 人日，用于排期，不是承诺。受控 Excel 测试环境、财务确认和超预算性能优化可能增加日历时间。P1 小样本原型通过后再投入大文件任务流程；各阶段保持可单独审查和回滚。

兼容 API：旧 format=xlsx 默认仍为证据快照；新公式模式显式申请任务并返回任务 ID。服务端返回实际导出模式与版本，前端不凭文件后缀推断。

+

09 / 严格测试与上线门禁

写入公式不等于公式算对。

openpyxl 不执行计算；设置自动重算或 fullCalcOnLoad 也不是测试结果。必须在目标 Excel 引擎执行、保存，再同时读取公式和结果缓存验证。

测试层必须覆盖的场景通过标准
结构与公式覆盖每页、命名范围、Table、单元格类型、公式依赖、说明及数据验证；空数据导出。衍生区域全覆盖真实公式，基础输入无意外公式；无循环引用、损坏修复提示或外部链接。
财务黄金样例两种 ROI、direct/lookup 毛利、退款、负数、0、缺成本、部分缺字段、缺期、基期0、新增/消失分组。与独立 Decimal 预期一致到批准精度；人数完全相等；缺失不能变0。
客户去重同人跨科室/产品/月，重复明细、自营与非自营交集、空客户字段。总体不相加子组人数；升单率分子去重正确；缺客户状态与系统一致。
真实引擎重算Excel Windows / Mac；打开重算保存；分别读取 formula 与 data_only；修改收入、成本、原价及毛利率后再次计算。受影响链路变化符合独立预期，非依赖指标不误变。扫描 #REF!、#DIV/0!、#VALUE!、#NAME?、#SPILL!；仅允许明确缺失状态对应的 #N/A。
精度极限15位边界、24位小数、极大金额、正负抵消、舍入临界值、33万行累计差异。精确文本不变；超出政策明确报警或拒绝；不通过放宽误差掩盖错误。
注入与越权恶意标签/原始公式、工作表名特殊字符、跨组织/科室、任务 ID 猜测、下载前撤权。无执行不可信公式、外链或敏感原值泄露；未授权下载拒绝并审计。
任务可靠性重复提交、worker中断、磁盘不足、超时、并发、到期清理。无半成品下载、孤儿文件和静默截断；失败可解释，恢复无重复发布。
前端与兼容回归两模式选择、状态、错误重试、下载、移动端；现有 CSV/XLSX。浏览器端到端通过；旧证据版仍保持精度与公式注入防护。新公式版测试只允许白名单公式，不删除旧安全断言。
真实附件全量332,830行；2025-08/2026-08及原映射；全量与分组对账。行数不缺失，金额满足政策，人数为33,147 / 38,361；缺成本 ROI 不可用，未配置自营范围不擅自补值。

发布清单（全部待实施验证）

上线先小范围授权用户试用公式模式，旧证据导出保留作为回退。只有 Excel 导出门禁通过，不意味着整个财务系统所有商业上线门禁自动满足。

+

10 / 需确认的业务决策

可直接采用的工程默认

  • 双模式并存，公式版 XLSX 不含宏和外链。
  • 精确证据 + 数值计算 + 对账，不牺牲原始精度。
  • Microsoft 365 为首版公式验收目标。
  • 固定范围、指定输入可改，结构变化重新导出。
  • 大文件异步、授权下载、客户去标识。

上线前必须由财务确认

  • 收入继续使用 N 列，还是另需 I 列实收口径？
  • 舍入在每行还是最终汇总发生？使用哪种舍入规则与容差？
  • 毛利率允许本地模拟吗？直接毛利是否允许修改？
  • 自营产品集合、退款计人、成本归属及 ROI 口径是什么？
  • 是否必须支持旧版 Excel/WPS，以及首次打开前就有结果缓存？

建议实施顺序：先确认口径与 Excel 版本 → 小样本真正公式与重算验证 → 33 万行任务化及性能测试 → 前端完整交互与财务签字。不要先把导出值改成数值、加几条 SUM，就标记需求完成。

+
+ + diff --git a/docs/improvement-plan.html b/docs/improvement-plan.html new file mode 100644 index 0000000..0ae7886 --- /dev/null +++ b/docs/improvement-plan.html @@ -0,0 +1,36 @@ + + + + + +TraceIQ 完善项与执行路线图 + + + +
TRACEIQ / 优先级核查与执行 / 2026-09-24

完善项与执行路线图

以当前源码、自动化测试和可访问环境为依据。测试通过不等同于业务签字或生产上线。

+
+
本轮基线：后端 40 passed，finance 覆盖率 86.72%（85% 门槛）；隔离 SQLite Playwright 3 passed、原附件专测 1 skipped；Ruff、格式、Django system check、迁移检查、TypeScript/Vite 构建、npm audit（0 漏洞）与 pip-audit 通过。production 模式 Django check --deploy 零告警。Docker daemon 未运行，本机无 LibreOffice。Microsoft Excel、332,830 行公式工作簿、真实 AI/合规及商业部署仍未验收。
+

P0 · 先确保验收证据可信

+
P0.1 · 已完成 / 已复核

维护真实的源码与验收基线

旧文档曾错误声称 tests/test_finance.py 为空并把公式导出标记为未实现；当前源码有 40 项后端测试，公式工作簿、说明页、对话新增列、追问与下载均已实现。本轮路线图按当前代码、测试结果和外部门禁重建。

+
P0.2 · 已修复并通过专项回归

隐藏导出任务的内部异常

状态接口不再把 worker 异常文本发给用户，只返回通用失败提示；前端显示任务编号，数据库保留内部诊断。失败任务脱敏、临时文件删除及状态访问专项 8 passed。

+
P0.3 · 本机生产配置检查通过

验证 production 部署安全配置

production 模式、有效测试密钥、受信 HTTPS 代理下 manage.py check --deploy 零告警。真实域名、TLS 终止及代理头行为仍需部署环境验收。

+
P0.4 · 外部目标引擎

Microsoft Excel 公式重算与兼容验收

目标 Excel Windows/Mac 打开、重算、保存并核对公式及缓存；覆盖缺期、零分母、退款/负值、ROI、毛利来源和输入修改依赖。LibreOffice/openpyxl 不能替代目标 Excel 证据。

+
P0.5 · 真实数据与资源基准

332,830 行完整公式工作簿

公式导出当前限制 100,000 行/200 组。用原附件和可靠 PostgreSQL 验证行数、金额、人数、耗时、峰值内存、文件大小及 Excel 首开/重算；未完成前保持显式拒绝，不静默截断或放宽上限。

+
P0.6 · 外部业务/安全责任人

财务口径、真实 AI 与商业发布批准

签署实际收入与实收、退款、成本、毛利、ROI、自营集合及舍入规则；审批外部模型数据处理/出境并演练故障；完成生产密钥、TLS、备份恢复、SLA、渗透测试和商业许可证核验。审批前 AI 保持关闭。

+
+

P1 · 代码层可靠性与可用性

+
P1.1 · Docker daemon 未运行

镜像与 Compose 运行验收

Docker CLI 可用但 daemon 不可连接，本机暂不能构建或启动。daemon 可用后验证镜像、健康检查、worker、清理服务与重启恢复；远端 CI 需仓库平台实际触发。

+
P1.2 · 产品范围待确认

自定义指标定义保存与复用

定义当前只用于当次工作簿，不持久化到主分析页或跨会话定义库。需明确个人/组织共享、版本审批、删除和审计规则，再开发持久化。

+
P1.3 · 需产品确认后开发

大数据与高基数交互

当前汇总按请求返回分组、证据使用分页；AG Grid SSRM/Pivot/Tree Data 等计划功能尚未实现。先以目标数据规模和典型交互决定是否投入服务端行模型，避免未经验证引入商业授权与运维负担。

+
+

P2 · 范围扩展

+
P2.1 · 财务定义确认后

扩展公式语言和缺失策略

当前白名单仅支持现有指标/常数和加减乘除，最大 8 层；不接受任意 Excel 文本、字段引用、跨明细新聚合、按零替代或跳过缺失。每项扩展需独立财务黄金样例、注入测试和公式重算验证。

+
P2.2 · 目标明确后

分析及工作簿操作增强

可评估历史分位异常、告警、证据期间切换、明细表结构变更后的公式扩展、模板版本和工作簿内定义管理；需先给出场景和验收指标。

+
+

本轮执行结果

项目结果剩余门禁
源码与路线图基线已完成；纠正文档过时状态以当前代码/测试为准
导出错误脱敏8 项专项后端测试通过内部诊断仅保留服务端
Production deploy checks零告警真实域名、TLS/代理验收
全量本机回归40 passed / 86.72%；Playwright 3 passed、1 skipped原附件场景需配置隔离数据集
商业发布门禁尚未通过Excel、真实规模、业务/AI审批、容器和灾备
+

发布判定原则

本地质量门禁全部通过，只证明自动化覆盖范围内的代码状态。商用发布还需要 Microsoft Excel 结果、真实规模资源记录、财务口径签字、AI 数据处理批准和目标基础设施安全/灾备验证。任何一项未满足时，状态保持“未完成发布验收”。

+
+ + diff --git a/docs/improvement-roadmap-2026-09-24-history.html b/docs/improvement-roadmap-2026-09-24-history.html new file mode 100644 index 0000000..56e75e6 --- /dev/null +++ b/docs/improvement-roadmap-2026-09-24-history.html @@ -0,0 +1,42 @@ + + + + + +财务 Agent 改进路线图 + + + +
TRACEIQ / 工程质量与产品改进 / 2026-09-24

财务 Agent 改进路线图

按财务正确性、测试可信度、业务覆盖和发布风险排序；实现状态与验收状态分开记录。

+
+
当前结论：对话新增指标、缺项追问、白名单公式编译与公式工作簿已实现；本轮专项后端 6 项、Ruff、前端类型检查及生产构建通过。生产发布仍被空的历史测试源码、公式引擎重算、真实附件规模和财务/AI 合规审批阻断。
+

P0 · 发布阻断

+
P0.1 · 阻断中

恢复完整源码测试基线

tests/test_finance.py 当前为 0 字节，导致既有财务、权限、导入、分析与公式导出回归测试缺失。先从可靠备份恢复，再运行 pytest、Ruff、Django、迁移、生产构建及 Playwright。旧 .pyc 的 69 passed、1 failed、1 skipped 使用旧断言，不是当前源码验收证据。

+
P0.2 · LibreOffice 自定义公式与缓存已验证；Excel 待验

公式/缓存一致性验证

人数逻辑使用 CustomerRoster + COUNTIFS。本轮从当前源码生成基础及新增列工作簿，交由 LibreOffice 重算后读取；含自定义列的文件保留 136 个公式单元格，自定义“收入成本差”公式缓存及汇总引用缓存均为 2,000。仍需补足缺失期、零分母及负值等边界对账，并在 Microsoft Excel 桌面版复验。

+
P0.3 · 待目标环境

Microsoft Excel Windows/Mac 验收

在目标桌面 Excel 打开、强制重算、保存，读取公式和结果缓存；覆盖缺期、零人数、退款/负数、两种 ROI、毛利来源及编辑金额后的依赖变化。

+
P0.4 · 范围阻断

332,830 行附件全量导出与资源预算

当前上限 100,000 行/200 组。用原映射和可靠 PostgreSQL 验证完整导出、行数/金额/人数对账、耗时、峰值内存、文件体积及 Excel 首开/重算；未经实测不放宽限制，不静默截断。

+
P0.5 · 外部决策

财务口径与 AI 合规

签署实际收入/实收、退款、成本归属、毛利、ROI、自营范围和舍入口径；完成外部模型联调、隐私/数据出境审批及故障演练。批准前保持外部 AI 关闭。

+
+

P1 · 可靠性与运行质量

+
P1.1 · 已实现；专项 3 项通过

过期工作簿与孤儿临时文件清理

新增 cleanup_formula_exports 管理命令和 Compose 常驻清理服务。仅清理过期成品及超过 24 小时的临时文件，限制导出根目录，跳过符号链接；清理异常保留重试线索。已覆盖过期文件、临时文件年龄和目录外路径测试。

+
P1.2 · 功能实现；单测及新增列 E2E 通过

对话定义新增汇总列并导出公式

用户可描述列名、含义、计算、范围及缺失/零分母策略；缺项时模型必须追问，确认后服务端只接受白名单表达式树并编译为 Excel 公式。工作簿包含汇总及自定义指标页、使用说明和对账页。专项单测覆盖追问、表达式拒绝与公式输出；本轮 Playwright 已验证登录、字段导入、缺项追问、补充确认、工作簿生成和下载。

+
P1.3 · 待目标环境

部署、安全与灾备验收

完成镜像构建、TLS、数据库认证、备份恢复和 RPO/RTO 演练、并发/SLA 测试、渗透测试、密钥轮换、审计告警及商用依赖许可证核查。

+
+

P2 · 产品能力扩展

+
P2.1

已签口径下补充分析能力

多收入字段与消费比率、生效期毛利率、退款切换、结构占比、历史异常阈值和告警投递。

+
P2.2

自定义指标定义复用

定义保存/版本与差异审阅；新运算符、聚合或缺失策略须通过白名单、独立财务样例和安全审查。

+
P2.3

工作簿及跨平台体验

证据期间切换、平板与无障碍优化、可读的公式错误状态、更多电子表格引擎兼容验证。

+
+

完成门禁

+ + + + + + +
门禁证据状态
源码测试恢复原测试文件并完整执行阻断：源码缺失
静态与构建Ruff、Django、迁移、前端构建本轮独立复跑通过；源码恢复后仍需联跑
网页工作流新增指标、桌面、手机 Playwright3 项通过，1 项原附件全量场景按配置跳过
公式重算当前版本 LibreOffice 与 Microsoft ExcelLibreOffice 自定义公式重算/缓存通过；边界样例与 Excel 待验
真实规模原附件全量导出与资源报告待验；当前上限 100,000 行
商业发布财务签字、AI 合规、安全与灾备待外部审批/目标环境
+
+ + diff --git a/docs/improvement-roadmap.html b/docs/improvement-roadmap.html new file mode 100644 index 0000000..7ba55bf --- /dev/null +++ b/docs/improvement-roadmap.html @@ -0,0 +1,44 @@ + + + + + +TraceIQ 财务 Agent 改进路线图 + + + +
TRACEIQ / 工程质量与产品改进 / 2026-09-24

财务 Agent 改进路线图

依据当前仓库和本轮实测重排；实现状态、外部验收与生产资格分别记录。

+
+
当前结论：公式导出审查已修复文本条件、未分类对账、公式注入、任务路径、文件权限与失败恢复问题，并改为流式写入；后端 61 passed、finance 覆盖率 87.66%，LibreOfficeDev 公式重算共 4 项通过。100,000 行产品上限实测峰值 RSS 约 136.4 MB。Microsoft Excel 实机、原附件 332,830 行范围方案、真实 AI/财务审批和目标环境验收仍未完成，不能认定具备生产商用资格。
+

P0 · 正确性与发布阻断

+
P0.1 · 本机 LibreOffice 回归通过

公式重算建立可重复黄金测试

当前共 4 项 LibreOfficeDev 重算测试：test_formula_recalculation.py 3 项、test_formula_recalculation_gross.py 1 项，覆盖自定义汇总及变化率、明细收入编辑联动、零成本 ROI、缺失基期及 direct/lookup 毛利来源。它们只证明这些具体样例在 LibreOffice 中的结果，不代表 Microsoft Excel 兼容或财务签字。

+
P0.2 · 目标环境门禁

Microsoft Excel Windows / Mac 实机验收

本机曾以 Microsoft Excel 打开测试工作簿，公式栏确认汇总公式包含 COUNTIFS / SUMIFS；但 Excel 显示只读（View Only / 需激活 Microsoft 365），无法保存重算副本，且检查到的样例分组为 Test、与预期不匹配。故仅有“公式可见”的证据，Excel 重算结果、缓存和业务样例均未验收。后续须在可编辑的目标 Windows/Mac Excel 中用受控正确样例强制重算、保存，并核验函数兼容、数据表引用、错误单元格和输入修改依赖链。

+
P0.3 · 100,000 行产品上限已优化并实测

流式公式工作簿与原附件范围决策

公式版本 3 已将所有工作表改为 openpyxl write-only 顺序写入，同时保留 Data 与 CustomerRoster 表格、结构化引用、说明、证据和对账。隔离 SQLite 合成基准：20,000 行 5.26 秒、3.27 MB、峰值 RSS 77.9 MB；100,000 行 25.96 秒、16.22 MB、峰值 RSS 136.4 MB，ZIP/OOXML 完整性及公式读取通过。生产限制仍为 100,000 行/200 组；真实附件 332,830 行不在当前产品范围，下一步必须选择可审计分卷或明确提高上限后再做原附件流式资源、打开和重算验收。

+
P0.4 · 财务审批

冻结收入、退款、毛利及 ROI 口径

由财务负责人签署 N 列实际收入与 I 列实收、原价、退款、成本归属、毛利率、自营产品集、升单定义与舍入精度。原附件成本全缺，ROI 应维持不可用；不得以技术测试替代业务签字。

+
P0.5 · 隐私 / 模型外部审批

外部 AI 真实联调与数据处理审批

在批准前保持 AI 关闭。审批后验证模型版本、超时、限流、拒答、审计脱敏、数据处理与出境依据；模型只提出列定义，服务端继续严格校验白名单表达式并要求用户确认。

+
+

P1 · 商用运维与可恢复性

+
P1.1 · 本地待执行 / 环境受限

Docker 镜像与生产 Compose 验收

本轮完整生产变量下 Django check --deploy 无告警，依赖审计为零已知漏洞；Docker daemon 当前不可连接，镜像构建尚未验证。daemon 恢复后构建镜像、运行迁移和健康检查，确认非 root、只读根文件系统、卷权限及 worker。

+
P1.2 · 目标环境门禁

备份恢复、并发和安全运营

验收 TLS、密钥轮换、最小权限、加密备份及恢复 RPO/RTO、队列积压/失败告警、访问日志脱敏、容量阈值、并发 SLA 和独立安全审查；AG Grid Enterprise 使用前确认商业许可。

+
P1.3 · 部分已实现；范围受限

对话定义新增汇总列并导出公式

用户可描述列名、含义、计算、适用范围及缺失处理；信息不全时追问并结合完整对话复核。公式只使用数据集当前可计算的既有指标与白名单运算，服务端再次校验数据映射；确认后写入公式工作簿并带说明。任意 Excel 函数、跨明细新聚合、定义保存复用及导出后自动扩行未实现。

+
+

P2 · 扩展能力

+
P2.1

签字口径下扩展分析

多收入字段与消费比率、生效期毛利率、退款策略、结构占比、历史异常阈值和告警投递。

+
P2.2

跨平台与辅助功能

补齐平板、键盘/屏幕阅读器、证据期间切换和目标表格软件兼容矩阵；Web 可复用不等同于 Excel 文件跨引擎兼容。

+
+

本轮证据

+ + + + + + + + +
项目结果边界
后端及覆盖率61 passed；finance 87.66%SQLite 测试；包含流式工作簿、公式语义、注入防护、任务路径/权限/恢复及 LibreOffice 重算；未替代 PostgreSQL CI
LibreOffice 公式重算共 4 项通过（test_formula_recalculation.py 3 项、test_formula_recalculation_gross.py 1 项），direct/lookup 毛利来源测试包含在这 4 项内LibreOfficeDev；Microsoft Excel 仅确认公式可见，未完成可保存重算验收
流式容量基准100,000 行：25.96 秒、16.22 MB、峰值 RSS 136.4 MB；20,000 行：5.26 秒、3.27 MB、峰值 RSS 77.9 MB隔离 SQLite 合成数据，本机单次测量；生产上限保持 100,000 行，不能替代目标环境并发 SLA
浏览器回归3 passed，1 skipped跳过需显式指定附件的原附件 E2E
构建 / 静态检查Ruff、Prettier、TypeScript/Vite、Django check、迁移检查通过依赖审计报告 0 个已知漏洞；不等于渗透测试
生产配置检查完整临时生产配置下 check --deploy 无告警不代表目标基础设施上线验收
Compose / Docker未通过仓库无真实 .env；Docker daemon 未运行，未构建镜像
Excel / 原附件公式Excel 打开样本并确认公式存在；PostgreSQL 对账完成；旧非流式版本曾生成 332,830 行Excel 只读，无法保存重算副本；所见分组与预期不匹配。原附件超过 100,000 行产品上限，公式版本 3 尚未对 332,830 行做越界放宽测试
+

执行顺序与完成规则

顺序：维护 P0.1 黄金重算 → P0.2 Microsoft Excel → P0.3 资源优化 → P0.4 / P0.5 外部审批 → P1 → P2。本轮补齐了可重复的金额编辑、零成本与缺失基期重算测试，并复核新增列映射保护。LibreOffice 不替代 Excel，财务/AI 审批和目标基础设施测试需对应责任方及环境。

测试必须覆盖代码实际行为；每项门禁记录日期、环境、版本、样例、结果和限制。不能用 mock、静态公式存在或 build 通过声称生产已验收。

+
+ + diff --git a/docs/remaining-issues-2026-09-26.html b/docs/remaining-issues-2026-09-26.html new file mode 100644 index 0000000..4910419 --- /dev/null +++ b/docs/remaining-issues-2026-09-26.html @@ -0,0 +1,77 @@ + + + + + +财务 Agent：剩余问题与修复清单 · 2026-09-26 + + + +
TRACEIQ / 工程现状与发布门禁 / 2026-09-26 / Asia/Singapore

财务 Agent：剩余问题与修复清单

面向下一轮开发、财务验收及生产部署。按当前源码和最近完成的测试记录整理，不将历史计划或测试通过等同于商用准入。

+
+ +

一、结论与阅读方式

+
目前不能认定达到生产环境商用标准。

公式导出、说明页、对话追问、新增公式列、个人定义保存复用与证据分页已经实现；仍有需要修复的安全/交互边界、未完成的产品能力，以及必须在目标 Excel、数据库、基础设施和业务审批环境关闭的门禁。

+

本次交付仅新增本报告，不修改业务代码、不执行新的数据库迁移。以下“已完成”包括此前连续开发完成的改动；“静态确认”指源码可直接看出的行为，尚未新增专项复现测试，不能冒充测试复现结果。

+ +
+

二、当前已经完成什么

+
+ + + + + +
能力已完成状态不能据此推断的内容
公式工作簿版本 3，流式生成计算明细、汇总、使用说明、参数口径、人数去重、原始证据、对账；有自定义列时增加自定义指标页。不是任意 Excel 模板生成器；不是全部输入/结构变化都能正确重算。
对话新增列描述不全时追问，补充后确认；服务端校验表达式树及数据映射可用性。支持既有指标、常数、加减乘除。真实外部模型未完成联调；自然语言适用范围不等于执行任意筛选、聚合规则。
个人定义库按用户与数据集隔离，保存、同名更新、列出、载入、删除；保存和导出时校验；迁移文件 0004 已存在，隔离测试库已执行。不是团队共享定义库；没有草稿续聊、版本审批、并发编辑冲突保护。非测试环境是否迁移仍需核对。
证据与文件导出本期/基期切换、每页 50 条及翻页；CSV/XLSX 证据以临时文件和 FileResponse 分块发送；金额文本和公式注入防护保留。前端仍用 Blob 接收完整下载文件；临时磁盘/tmpfs、并发容量尚无生产 SLA。
导出可靠性任务专属路径、0600 文件与 0700 目录、失败状态恢复、随机临时文件、授权范围检查、过期清理。并非所有符号链接异常路径都安全，见 F01；也未完成磁盘故障及生产崩溃恢复演练。
+

最近完整测试记录

+ +

现有 acceptance.md / improvement-roadmap.html 仍存在 61 项测试、“没有定义库”等旧表述；历史 Excel 计划也有相互矛盾的状态提示。当前状态以本报告与最近完成的实测为准，后续应统一维护单一验收入口。

+
+

三、需要继续修复的代码问题

+
F01 · P0 · 静态确认 · 文件安全

失败清理可能沿符号链接父目录删除非任务文件

位置：finance/exporting.py → process_formula_export()。先设置 target，再检查 target.parent 是否为符号链接；拒绝后进入异常分支，仅检查 path.is_symlink()，未检查父目录。父目录为链接而最终文件本身为普通文件时，unlink 会跟随父目录删除链接目标下同名文件。

影响：需要本地存储目录被操纵这一前提，不能描述为普通远程用户已可利用；但违背“异常清理只删除自己创建的文件”的边界。

推荐修复：路径完整验证前不要将目标登记为可清理文件；只清理本次实际创建、且位于受控目录中的文件，必要时用目录描述符及不跟随链接的文件操作。不要用失败清理代替旧孤儿文件回收。

验收：父目录链接到受保护目录，预置同名文件；执行必须失败且受保护文件字节不变，任务为 failed；覆盖根目录链接、目标链接、写盘失败与正常重试。

+
F02 · P1 · 静态确认 · 故障清理

清理失败被忽略，敏感孤儿文件缺少回收路径

位置：finance/exporting.py 与 cleanup_formula_exports.py。异常清理吞掉 OSError 后清空 job.path；清理器主要处理已成功且过期的文件及旧 .tmp，失败任务残留 .xlsx 可能不再被追踪。

修复：记录不含敏感数据的清理失败日志及任务编号，为安全范围内的孤儿 .xlsx 制定保留期与回收规则；不要删除仍处于活动任务的文件。

验收：注入 unlink 权限错误，任务失败状态仍保存；可观察到告警，恢复权限后回收成功，其他任务文件不受影响。

+
F03 · P1 · 静态确认 · 证据弹窗

重复点击当前期间可能让已加载列表停在“加载中”

位置：frontend/src/dialogs.tsx → EvidenceDialog.selectSide()。第 0 页点击当前已选期间会清空 result，但 side/page 没变化，依赖这些值的 effect 不会重新请求。

修复：同期间同页直接返回，或明确触发刷新；补加载、空集、失败重试状态。

验收：当前期间重复点击、快速本期/基期切换、第二页切期间、空列表和请求失败均不显示过期数据或永久加载。

+
F04 · P1 · 静态确认 · 定义复用后的对话

载入已保存定义后追问，没有把原定义带给模型

位置：FormulaExportDialog.loadDefinition()/send()。载入定义会清空 messages，随后只发送新增描述；“把刚才公式改成乘以 100”之类指代不能可靠关联已载入表达式。

修复：在可见、经同意的对话上下文中明确附带原定义，或建立服务端定义引用；仍要求重新确认，不让旧定义静默覆盖新解释。选择空选项也应同步清理当前定义，避免选择状态与预览不一致。

验收：保存 A、重开载入 A、发送依赖 A 的修改，检查模型请求实际包含 A；未发送补充时禁止生成，修改后生成的公式符合新定义。

+
F05 · P1 · 静态确认 · 定义库并发与审计

50 条上限不是原子约束，同名更新没有冲突检测

位置：finance/formula_views.py → formula_definitions()。先 count 再 update_or_create，并发不同名字可能突破上限；同名更新采用最后写入覆盖，未携带 revision。保存/删除与审计也不在同一外层事务内。

修复：事务内锁定稳定的用户/数据集所属行，原子检查配额；引入 revision 和 409 冲突响应；变更和审计一同提交，保存增加合理限流。前端明确同名更新行为。

验收：PostgreSQL 并发保存不能产生第 51 条；两个标签页更新不静默丢失；模拟审计写入失败后定义变更回滚。

+
F06 · P1 · 静态确认 · 工作簿可解释性

“产品毛利率可编辑”说明与实际公式引用不一致

位置：finance/exporting.py → 参数与口径 / GrossCalc。参数页毛利率写成字典文本并标为可编辑，实际明细公式引用每行 MarginRate，修改参数页文本不会联动。说明页还提到“本页分组维度”，但没有完整的维度键与显示名称字典。

推荐：近期明确参数页为只读快照，写清需要修改计算明细的哪类输入；如必须按产品统一模拟，则实现真实产品毛利率参数表与 lookup 依赖，并验证 direct 毛利不受影响。

验收：每个标为可编辑的输入都对应真实依赖；收入、成本、原价、直接毛利、lookup 毛利率分别改值重算，说明页列出字段、单位、来源、缺失含义和固定结构边界。

+
F07 · P1 · 静态确认 · 财务校验范围

对账采用统一 0.01 阈值，自定义公式没有独立服务器参考

位置：finance/exporting.py → 对账检查。人数、金额、比率共享差异阈值；对小数形式的比率，0.01 是一个百分点，未必适用业务。自定义列标记“需业务验收”，不是自动对账通过。

修复：财务批准后按指标配置比较规则：人数严格相等、金额按币种和舍入精度、比率独立阈值；用 Decimal 白名单表达式解释器计算自定义指标参考，不能执行任意代码。保持原始金额文本不变。

验收：阈值边界、大数、抵消、极小值、除零、缺失和多层表达式独立黄金样例通过；不得为消除报错而放宽容差。

+
F08 · P1 · 验证缺口 · 权限撤销与任务生命周期

执行/下载阶段需要覆盖角色撤销、超时与崩溃恢复

当前有组织、任务所有者、科室范围检查，但 worker 不重新要求 admin/analyst 角色；下载接口也未声明同创建接口的角色白名单。是否允许被降为 viewer 的账号下载旧导出，需要明确政策，不能默认授权语义已完整。

推荐：敏感公式工作簿在申请、执行、下载时统一核验角色、组织、账户有效性和科室；补任务超时、重复请求、过期下载、worker 发布后崩溃的测试及运维告警。

验收：排队后撤权、成功后撤权均按签署政策处理；半成品不能下载；重启不重复发布或无限停留 processing。生产 PostgreSQL 锁语义单独验证。

+
+

四、尚未完成的功能与产品边界

+
+ + + + + + + + + +
优先级 / 功能当前缺口建议与完成标准
P1 · 新列出现在网页汇总表当前新增指标主要进入导出的自定义指标页，并未成为主分析表持久化可计算列。若原需求包含网页表格，需新增受控服务器计算、列选择与展示；网页结果、导出公式、Decimal 参考使用同一语义，缺失不转零。
P1 · 多列编辑与定义生命周期API 最多 10 列，但对话弹窗生成时仅提交当前单列；未完成草稿续聊、版本历史、团队共享和审批。先做当前数据集的多列列表与显式确认，后做草稿。个人定义保存复用已完成，不应重复开发。团队共享另设权限和版本策略。
P1 · 适用范围与表达式能力scope 为说明文本，实际按当前分组、本期/基期计算；不能表达任意条件、跨明细新聚合、任意 Excel 函数、维度列或其他缺失策略。模型明确拒绝不支持范围，不用自然语言说明冒充已执行规则。扩展时结构化范围并加入服务端白名单及黄金测试。
P1 · Excel 编辑防误操作人数辅助表、期间、身份、维度为固定快照；追加/删除明细不能保证全部汇总与去重同步。输入验证、公式保护、完整字段字典未全部实现。首版推荐固定结构、明确可编辑金额列并防误操作；结构改变需重新导出。表保护不是加密或真正权限控制。
P1 · 原附件 332,830 行导出当前产品仍限制 100,000 行 / 200 组；不能一次完整导出原附件。推荐可审计分卷并保留全局汇总/去重口径，不能把分卷人数简单相加。备选是正式提高上限后验收内存、并发、Excel 首开和改值重算，不能仅改常量。
P1 · 结果缓存与大文件下载openpyxl 不执行公式；未重算文件的预览/缓存读取可能为空。浏览器 Blob 下载完整驻留内存；生产 /tmp 为 tmpfs，临时文件仍计入容器内存。明确必须由目标引擎重算；如需要服务器生成已缓存副本，独立评估计算引擎。大文件直接附件下载及临时存储配额需实测。
P1 · 高基数与可访问性聚合分组全量返回；缺高基数结果窗口、平板专项、键盘/屏幕阅读器专项。浏览器自动测试不等于人工可访问性验收。先测真实高基数与并发，再决定分页或 SSRM；补 focus、Esc 关闭、错误提示、表头关联与触控专项。
P2 · 扩展业务能力多收入字段口径、消费比率、生效期毛利率、退款策略、结构占比、历史阈值、告警渠道和团队调查分派未完整实现。先冻结业务定义，再逐项开发，不擅自以现有收入字段替代不同含义。
P2 · 跨平台与高级网格已有响应式 Web，没有原生客户端；AG Grid SSRM/Pivot/Tree Data/Master-Detail 未完整接入。Web 是复用基础，不等于原生交付。按真实需求和商业授权选择 PWA/桌面封装或高级网格。
+
+

五、必须外部完成的发布门禁

+
+ + + + + + +
门禁现状 / 责任方关闭证据
P0 · Microsoft Excel 实机历史只读 Excel 仅确认公式可见，无法保存重算；样例分组也与预期不符。财务、QA、目标终端负责人。在可编辑 Windows/Mac Excel 用受控样例打开、强制重算、保存；读取公式和结果缓存，改值后依赖链正确，无非预期错误与修复提示。
P0 · 财务签字与精度政策原附件收入/实收定义有歧义，成本全缺；财务负责人。签署收入、退款、原价、成本归属、毛利、ROI、自营、升单和容差。缺成本 ROI 保持不可用；不以技术测试替代业务认可。
P0 · AI 真实联调及隐私外部响应主要以 mock 验证；对话文本与筛选标签可能包含敏感信息。合规、数据负责人、AI 服务负责人。审批数据处理/出境、模型版本、留存策略；真实验证超时、拒答、限流、非法输出、复杂 schema 接受度。批准前保持 AI 关闭。
P0 · 目标基础设施历史 Docker daemon 不可用、真实 .env 不存在；配置已交付，不等于构建和运行已验收。运维、安全。目标镜像构建、迁移 0004、非 root/卷权限、TLS、代理、数据库认证、队列、备份恢复、日志脱敏、安全扫描与许可证通过。
P1 · 当前版本 PostgreSQL 与 SLA最近 65 项全量是 SQLite；历史 PostgreSQL 对账/测试不能替代本版并发和锁语义。在独立 PostgreSQL 执行当前测试、并发定义保存、worker 恢复、导出压力；记录 P95、资源预算、备份恢复 RPO/RTO。
P1 · 安全运营和许可没有完整 SSO/MFA、自助找回、数据库 RLS、不可变审计和外部告警；并非所有产品必须全选，但必须作安全决策。风险评估明确必需项；验证加密存储、密钥轮换、权限回收和安全审查；启用 Enterprise 前取得对应商业许可。
+
+

六、推荐修复顺序

+
  1. 安全收尾：先 F01、F02、F08，明确文件清理边界与撤权政策，添加失败注入和越权回归。
  2. 日常交互：修 F03、F04；补定义库删除、空选项、失败重试、快速切换和重复点击 E2E。
  3. 持久化可靠性：修 F05，做 PostgreSQL 事务、配额和冲突测试；迁移验证应覆盖既有数据库。
  4. 工作簿真实性：修 F06，完善输入/参数字典与保护；F07 的容差需先由财务确认，再增加独立数值对账。
  5. 补齐原始用户目标：确认网页主汇总表也需要新列后，实现网页计算、多列选择与导出一致性；完成草稿续聊。
  6. 发布前：解决原附件范围，执行 Excel、真实 AI、财务、安全、基础设施和并发验收。任何一项未关闭都保留明确限制，不标记“全功能商用完成”。
+

没有给出未经测量的工期承诺。前三项主要由工程承担；财务口径、数据处理审批和目标环境依赖必须明确责任人，不能通过延长本地编码替代。

+
+

七、下一轮验收与交付清单

+ +

源码与相关材料

+

公式生成与任务发布 · 定义库与导出 API · 证据导出与分页 · 网页对话与证据交互 · 测试目录

+

既有验收记录（部分状态待同步） · 历史路线图 · 最初 Excel 实施方案（历史设计，不代表现状）

+
+
+ + diff --git a/finance/__init__.py b/finance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/finance/analysis.py b/finance/analysis.py new file mode 100644 index 0000000..70f5bce --- /dev/null +++ b/finance/analysis.py @@ -0,0 +1,275 @@ +import re +from decimal import Decimal + +from django.db.models import Count, Q, Sum + +from .models import Fact + +METRICS = { + "revenue": "实际收入", + "cost": "成本", + "gross": "毛利贡献", + "roi": "ROI", + "customers": "消费人数", + "basket": "客单价", + "margin": "毛利率", + "discount_rate": "产品扣率", + "self_revenue": "自营收入", + "upgrade_rate": "升单率", + "list_basket": "原价客单价", +} + + +def dimensions(ds): + return [ + {"key": c["key"], "name": c["name"], "description": c.get("description", "")} + for c in ds.mapping["columns"] + if c["role"] == "dimension" + ] + + +def default_preferences(ds): + dims = [d["key"] for d in dimensions(ds)] + preferred = [ + d + for d in ["department", "subject", "doctor_level", "doctor", "product", "product_subcategory"] + if d in dims + ] + return { + "entry": "overview", + "comparison": "yoy", + "hierarchy": preferred or dims[:5], + "metric": "revenue", + "threshold": 0.15, + "min_impact": 1000, + "min_samples": 5, + "show_heatmap": True, + "show_queue": True, + "show_copilot": True, + "alerts": True, + } + + +def month_index(s): + if not isinstance(s, str) or not re.fullmatch(r"\d{4}-\d{2}", s): + raise ValueError("期间格式必须为 YYYY-MM") + year, month = map(int, s.split("-")) + if not 1900 <= year <= 2200 or not 1 <= month <= 12: + raise ValueError("期间无效") + return year * 12 + month - 1 + + +def month_text(index): + year, month = divmod(index, 12) + return f"{year:04d}-{month + 1:02d}" + + +def context(ds, body): + if not isinstance(body, dict): + raise ValueError("查询格式错误") + end = body.get("end", ds.periods[-1]) + start = body.get("start", end) + a, b = month_index(start), month_index(end) + if b < a or b - a > 59: + raise ValueError("期间范围必须为 1 至 60 个月") + mode = body.get("comparison", "yoy") + if mode not in ("yoy", "mom", "custom"): + raise ValueError("比较口径无效") + length = b - a + 1 + base_end = ( + month_index(body.get("baseline_end", "")) + if mode == "custom" + else b - (12 if mode == "yoy" else length) + ) + current = [month_text(i) for i in range(a, b + 1)] + baseline = [month_text(i) for i in range(base_end - length + 1, base_end + 1)] + if set(current) & set(baseline): + raise ValueError("本期与基期不能重叠") + filters = body.get("filters", {}) + allowed = {d["key"] for d in dimensions(ds)} + if not isinstance(filters, dict) or any( + k not in allowed or not isinstance(v, str) or len(v) > 256 for k, v in filters.items() + ): + raise ValueError("筛选维度无效") + group = body.get("group", next(iter(dimensions(ds)))["key"]) + if not isinstance(group, str) or group not in allowed: + raise ValueError("分组维度不存在") + metric = body.get("metric", "revenue") + if not isinstance(metric, str) or metric not in METRICS: + raise ValueError("指标不存在") + missing = sorted(set(current + baseline) - set(ds.periods)) + return { + "start": start, + "end": end, + "comparison": mode, + "baseline_end": baseline[-1], + "current": current, + "baseline": baseline, + "missing": missing, + "filters": filters, + "group": group, + "metric": metric, + } + + +def scoped_facts(ds, member, filters=None): + qs = Fact.objects.filter(dataset=ds) + if member.departments: + qs = qs.filter(dimensions__department__in=member.departments) + for key, value in (filters or {}).items(): + qs = qs.filter(**{f"dimensions__{key}": value}) + return qs + + +def aggregations(products=None): + return { + "rows": Count("id"), + "revenue_sum": Sum("revenue"), + "cost_sum": Sum("cost"), + "gross_sum": Sum("gross"), + "cost_n": Count("cost"), + "gross_n": Count("gross"), + "customer_n": Count("customer"), + "customers": Count("customer", distinct=True), + "list_sum": Sum("list_amount"), + "list_n": Count("list_amount"), + "self_sum": Sum("revenue", filter=Q(dimensions__product__in=products or [])), + "upgraded": Count("customer", distinct=True, filter=Q(dimensions__product__in=products or [])), + } + + +def ratio(a, b): + return a / b if a is not None and b is not None and b > 0 else None + + +def metrics(agg, formula, products=None): + n = agg["rows"] + revenue = agg["revenue_sum"] or Decimal(0) + cost = (agg["cost_sum"] or Decimal(0)) if n and agg["cost_n"] == n else None + gross = (agg["gross_sum"] or Decimal(0)) if n and agg["gross_n"] == n else None + customers = agg["customers"] if n and agg["customer_n"] == n else None + profit = revenue - cost if cost is not None else None + if formula == "gross_less_cost_over_cost": + profit = gross - cost if gross is not None and cost is not None else None + return { + "rows": n, + "revenue": revenue, + "cost": cost, + "gross": gross, + "customers": customers, + "roi": ratio(profit, cost), + "basket": ratio(revenue, customers), + "margin": ratio(gross, revenue), + "discount_rate": ratio(revenue, agg["list_sum"]) if n and agg["list_n"] == n else None, + "self_revenue": (agg["self_sum"] or Decimal(0)) if products else None, + "upgrade_rate": ratio(Decimal(agg["upgraded"]), customers) if products else None, + "list_basket": ratio(agg["list_sum"], customers) if n and agg["list_n"] == n else None, + } + + +def comparison(current, baseline, comparable): + result = {} + for key in METRICS: + c, b = current.get(key), baseline.get(key) + delta = c - b if comparable and c is not None and b is not None else None + change = Decimal(delta) / abs(Decimal(b)) if delta is not None and b != 0 else None + result[key] = {"current": c, "baseline": b if comparable else None, "delta": delta, "change": change} + return result + + +def analyze(ds, member, body, preferences): + ctx = context(ds, body) + qs = scoped_facts(ds, member, ctx["filters"]) + formula = ds.mapping["roi_formula"] + products = ds.mapping.get("self_operated_products", []) + total_c = metrics( + qs.filter(period__in=ctx["current"]).aggregate(**aggregations(products)), formula, products + ) + total_b = metrics( + qs.filter(period__in=ctx["baseline"]).aggregate(**aggregations(products)), formula, products + ) + comparable = not ctx["missing"] + if set(ctx["current"]) - set(ds.periods): + total_c = {**total_c, **{key: None for key in METRICS}} + summary = comparison(total_c, total_b, comparable) + group = "dimensions__" + ctx["group"] + buckets = {} + for side in ("current", "baseline"): + for row in qs.filter(period__in=ctx[side]).values(group).annotate(**aggregations(products)): + buckets.setdefault(row[group], {})[side] = metrics(row, formula, products) + + def empty_side(total): + return { + "rows": 0, + **{ + k: Decimal(0) + if k in ("revenue", "cost", "gross", "customers", "self_revenue") and total.get(k) is not None + else None + for k in METRICS + }, + } + + rows = [] + metric = ctx["metric"] + parent_delta = summary[metric]["delta"] + threshold = Decimal(str(preferences["threshold"])) + impact_floor = Decimal(str(preferences["min_impact"])) + for label, sides in buckets.items(): + c, b = sides.get("current", empty_side(total_c)), sides.get("baseline", empty_side(total_b)) + if set(ctx["current"]) - set(ds.periods): + c = {**c, **{key: None for key in METRICS}} + values = comparison(c, b, comparable) + v = values[metric] + change, delta = v["change"], v["delta"] + impact = ( + abs(delta) + if metric in ("revenue", "gross", "cost", "self_revenue") and delta is not None + else abs(values["revenue"]["delta"] or 0) + ) + adverse = change is not None and (change >= threshold if metric == "cost" else change <= -threshold) + anomaly = bool( + adverse and impact >= impact_floor and min(c["rows"], b["rows"]) >= preferences["min_samples"] + ) + contribution = ( + delta / parent_delta + if delta is not None and parent_delta and metric in ("revenue", "cost", "gross", "self_revenue") + else None + ) + rows.append( + { + "label": label or "（未分类）", + "values": values, + "rows": c["rows"], + "baseline_rows": b["rows"], + "anomaly": anomaly, + "impact": impact, + "contribution": contribution, + "filters": {**ctx["filters"], ctx["group"]: label or "（未分类）"}, + } + ) + rows.sort(key=lambda r: (not r["anomaly"], -r["impact"], r["label"])) + observed = { + r["period"]: metrics(r, formula, products) + for r in qs.values("period").annotate(**aggregations(products)) + } + trend = [] + for index in range(month_index(ds.periods[0]), month_index(ds.periods[-1]) + 1): + period = month_text(index) + values = observed.get(period) + if values is None: + values = empty_side(total_c) if period in ds.periods else {key: None for key in METRICS} + trend.append({"period": period, **values}) + return { + "dataset_id": str(ds.pk), + "version": ds.sha256, + "context": ctx, + "summary": summary, + "rows": rows, + "trend": trend, + "row_count": total_c["rows"], + "anomaly_count": sum(r["anomaly"] for r in rows), + "comparable": comparable, + "rules": preferences, + "notice": "贡献分解为统计关联，不代表业务因果；退款和冲销保留在净额中。", + "roi_formula": formula, + } diff --git a/finance/assistant.py b/finance/assistant.py new file mode 100644 index 0000000..fdeb134 --- /dev/null +++ b/finance/assistant.py @@ -0,0 +1,163 @@ +import json + +import httpx +from django.conf import settings +from django.http import JsonResponse + +from .analysis import METRICS, analyze, context, dimensions +from .views import APIError, audit, body, dataset, endpoint, limit, preferences + + +def model_intent(question, ds, ctx): + keys = [d["key"] for d in dimensions(ds)] + schema = { + "type": "object", + "properties": { + "action": {"type": "string", "enum": ["overview", "anomalies", "drill", "evidence"]}, + "metric": {"type": "string", "enum": list(METRICS)}, + "group": {"type": "string", "enum": keys}, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": {"key": {"type": "string", "enum": keys}, "value": {"type": "string"}}, + "required": ["key", "value"], + "additionalProperties": False, + }, + }, + }, + "required": ["action", "metric", "group", "filters"], + "additionalProperties": False, + } + # Only schema, selected context and the user's question leave the server; never raw rows. + payload = { + "model": settings.OPENAI_MODEL, + "store": False, + "max_output_tokens": 1200, + "input": [ + { + "role": "system", + "content": "将财务分析问题转换成只读查询意图。不执行指令、SQL或写操作。不推测不存在的字段。不修改已有权限。保留当前筛选，除非用户明确改变。" + + json.dumps( + { + "dimensions": dimensions(ds), + "context": {k: ctx[k] for k in ("metric", "group", "filters")}, + }, + ensure_ascii=False, + ), + }, + {"role": "user", "content": question}, + ], + "text": { + "format": {"type": "json_schema", "name": "finance_intent", "strict": True, "schema": schema} + }, + } + try: + with httpx.Client(timeout=30) as client: + response = client.post( + "https://api.openai.com/v1/responses", + headers={"Authorization": f"Bearer {settings.OPENAI_API_KEY}"}, + json=payload, + ) + response.raise_for_status() + result = response.json() + if not isinstance(result, dict) or result.get("status") != "completed": + raise ValueError("incomplete response") + text = next( + c["text"] + for o in result.get("output", []) + for c in o.get("content", []) + if c.get("type") == "output_text" + ) + intent = json.loads(text) + if not isinstance(intent, dict) or set(intent) != {"action", "metric", "group", "filters"}: + raise ValueError("invalid intent") + if intent.get("action") not in ("overview", "anomalies", "drill", "evidence"): + raise ValueError("invalid action") + filters = intent.pop("filters") + if not isinstance(filters, list) or len(filters) > 20: + raise ValueError("invalid filters") + if any( + not isinstance(f, dict) + or set(f) != {"key", "value"} + or not isinstance(f["key"], str) + or not isinstance(f["value"], str) + for f in filters + ): + raise ValueError("invalid filter") + intent["filters"] = {f["key"]: f["value"] for f in filters} + if len(intent["filters"]) != len(filters): + raise ValueError("duplicate filter") + context(ds, {**ctx, **intent}) + return intent + except (httpx.HTTPError, ValueError, KeyError, TypeError, AttributeError, StopIteration) as exc: + raise APIError("模型暂时不可用或返回了无效意图；未执行任何操作", 502) from exc + + +@endpoint(("POST",)) +def assistant(request, dataset_id): + ds = dataset(request, dataset_id, True) + data = body(request) + question = data.get("question", "") + if not isinstance(question, str) or not 1 <= len(question.strip()) <= 2000: + raise APIError("问题需为 1 至 2,000 字") + limit(f"assistant:{request.user.pk}", 20, 60) + ctx = context(ds, data.get("context", {})) + pref, _ = preferences(request, ds) + use_model = data.get("use_model", False) + if type(use_model) is not bool: + raise APIError("AI 模式必须为布尔值") + if use_model: + if not (settings.AI_ENABLED and settings.OPENAI_API_KEY and settings.OPENAI_MODEL): + raise APIError("尚未配置外部 AI 服务", 503) + if data.get("consent") is not True: + raise APIError("请先同意发送问题与当前筛选上下文", 400) + intent = model_intent(question, ds, ctx) + else: + action = ( + "evidence" + if "明细" in question or "证据" in question + else "anomalies" + if "异常" in question + else "drill" + if "下钻" in question + else "overview" + ) + intent = {"action": action, "metric": ctx["metric"], "group": ctx["group"], "filters": ctx["filters"]} + for k, label in METRICS.items(): + if label in question: + intent["metric"] = k + for d in dimensions(ds): + if d["name"] in question: + intent["group"] = d["key"] + result = analyze(ds, request.member, {**ctx, **intent}, pref) + metric = intent["metric"] + value = result["summary"][metric] + if not result["comparable"]: + answer = "缺少期间 " + "、".join(result["context"]["missing"]) + "，不能给出比较或异常结论。" + elif value["current"] is None: + answer = f"当前数据缺少计算{METRICS[metric]}所需的完整字段。" + else: + change = value["change"] + change_text = f"{change:+.1%}" if change is not None else "不可计算（基期为零或数据缺失）" + answer = f"当前范围{METRICS[metric]}为 {value['current']:,.4f}，较基期变化 {change_text}。按当前规则识别出 {result['anomaly_count']} 个异常分组。" + candidates = [r for r in result["rows"] if r["anomaly"]][:3] + if candidates: + answer += " 优先核查：" + "、".join(r["label"] for r in candidates) + "。" + answer += " 这是数据贡献分解，不是因果结论。" + audit( + request, "assistant.query", ds.pk, {"provider": "openai" if use_model else "local", "intent": intent} + ) + return JsonResponse( + { + "answer": answer, + "intent": intent, + "context": result["context"], + "version": ds.sha256, + "provider": "openai" if use_model else "local", + "evidence": [ + {"label": r["label"], "filters": r["filters"], "values": r["values"][metric]} + for r in result["rows"][:5] + ], + } + ) diff --git a/finance/exporting.py b/finance/exporting.py new file mode 100644 index 0000000..28e5e18 --- /dev/null +++ b/finance/exporting.py @@ -0,0 +1,644 @@ +import hashlib +import hmac +import os +import secrets +import warnings +from datetime import timedelta +from decimal import Decimal, InvalidOperation +from pathlib import Path + +from django.conf import settings +from django.utils import timezone +from openpyxl import Workbook +from openpyxl.cell import WriteOnlyCell +from openpyxl.styles import Font, PatternFill +from openpyxl.worksheet.table import Table, TableColumn, TableStyleInfo + +from .analysis import METRICS, aggregations, context, dimensions, metrics, scoped_facts +from .formula_columns import ( + RESERVED_DATA_HEADERS, + ensure_available_metrics, + metrics_used, + to_excel, + validate_column, +) +from .models import FormulaExport, Membership + +FORMULA_VERSION = "3" +MAX_ROWS = 100_000 +MAX_GROUPS = 200 + + +def _excel_number(value): + if value is None: + return None + number = float(value) + if not Decimal(str(number)).is_finite(): + raise ValueError("金额超出 Excel 数值范围，公式工作簿已拒绝生成") + return number + + +def _text_cell(sheet, value): + cell = WriteOnlyCell(sheet, value="" if value is None else str(value)) + cell.data_type = "s" + return cell + + +def _styled_row(sheet, values, fill, font): + cells = [] + for value in values: + cell = value if hasattr(value, "data_type") else WriteOnlyCell(sheet, value=value) + cell.fill, cell.font = fill, font + cells.append(cell) + return cells + + +def _table(name, ref, headers, style): + table = Table(displayName=name, ref=ref) + table.tableColumns = [TableColumn(id=index, name=header) for index, header in enumerate(headers, 1)] + table.tableStyleInfo = TableStyleInfo(name=style, showRowStripes=True, showColumnStripes=False) + return table + + +def _add_table(sheet, table): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="In write-only mode you must add table columns manually") + sheet.add_table(table) + + +def _quote(value): + escaped = str(value).replace("~", "~~").replace("*", "~*").replace("?", "~?").replace('"', '""') + return '"' + escaped + '"' + + +def _text_criterion(value): + criterion = "=" + str(value) + escaped = criterion.replace("~", "~~").replace("*", "~*").replace("?", "~?") + if len(escaped) > 255: + raise ValueError("分组或筛选值超过 Excel criteria 的 255 字符上限，请缩短标签") + return _quote(criterion) + + +def _checked_formula(value): + if len(value) > 8192: + raise ValueError("生成公式超过 Excel 8,192 字符上限，请减少筛选条件或缩短标签") + return value + + +def _criteria(filters, group_key, label, periods): + terms = [] + for key, value in sorted(filters.items()): + terms.extend((f"Data[{key}]", _text_criterion(value))) + terms.extend((f"Data[{group_key}]", _text_criterion(label))) + terms.extend(("Data[Period]", _quote(">=" + periods[0]), "Data[Period]", _quote("<=" + periods[-1]))) + return ",".join(terms) + + +def _sum(column, criteria): + return f"SUMIFS(Data[{column}],{criteria})" + + +def _column_name(index): + name = "" + while index: + index, remainder = divmod(index - 1, 26) + name = chr(65 + remainder) + name + return name + + +def _metric_formula(key, label, periods, side_col, metric_rows, filters, group_key, products, roi_formula): + criteria = _criteria(filters, group_key, label, periods) + rows = f"COUNTIFS({criteria})" + revenue, cost = _sum("Revenue", criteria), _sum("Cost", criteria) + gross, list_sum = _sum("GrossCalc", criteria), _sum("ListAmount", criteria) + cost_n = f'COUNTIFS({criteria},Data[Cost],"<>")' + gross_n = f'COUNTIFS({criteria},Data[GrossCalc],"<>")' + list_n = f'COUNTIFS({criteria},Data[ListAmount],"<>")' + customer_period = "Current" if side_col == "C" else "Baseline" + customer_upgrade = f"{customer_period}Upgraded" + customers = ( + f"COUNTIFS(CustomerRoster[Group],{_text_criterion(label)},CustomerRoster[{customer_period}],1)" + ) + if key == "revenue": + return f"=IF({rows}=0,0,{revenue})" + if key == "cost": + return f"=IF({rows}=0,0,IF({cost_n}={rows},{cost},NA()))" + if key == "gross": + return f"=IF({rows}=0,0,IF({gross_n}={rows},{gross},NA()))" + if key == "customers": + return f"=IF({rows}=0,NA(),IF(COUNTIFS({criteria},Data[CustomerPresent],0)>0,NA(),{customers}))" + if key == "basket": + ref = f"{side_col}{metric_rows[(label, 'customers')]}" + return f"=IF(OR({rows}=0,ISNA({ref}),{ref}=0),NA(),{revenue}/{ref})" + if key == "margin": + return f"=IF(OR({rows}=0,{revenue}<=0,{gross_n}<>{rows}),NA(),{gross}/{revenue})" + if key == "discount_rate": + return f"=IF(OR({rows}=0,{list_n}<>{rows},{list_sum}<=0),NA(),{revenue}/{list_sum})" + if key == "self_revenue": + return ( + "=NA()" + if not products + else f"=IF({rows}=0,0,SUMIFS(Data[Revenue],{criteria},Data[SelfOperated],1))" + ) + if key == "upgrade_rate": + if not products: + return "=NA()" + ref = f"{side_col}{metric_rows[(label, 'customers')]}" + upgraded = ( + f"COUNTIFS(CustomerRoster[Group],{_text_criterion(label)},CustomerRoster[{customer_upgrade}],1)" + ) + return f"=IF(OR(ISNA({ref}),{ref}<=0),NA(),{upgraded}/{ref})" + if key == "list_basket": + ref = f"{side_col}{metric_rows[(label, 'customers')]}" + return f"=IF(OR({rows}=0,{list_n}<>{rows},ISNA({ref}),{ref}<=0),NA(),{list_sum}/{ref})" + numerator = revenue if roi_formula == "profit_over_cost" else gross + complete = "FALSE" if roi_formula == "profit_over_cost" else f"{gross_n}<>{rows}" + return f"=IF(OR({rows}=0,{cost_n}<>{rows},{cost}<=0,{complete}),NA(),({numerator}-{cost})/{cost})" + + +def build_formula_workbook(ds, member, export_context, output): + custom_columns = [validate_column(item) for item in export_context.get("custom_columns", [])] + for item in custom_columns: + ensure_available_metrics(ds, item) + if len(custom_columns) > 10: + raise ValueError("每次最多添加 10 个自定义指标列") + ctx = context(ds, export_context) + qs = ( + scoped_facts(ds, member, ctx["filters"]) + .filter(period__in=ctx["current"] + ctx["baseline"]) + .order_by("source_row") + ) + count = qs.count() + if count > MAX_ROWS: + raise ValueError(f"公式工作簿最多支持 {MAX_ROWS:,} 行，请缩小范围") + group_key = ctx["group"] + groups = list( + qs.order_by().values_list(f"dimensions__{group_key}", flat=True).distinct()[: MAX_GROUPS + 1] + ) + group_values = {} + excel_labels = {} + for value in groups: + label = str(value or "（未分类）") + if label in group_values and group_values[label] != value: + raise ValueError("分组存在与系统缺失值标签冲突的真实值，公式工作簿已拒绝生成") + folded = label.casefold() + if folded in excel_labels and excel_labels[folded] != label: + raise ValueError("分组名称仅大小写不同，Excel 无法可靠区分，公式工作簿已拒绝生成") + group_values[label] = value + excel_labels[folded] = label + if len(groups) > MAX_GROUPS: + raise ValueError(f"所选维度超过 {MAX_GROUPS} 个分组，请缩小范围") + labels = sorted(group_values) + mapping = ds.mapping + products, rates = mapping.get("self_operated_products", []), mapping.get("margin_rates", {}) + if len(products) > 100 or len(rates) > 2000: + raise ValueError("产品口径超过公式工作簿的支持上限") + dims = dimensions(ds) + dim_headers = [d["key"] for d in dims] + dim_names = {d["key"]: d["name"] for d in dims} + if RESERVED_DATA_HEADERS.intersection(key.casefold() for key in dim_headers): + raise ValueError("维度标识与公式工作簿保留列名冲突") + wb = Workbook(write_only=True) + summary = wb.create_sheet("分析汇总") + instructions = wb.create_sheet("使用说明") + parameters = wb.create_sheet("参数与口径") + data = wb.create_sheet("计算明细") + customers_sheet = wb.create_sheet("人数去重") + evidence = wb.create_sheet("原始证据") + reconcile = wb.create_sheet("对账检查") + custom_sheet = wb.create_sheet("自定义指标") if custom_columns else None + green, white = PatternFill("solid", fgColor="17664F"), Font(color="FFFFFF", bold=True) + summary.freeze_panes = "A2" + summary.sheet_view.showGridLines = False + for sheet in (instructions, parameters, reconcile): + sheet.freeze_panes = "A2" + sheet.sheet_view.showGridLines = False + data.freeze_panes = "A2" + customers_sheet.freeze_panes = "A2" + customers_sheet.sheet_view.showGridLines = False + if custom_sheet: + custom_sheet.freeze_panes = "B2" + custom_sheet.sheet_view.showGridLines = False + custom_sheet.auto_filter.ref = f"A1:{_column_name(1 + 4 * len(custom_columns))}{len(labels) + 1}" + summary.append(_styled_row(summary, ["公式分析工作簿", "本地可重算副本", "修改不回写系统"], green, white)) + summary.append(["数据版本", ds.sha256, "公式版本", FORMULA_VERSION]) + summary.append(["筛选范围", str(ctx["filters"]), "分组维度", _text_cell(summary, dim_names[group_key])]) + summary.append( + [ + "本期", + f"{ctx['current'][0]} 至 {ctx['current'][-1]}", + "基期", + f"{ctx['baseline'][0]} 至 {ctx['baseline'][-1]}", + ] + ) + summary.append( + _styled_row(summary, ["指标", "分组", "本期", "基期", "差额", "变化率", "状态"], green, white) + ) + all_metrics = {**METRICS, **{f"custom_{i}": item["name"] for i, item in enumerate(custom_columns)}} + metric_rows = {} + row = 6 + for label in labels: + for key in all_metrics: + metric_rows[(label, key)] = row + row += 1 + + headers = [ + "Period", + *dim_headers, + "CustomerToken", + "Revenue", + "Cost", + "GrossInput", + "MarginRate", + "GrossSource", + "ListAmount", + "SelfOperated", + "CustomerPresent", + "RowCount", + "GrossCalc", + ] + data.freeze_panes = "A2" + if count: + _add_table( + data, + _table( + "Data", + f"A1:{_column_name(len(headers))}{count + 1}", + headers, + "TableStyleMedium2", + ), + ) + data.append(headers) + token_secret = secrets.token_bytes(32) + customer_roster = {} + gross_index = next((col["index"] for col in mapping["columns"] if col["role"] == "gross"), None) + for row_number, fact in enumerate(qs.iterator(chunk_size=2000), start=2): + product = fact.dimensions.get("product", "") + direct = None + if ( + gross_index is not None + and gross_index < len(fact.raw) + and fact.raw[gross_index] not in (None, "") + ): + try: + direct = _excel_number(Decimal(str(fact.raw[gross_index]).replace(",", ""))) + except (InvalidOperation, ValueError): + pass + rate = _excel_number(Decimal(str(rates[product]))) if product in rates else None + customer = fact.customer or "" + token = hmac.new(token_secret, customer.encode(), hashlib.sha256).hexdigest()[:24] if customer else "" + if customer: + group_label = str(fact.dimensions.get(group_key) or "（未分类）") + record = customer_roster.setdefault( + (group_label, token), + {"current": False, "baseline": False, "current_upgraded": False, "baseline_upgraded": False}, + ) + if fact.period in ctx["current"]: + record["current"] = True + record["current_upgraded"] |= product in products + if fact.period in ctx["baseline"]: + record["baseline"] = True + record["baseline_upgraded"] |= product in products + dim_values = [fact.dimensions.get(d["key"], "（未分类）") or "（未分类）" for d in dims] + values = [ + fact.period, + *dim_values, + token, + _excel_number(fact.revenue), + _excel_number(fact.cost), + direct, + rate, + "direct" if direct is not None else ("lookup" if rate is not None else "missing"), + _excel_number(fact.list_amount), + int(product in products), + int(bool(customer)), + 1, + None, + ] + row = [ + _text_cell(data, value) if i <= len(dims) + 1 or i == len(dims) + 6 else value + for i, value in enumerate(values) + ] + row[-1] = WriteOnlyCell( + data, + value=( + f'=IF({_column_name(len(dims) + 7)}{row_number}="direct",' + f"{_column_name(len(dims) + 5)}{row_number}," + f'IF({_column_name(len(dims) + 7)}{row_number}="lookup",' + f"{_column_name(len(dims) + 3)}{row_number}*" + f'{_column_name(len(dims) + 6)}{row_number},""))' + ), + ) + data.append(row) + customer_headers = [ + "Group", + "CustomerToken", + "Current", + "Baseline", + "CurrentUpgraded", + "BaselineUpgraded", + ] + _add_table( + customers_sheet, + _table( + "CustomerRoster", + f"A1:F{len(customer_roster) + 1 if customer_roster else 2}", + customer_headers, + "TableStyleMedium4", + ), + ) + customers_sheet.append(customer_headers) + for (group_label, token), flags in customer_roster.items(): + customers_sheet.append( + [ + _text_cell(customers_sheet, group_label), + _text_cell(customers_sheet, token), + int(flags["current"]), + int(flags["baseline"]), + int(flags["current_upgraded"]), + int(flags["baseline_upgraded"]), + ] + ) + if not customer_roster: + customers_sheet.append(["", "", 0, 0, 0, 0]) + evidence.append( + [ + "来源工作表", + "来源行号", + *[_text_cell(evidence, d["name"]) for d in dims], + "期间", + "客户 Token", + "收入原值", + "成本原值", + "毛利原值", + "原价原值", + ] + ) + for source_row, period, values, customer, revenue, cost, gross, list_amount in qs.values_list( + "source_row", "period", "dimensions", "customer", "revenue", "cost", "gross", "list_amount" + ).iterator(chunk_size=2000): + token = ( + hmac.new(token_secret, (customer or "").encode(), hashlib.sha256).hexdigest()[:24] + if customer + else "" + ) + values = [ + mapping["sheet"], + source_row, + *[values.get(d["key"], "（未分类）") for d in dims], + period, + token, + str(revenue), + "" if cost is None else str(cost), + "" if gross is None else str(gross), + "" if list_amount is None else str(list_amount), + ] + evidence.append([_text_cell(evidence, value) for value in values]) + + parameters.append(_styled_row(parameters, ["参数", "值", "说明", "可编辑"], green, white)) + for row in [ + [ + "收入字段", + _text_cell( + parameters, + next((c["name"] for c in mapping["columns"] if c["role"] == "revenue"), "未命名"), + ), + "使用导入时确认的收入字段", + "否", + ], + ["ROI 口径", mapping["roi_formula"], "按导入映射保存的公式分支", "否"], + ["自营产品集合", _text_cell(parameters, ", ".join(products)), "升单率和自营收入口径", "否"], + ["产品毛利率", str(rates), "直接毛利缺失时的可编辑计算输入", "是"], + [ + "Excel 数值精度", + "IEEE-754 双精度", + "精确 Decimal 在证据页按文本保存；公式不保证 15 位以上有效数字", + "否", + ], + ["重算范围", f"最多 {MAX_ROWS:,} 行、{MAX_GROUPS} 组", "超限需缩小导出范围", "否"], + ["兼容性", "SUMIFS/COUNTIFS", "移除动态数组人数公式；仍需目标 Excel 桌面版本验收", "否"], + ]: + parameters.append(row) + for definition in custom_columns: + parameters.append( + [ + "自定义列", + _text_cell(parameters, definition["name"]), + _text_cell(parameters, definition["meaning"]), + "公式由服务端生成", + ] + ) + instructions.append(_styled_row(instructions, ["主题", "说明"], green, white)) + for row in [ + ["用途", "公式分析工作簿；本地修改不回写系统。"], + ["行粒度", "计算明细每行对应一个 Fact，不等同患者就诊数。"], + ["维度列", "列名为数据导入映射中的维度键；显示名称见本页分组维度。"], + ["毛利", "直接毛利优先，否则收入×产品毛利率；无法计算则不可用。毛利贡献不代表净利润。"], + ["ROI", "按选定口径；缺成本、成本非正或所需指标缺失时不可用。"], + ["人数", "客户 token 跨行去重；缺客户编码时不可用。"], + [ + "人数去重", + "人数去重页按分组保存随机 token 及本期/基期/升单标记；身份、分组和期间标记属于固定快照，不可修改。", + ], + ["变化率", "差额/ABS(基期)；基期为0或期间缺失时不可用。"], + ["精度", "原始 Decimal 以文本留存；Excel 双精度存在细微误差可能。"], + ["隐私", "客户标识替换为本文件随机 token，文件仍按敏感财务信息保护。"], + ]: + instructions.append(row) + for definition in custom_columns: + instructions.append( + [ + _text_cell(instructions, definition["name"]), + _text_cell( + instructions, + f"含义：{definition['meaning']}；范围：{definition['scope']}；缺失/零分母：{definition['missing']}。", + ), + ] + ) + if custom_sheet: + custom_sheet.append( + [ + "分组", + *[ + _text_cell(custom_sheet, f"{item['name']}（{suffix}）") + for item in custom_columns + for suffix in ("本期", "基期", "差额", "变化率") + ], + ] + ) + for label in labels: + values = [_text_cell(custom_sheet, label)] + for index, _definition in enumerate(custom_columns): + formula_row = metric_rows[(label, f"custom_{index}")] + values.extend( + [ + f"='分析汇总'!C{formula_row}", + f"='分析汇总'!D{formula_row}", + f"='分析汇总'!E{formula_row}", + f"='分析汇总'!F{formula_row}", + ] + ) + custom_sheet.append(values) + for label in labels: + for key in METRICS: + row = metric_rows[(label, key)] + formulas = [] + for side, periods in enumerate((ctx["current"], ctx["baseline"])): + if set(periods) & set(ctx["missing"]): + formulas.append("=NA()") + continue + col = "C" if side == 0 else "D" + formulas.append( + _checked_formula( + _metric_formula( + key, + label, + periods, + col, + metric_rows, + ctx["filters"], + group_key, + products, + mapping["roi_formula"], + ) + ) + ) + summary.append( + [ + METRICS[key], + _text_cell(summary, label), + formulas[0], + formulas[1], + f"=IFERROR(C{row}-D{row},NA())", + f"=IFERROR(IF(D{row}=0,NA(),E{row}/ABS(D{row})),NA())", + f'=IF(ISNA(C{row}),"本期不可用",IF(ISNA(D{row}),"基期不可用","可用"))', + ] + ) + for index, definition in enumerate(custom_columns): + key, row = f"custom_{index}", metric_rows[(label, f"custom_{index}")] + formulas = [] + for side in range(2): + periods = ctx["current"] if side == 0 else ctx["baseline"] + if set(periods) & set(ctx["missing"]): + formulas.append("=NA()") + continue + col = "C" if side == 0 else "D" + refs = {metric: f"{col}{metric_rows[(label, metric)]}" for metric in METRICS} + expr = to_excel(definition["expression"], refs) + used = metrics_used(definition["expression"]) + invalid = ",".join(f"ISNA({refs[metric]})" for metric in sorted(used)) + formula = ( + f"IF(OR({invalid}),NA(),IFERROR({expr},NA()))" if invalid else f"IFERROR({expr},NA())" + ) + formulas.append(_checked_formula(f"={formula}")) + summary.append( + [ + _text_cell(summary, definition["name"]), + _text_cell(summary, label), + formulas[0], + formulas[1], + f"=IFERROR(C{row}-D{row},NA())", + f"=IFERROR(IF(D{row}=0,NA(),E{row}/ABS(D{row})),NA())", + f'=IF(ISNA(C{row}),"本期不可用",IF(ISNA(D{row}),"基期不可用","可用"))', + ] + ) + + reconcile.append( + _styled_row( + reconcile, + ["指标", "分组", "期间侧", "服务器 Decimal 参考", "公式结果", "差异", "状态"], + green, + white, + ) + ) + reconcile_row = 2 + for label in labels: + for side_name, periods, col in (("本期", ctx["current"], "C"), ("基期", ctx["baseline"], "D")): + side_qs = qs.filter(period__in=periods, **{f"dimensions__{group_key}": group_values[label]}) + reference = metrics(side_qs.aggregate(**aggregations(products)), mapping["roi_formula"], products) + for key, name in METRICS.items(): + formula_row = metric_rows[(label, key)] + value = reference[key] + reconcile.append( + [ + name, + _text_cell(reconcile, label), + side_name, + "" if value is None else str(value), + f"='分析汇总'!{col}{formula_row}", + f'=IFERROR(E{reconcile_row}-D{reconcile_row},"")', + f'=IF(D{reconcile_row}="","无参考",IF(ISNUMBER(F{reconcile_row}),IF(ABS(F{reconcile_row})<0.01,"通过","待复核"),"待复核"))', + ] + ) + reconcile_row += 1 + for index, definition in enumerate(custom_columns): + formula_row = metric_rows[(label, f"custom_{index}")] + reconcile.append( + [ + _text_cell(reconcile, definition["name"]), + _text_cell(reconcile, label), + side_name, + "自定义公式无服务器参考", + f"='分析汇总'!{col}{formula_row}", + "", + "需业务验收", + ] + ) + reconcile_row += 1 + wb.calculation.fullCalcOnLoad = True + wb.calculation.forceFullCalc = True + wb.calculation.calcMode = "auto" + wb.save(output) + return {"rows": count, "groups": len(labels), "formula_version": FORMULA_VERSION} + + +def process_formula_export(export_id): + job = FormulaExport.objects.select_related("dataset", "user").get(pk=export_id) + temporary = None + target = None + try: + root = Path(settings.MEDIA_ROOT) / "formula-exports" + if root.is_symlink(): + raise ValueError("公式导出目录不能是符号链接") + root.mkdir(parents=True, exist_ok=True, mode=0o700) + root.chmod(0o700) + target = root / str(job.user_id) / f"{job.pk}.xlsx" + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if target.parent.is_symlink() or target.is_symlink(): + raise ValueError("公式导出路径不能是符号链接") + target.parent.chmod(0o700) + temporary = target.with_name(f"{job.pk}.{secrets.token_hex(8)}.tmp") + member = Membership.objects.get(user=job.user, organization=job.dataset.organization) + if set(member.departments) != set(job.departments): + raise PermissionError("科室授权已变更，请重新提交导出") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as file: + stats = build_formula_workbook(job.dataset, member, job.context, file) + temporary.replace(target) + job.path, job.status, job.error = str(target), "succeeded", "" + job.finished_at = timezone.now() + job.expires_at = timezone.now() + timedelta(hours=24) + job.save(update_fields=["path", "status", "error", "finished_at", "expires_at"]) + return stats + except Exception as exc: + for path in (temporary, target): + try: + if path is not None and not path.is_symlink(): + path.unlink(missing_ok=True) + except OSError: + pass + job.status, job.error, job.path = "failed", str(exc)[:500], "" + job.finished_at, job.expires_at = timezone.now(), None + job.save(update_fields=["status", "error", "path", "finished_at", "expires_at"]) + raise + + +def csv_safe(value): + text = str(value) if value is not None else "" + if not isinstance(value, str): + return text + return ( + "'" + text + if text.startswith(("\t", "\r", "\n")) or text.lstrip().startswith(("=", "+", "-", "@")) + else text + ) diff --git a/finance/formula_columns.py b/finance/formula_columns.py new file mode 100644 index 0000000..9981ceb --- /dev/null +++ b/finance/formula_columns.py @@ -0,0 +1,109 @@ +import math + +from .analysis import METRICS + +RESERVED_DATA_HEADERS = { + "period", + "customertoken", + "revenue", + "cost", + "grossinput", + "marginrate", + "grosssource", + "listamount", + "selfoperated", + "customerpresent", + "rowcount", + "grosscalc", +} + + +def available_metrics(ds): + roles = {column["role"] for column in ds.mapping["columns"]} + dimensions = {column.get("key") for column in ds.mapping["columns"] if column["role"] == "dimension"} + products = bool(ds.mapping.get("self_operated_products")) and "product" in dimensions + gross = "gross" in roles or (bool(ds.mapping.get("margin_rates")) and "product" in dimensions) + customer, list_amount, cost = "customer" in roles, "list_amount" in roles, "cost" in roles + available = {"revenue"} + if cost: + available.add("cost") + if gross: + available.update(("gross", "margin")) + if customer: + available.update(("customers", "basket")) + if list_amount: + available.add("discount_rate") + if customer and list_amount: + available.add("list_basket") + if cost and (ds.mapping.get("roi_formula") == "profit_over_cost" or gross): + available.add("roi") + if products: + available.add("self_revenue") + if customer: + available.add("upgrade_rate") + return {key: value for key, value in METRICS.items() if key in available} + + +OPERATORS = {"add": "+", "subtract": "-", "multiply": "*", "divide": "/"} + + +def validate_column(value): + if not isinstance(value, dict) or set(value) != {"name", "meaning", "scope", "missing", "expression"}: + raise ValueError("自定义列定义字段无效") + for key, limit in (("name", 80), ("meaning", 500), ("scope", 200), ("missing", 300)): + text = value[key] + if not isinstance(text, str) or not text.strip() or len(text) > limit: + raise ValueError(f"自定义列{key}说明不完整") + if value["missing"] != "结果不可用（#N/A）；零分母不可用": + raise ValueError("暂不支持自定义缺失值策略；请选择结果不可用（#N/A）及零分母不可用") + _validate_expression(value["expression"]) + return value + + +def _validate_expression(node, depth=0): + if depth > 8 or not isinstance(node, dict): + raise ValueError("自定义计算公式结构无效") + if set(node) == {"metric"}: + if not isinstance(node["metric"], str) or node["metric"] not in METRICS: + raise ValueError("公式引用了不支持的指标") + return + if set(node) == {"number"}: + if ( + type(node["number"]) not in (int, float) + or not math.isfinite(node["number"]) + or abs(node["number"]) > 1e12 + ): + raise ValueError("公式常数无效") + return + if ( + set(node) == {"operator", "left", "right"} + and isinstance(node["operator"], str) + and node["operator"] in OPERATORS + ): + _validate_expression(node["left"], depth + 1) + _validate_expression(node["right"], depth + 1) + return + raise ValueError("公式包含不支持的字段或运算") + + +def metrics_used(node): + if "metric" in node: + return {node["metric"]} + if "number" in node: + return set() + return metrics_used(node["left"]) | metrics_used(node["right"]) + + +def ensure_available_metrics(ds, column): + unavailable = metrics_used(column["expression"]) - available_metrics(ds).keys() + if unavailable: + names = "、".join(METRICS[key] for key in sorted(unavailable)) + raise ValueError(f"当前数据集未映射计算“{names}”所需字段，不能生成此自定义列") + + +def to_excel(node, references): + if "metric" in node: + return references[node["metric"]] + if "number" in node: + return str(node["number"]) + return f"({to_excel(node['left'], references)}{OPERATORS[node['operator']]}{to_excel(node['right'], references)})" diff --git a/finance/formula_views.py b/finance/formula_views.py new file mode 100644 index 0000000..898b741 --- /dev/null +++ b/finance/formula_views.py @@ -0,0 +1,328 @@ +import json +from pathlib import Path + +import httpx +from django.conf import settings +from django.http import FileResponse, JsonResponse +from django.utils import timezone + +from .analysis import METRICS, context, dimensions +from .formula_columns import available_metrics, ensure_available_metrics, validate_column +from .models import FormulaDefinition, FormulaExport +from .views import APIError, audit, body, dataset, endpoint, limit + + +def design_column(messages, ds, ctx): + if not (settings.AI_ENABLED and settings.OPENAI_API_KEY and settings.OPENAI_MODEL): + raise APIError("尚未配置外部 AI 服务", 503) + if not isinstance(messages, list) or not 1 <= len(messages) <= 12: + raise APIError("对话消息格式无效") + if any( + not isinstance(item, dict) + or set(item) != {"role", "content"} + or item["role"] not in ("user", "assistant") + or not isinstance(item["content"], str) + or len(item["content"]) > 1000 + for item in messages + ): + raise APIError("对话消息内容无效") + if ( + messages[0]["role"] != "user" + or messages[-1]["role"] != "user" + or any(left["role"] == right["role"] for left, right in zip(messages, messages[1:])) + ): + raise APIError("对话消息必须按用户描述、助手追问、用户补充的顺序提交") + + available = available_metrics(ds) + + def expression_schema(depth=0): + leaf = [ + { + "type": "object", + "properties": {"metric": {"type": "string", "enum": list(available)}}, + "required": ["metric"], + "additionalProperties": False, + }, + { + "type": "object", + "properties": { + "number": {"type": "number", "minimum": -1000000000000, "maximum": 1000000000000} + }, + "required": ["number"], + "additionalProperties": False, + }, + ] + if depth >= 8: + return {"anyOf": leaf} + return { + "anyOf": [ + *leaf, + { + "type": "object", + "properties": { + "operator": {"type": "string", "enum": ["add", "subtract", "multiply", "divide"]}, + "left": expression_schema(depth + 1), + "right": expression_schema(depth + 1), + }, + "required": ["operator", "left", "right"], + "additionalProperties": False, + }, + ] + } + + schema = { + "type": "object", + "properties": { + "complete": {"type": "boolean"}, + "questions": {"type": "array", "items": {"type": "string"}}, + "column": { + "anyOf": [ + {"type": "null"}, + { + "type": "object", + "properties": { + "name": {"type": "string"}, + "meaning": {"type": "string"}, + "scope": {"type": "string"}, + "missing": {"type": "string", "enum": ["结果不可用（#N/A）；零分母不可用"]}, + "expression": expression_schema(), + }, + "required": ["name", "meaning", "scope", "missing", "expression"], + "additionalProperties": False, + }, + ] + }, + }, + "required": ["complete", "questions", "column"], + "additionalProperties": False, + } + system = ( + "你是财务计算定义助手。只可用给定指标生成表达式树，不可输出公式文本、SQL或新字段。" + "available_metrics 是当前数据集可计算指标；其他指标依赖字段未映射，必须说明并追问，不可猜测或替代。" + "必须确认列名、含义、适用范围和缺失值/零分母处理。缺少任何信息时 complete=false、column=null，提出具体中文问题。" + "指标节点{metric:指标key}，常数{number:数值}，运算{operator:add|subtract|multiply|divide,left:...,right:...}。" + "本系统当前仅支持结果不可用（Excel #N/A）且零分母不可用，不支持按零替代或跳过缺失项。" + "用户补充信息后结合整段对话重新判断，只有公式含义无歧义且能表达时 complete=true。" + + json.dumps( + { + "metrics": METRICS, + "available_metrics": available, + "unavailable_metrics": {key: value for key, value in METRICS.items() if key not in available}, + "dimensions": dimensions(ds), + "context": {k: ctx[k] for k in ("current", "baseline", "filters", "group")}, + }, + ensure_ascii=False, + ) + ) + payload = { + "model": settings.OPENAI_MODEL, + "store": False, + "max_output_tokens": 1400, + "input": [{"role": "system", "content": system}, *messages], + "text": { + "format": { + "type": "json_schema", + "name": "formula_column_design", + "strict": True, + "schema": schema, + } + }, + } + try: + with httpx.Client(timeout=30) as client: + response = client.post( + "https://api.openai.com/v1/responses", + headers={"Authorization": f"Bearer {settings.OPENAI_API_KEY}"}, + json=payload, + ) + response.raise_for_status() + result = response.json() + if not isinstance(result, dict) or result.get("status") != "completed": + raise ValueError("incomplete response") + text = next(c["text"] for o in result["output"] for c in o["content"] if c["type"] == "output_text") + design = json.loads(text) + if ( + set(design) != {"complete", "questions", "column"} + or type(design["complete"]) is not bool + or not isinstance(design["questions"], list) + or len(design["questions"]) > 8 + ): + raise ValueError("invalid response") + if design["complete"]: + if design["column"] is None: + raise ValueError("complete response has no column") + design["column"] = validate_column(design["column"]) + try: + ensure_available_metrics(ds, design["column"]) + except ValueError as exc: + return {"complete": False, "questions": [str(exc)], "column": None} + if design["questions"]: + raise ValueError("complete response contains questions") + elif ( + design["column"] is not None + or not design["questions"] + or any(not isinstance(q, str) or not q.strip() or len(q) > 500 for q in design["questions"]) + ): + raise ValueError("incomplete response malformed") + return design + except (httpx.HTTPError, ValueError, KeyError, TypeError, StopIteration) as exc: + raise APIError("模型暂时不可用或返回了不安全的计算定义；未创建工作簿", 502) from exc + + +@endpoint(("POST",), roles=("admin", "analyst")) +def formula_design(request, dataset_id): + ds = dataset(request, dataset_id, True) + data = body(request) + if request.member.departments: + raise APIError("科室授权账号不可定义跨范围公式导出", 403) + ctx = context(ds, data.get("context", {})) + if data.get("consent") is not True: + raise APIError("请先同意发送列描述和筛选范围到外部 AI", 400) + limit(f"formula-design:{request.user.pk}", 20, 60) + result = design_column(data.get("messages"), ds, ctx) + audit(request, "formula_export.design", ds.pk, {"complete": result["complete"]}) + return JsonResponse(result) + + +@endpoint(("GET", "POST"), roles=("admin", "analyst")) +def formula_definitions(request, dataset_id): + ds = dataset(request, dataset_id, True) + if request.member.departments: + raise APIError("科室授权账号不可管理跨范围公式定义", 403) + if request.method == "GET": + items = FormulaDefinition.objects.filter(dataset=ds, user=request.user).order_by("normalized_name") + return JsonResponse( + { + "items": [ + { + "id": str(item.pk), + "definition": item.definition, + "updated_at": item.updated_at.isoformat(), + } + for item in items + ] + } + ) + + definition = validate_column(body(request).get("definition")) + try: + ensure_available_metrics(ds, definition) + except ValueError as exc: + raise APIError(str(exc)) from exc + normalized_name = definition["name"].strip().casefold() + existing = FormulaDefinition.objects.filter( + dataset=ds, user=request.user, normalized_name=normalized_name + ).first() + if not existing and FormulaDefinition.objects.filter(dataset=ds, user=request.user).count() >= 50: + raise APIError("每个数据集最多保存 50 个公式定义") + item, created = FormulaDefinition.objects.update_or_create( + dataset=ds, + user=request.user, + normalized_name=normalized_name, + defaults={"definition": definition}, + ) + audit(request, "formula_definition.save", item.pk, {"dataset": str(ds.pk), "name": definition["name"]}) + return JsonResponse( + { + "id": str(item.pk), + "definition": item.definition, + "updated_at": item.updated_at.isoformat(), + }, + status=201 if created else 200, + ) + + +@endpoint(("DELETE",), roles=("admin", "analyst")) +def formula_definition_delete(request, dataset_id, definition_id): + ds = dataset(request, dataset_id, True) + if request.member.departments: + raise APIError("科室授权账号不可管理跨范围公式定义", 403) + item = FormulaDefinition.objects.filter(pk=definition_id, dataset=ds, user=request.user).first() + if not item: + raise APIError("公式定义不存在", 404) + name = item.definition.get("name", "") + audit(request, "formula_definition.delete", item.pk, {"dataset": str(ds.pk), "name": name}) + item.delete() + return JsonResponse({"deleted": True}) + + +@endpoint(("POST",), roles=("admin", "analyst")) +def formula_export_create(request, dataset_id): + ds = dataset(request, dataset_id, True) + if request.member.departments: + raise APIError("科室授权账号不可创建公式导出", 403) + data = body(request) + ctx = context(ds, data) + definitions = data.get("custom_columns", []) + if not isinstance(definitions, list) or len(definitions) > 10: + raise APIError("自定义列最多 10 个") + for item in definitions: + validate_column(item) + try: + ensure_available_metrics(ds, item) + except ValueError as exc: + raise APIError(str(exc)) from exc + limit(f"formula-export:{request.user.pk}", 5, 3600) + job = FormulaExport.objects.create( + dataset=ds, + user=request.user, + context={**ctx, "custom_columns": definitions}, + departments=list(request.member.departments), + ) + audit( + request, + "formula_export.create", + job.pk, + {"dataset": str(ds.pk), "columns": [x["name"] for x in definitions]}, + ) + return JsonResponse({"id": str(job.pk), "status": job.status}, status=202) + + +def _owned_job(request, dataset_id, export_id): + job = FormulaExport.objects.filter( + pk=export_id, + dataset_id=dataset_id, + dataset__organization=request.member.organization, + user=request.user, + ).first() + if not job: + raise APIError("导出任务不存在", 404) + if set(request.member.departments) != set(job.departments): + raise APIError("授权范围已变化，不能访问此导出", 403) + return job + + +@endpoint(("GET",)) +def formula_export_status(request, dataset_id, export_id): + job = _owned_job(request, dataset_id, export_id) + return JsonResponse( + { + "id": str(job.pk), + "status": job.status, + "error": "生成失败，请联系管理员并提供任务编号" if job.status == "failed" else "", + "expires_at": job.expires_at.isoformat() if job.expires_at else None, + } + ) + + +@endpoint(("GET",)) +def formula_export_download(request, dataset_id, export_id): + job = _owned_job(request, dataset_id, export_id) + if job.status != "succeeded" or not job.path or not job.expires_at or job.expires_at <= timezone.now(): + raise APIError("文件尚未就绪或已过期", 410 if job.status == "succeeded" else 409) + export_root = Path(settings.MEDIA_ROOT) / "formula-exports" + candidate = Path(job.path) + expected = export_root / str(job.user_id) / f"{job.pk}.xlsx" + root = export_root.resolve() + path = candidate.resolve() + if ( + export_root.is_symlink() + or candidate.is_symlink() + or candidate.parent.is_symlink() + or path != expected.resolve() + or not path.is_relative_to(root) + or not path.is_file() + ): + raise APIError("导出文件不存在", 410) + audit(request, "formula_export.download", job.pk) + return FileResponse(path.open("rb"), as_attachment=True, filename="traceiq-formula-analysis.xlsx") diff --git a/finance/importing.py b/finance/importing.py new file mode 100644 index 0000000..63c47c7 --- /dev/null +++ b/finance/importing.py @@ -0,0 +1,378 @@ +import csv +import hashlib +import re +import zipfile +from datetime import date, datetime +from decimal import Decimal, InvalidOperation, getcontext +from itertools import islice + +from django.conf import settings +from django.db import transaction +from django.utils import timezone +from openpyxl import load_workbook + +from .formula_columns import RESERVED_DATA_HEADERS +from .models import Dataset, Fact + +getcontext().prec = 60 + +ROLES = { + "period", + "year", + "month", + "revenue", + "cost", + "gross", + "quantity", + "customer", + "list_amount", + "discount", + "dimension", + "ignore", +} +ALIASES = { + "period": ["期间", "日期", "period", "date"], + "year": ["年份", "年", "year"], + "month": ["月份", "月", "month"], + "revenue": ["实际收入", "收入", "revenue", "actual_revenue"], + "cost": ["成本", "cost"], + "gross": ["毛利", "毛利贡献", "gross_profit"], + "quantity": ["数量", "quantity"], + "customer": ["客户姓名", "客户编码", "customer_id"], + "list_amount": ["应收金额", "list_amount"], + "discount": ["折扣金额（去税）", "discount"], +} +DIMENSIONS = { + "科室（重分类）": "department", + "科室": "department", + "department": "department", + "学科分类": "subject", + "学科": "subject", + "医生等级": "doctor_level", + "医生": "doctor", + "医生ID": "doctor", + "产品一级分类": "product", + "产品二级分类": "product_subcategory", + "区域": "region", + "顾客来源": "source", +} + + +def safe_text(value): + if value is None: + return "" + if isinstance(value, (datetime, date)): + return value.isoformat() + return str(value)[:4000] + + +def validate_archive(file): + file.seek(0) + with zipfile.ZipFile(file) as archive: + infos = archive.infolist() + if len(infos) > 5000 or sum(i.file_size for i in infos) > 1024 * 1024 * 1024: + raise ValueError("工作簿解压体积超过安全限制") + if any(i.file_size / max(i.compress_size, 1) > 1000 for i in infos): + raise ValueError("工作簿压缩比异常") + if any("vbaProject" in i.filename or "externalLinks/" in i.filename for i in infos): + raise ValueError("不接受宏或外部链接工作簿") + file.seek(0) + + +def rows_for(path, sheet=None, formulas=False): + if str(path).lower().endswith(".csv"): + with open(path, encoding="utf-8-sig", newline="") as file: + yield from csv.reader(file) + else: + wb = load_workbook(path, read_only=True, data_only=not formulas, keep_links=False) + try: + yield from wb[sheet].iter_rows(values_only=True) + finally: + wb.close() + + +def profile_file(path): + if str(path).lower().endswith(".csv"): + sheets = [{"name": "CSV", "rows": None, "columns": None}] + else: + with open(path, "rb") as file: + validate_archive(file) + wb = load_workbook(path, read_only=True, data_only=True, keep_links=False) + sheets = [{"name": s.title, "rows": s.max_row, "columns": s.max_column} for s in wb] + wb.close() + previews = [] + for sheet in sheets: + preview_count = 600 if sheet["name"] == "产品毛利率" else 8 + rows = [list(map(safe_text, r)) for r in islice(rows_for(path, sheet["name"]), preview_count)] + if any(len(r) > 200 for r in rows): + raise ValueError("最多支持 200 列") + previews.append({**sheet, "preview": rows}) + return {"sheets": previews} + + +def suggest_mapping(profile, sheet=None, header=None): + s = next((s for s in profile["sheets"] if s["name"] == sheet), None) + s = s or next((s for s in profile["sheets"] if s["name"] == "明细"), profile["sheets"][0]) + rows = s["preview"] + if not rows: + raise ValueError("工作表为空") + if header is None: + + def score(row): + names = set(row) + return sum(any(a in names for a in aliases) for aliases in ALIASES.values()) + + header = max(range(min(5, len(rows))), key=lambda i: score(rows[i])) + 1 + if not isinstance(header, int) or not 1 <= header <= len(rows): + raise ValueError("表头行超出预览范围") + columns, used = [], set() + for i, name in enumerate(rows[header - 1]): + role = next( + (r for r, names in ALIASES.items() if name in names), + "ignore" if name in {"单价", "折扣金额", "实收金额"} else "dimension", + ) + if role != "dimension" and role in used: + role = "ignore" + # The reference workbook's L column is the agreed list-amount source. + if name == "应收金额" and i == 11: + for c in columns: + if c["role"] == "list_amount": + c["role"] = "ignore" + role = "list_amount" + used.add(role) + columns.append( + { + "index": i, + "name": name or f"列{i + 1}", + "description": rows[header - 2][i] if header > 1 and i < len(rows[header - 2]) else "", + "role": role, + "key": DIMENSIONS.get(name, f"field_{i}"), + "confidence": 1 if name in DIMENSIONS or role not in ("dimension", "ignore") else 0.5, + } + ) + rates = {} + for lookup in profile["sheets"]: + if lookup["name"] != "产品毛利率": + continue + name_index = rate_index = None + for row in lookup["preview"]: + if "产品分类" in row and "毛利率" in row: + name_index, rate_index = row.index("产品分类"), row.index("毛利率") + elif name_index is not None and len(row) > max(name_index, rate_index) and row[name_index]: + try: + rate = decimal(row[rate_index]) + if rate is not None and 0 <= rate <= 1: + rates[row[name_index]] = str(rate) + except ValueError: + pass + return { + "sheet": s["name"], + "header": header, + "columns": columns, + "margin_rates": rates, + "self_operated_products": [p for p in ("保健品", "检验检查", "医疗服务", "中药") if p in rates], + "roi_formula": "profit_over_cost", + "description": "", + } + + +def validate_mapping(mapping, profile): + if not isinstance(mapping, dict): + raise ValueError("映射格式错误") + sheets = {s["name"]: s for s in profile["sheets"]} + if ( + not isinstance(mapping.get("sheet"), str) + or mapping["sheet"] not in sheets + or type(mapping.get("header")) is not int + or not 1 <= mapping["header"] <= 8 + ): + raise ValueError("工作表或表头无效") + columns = mapping.get("columns", []) + if not isinstance(columns, list) or not 1 <= len(columns) <= 200: + raise ValueError("映射列数无效") + roles, keys, indexes = [], [], [] + for c in columns: + if ( + not isinstance(c, dict) + or not isinstance(c.get("role"), str) + or c.get("role") not in ROLES + or type(c.get("index")) is not int + or not 0 <= c["index"] < 200 + or not isinstance(c.get("name"), str) + or not 1 <= len(c["name"]) <= 256 + or not isinstance(c.get("description", ""), str) + or len(c.get("description", "")) > 4000 + ): + raise ValueError("字段映射无效") + indexes.append(c["index"]) + if c["role"] not in ("ignore", "dimension"): + roles.append(c["role"]) + if c["role"] == "dimension": + key = c.get("key", "") + if not isinstance(key, str) or not re.fullmatch(r"[a-z][a-z0-9_]{0,49}", key) or "__" in key: + raise ValueError("维度标识只允许小写字母、数字和单下划线") + keys.append(key) + if len(set(roles)) != len(roles) or len(set(keys)) != len(keys) or len(set(indexes)) != len(indexes): + raise ValueError("指标、维度标识或列索引重复") + if RESERVED_DATA_HEADERS.intersection(key.casefold() for key in keys): + raise ValueError("维度标识与公式工作簿保留列名冲突") + if "revenue" not in roles or not ("period" in roles or {"year", "month"}.issubset(roles)): + raise ValueError("必须映射实际收入以及期间（或年份和月份）") + if not keys: + raise ValueError("至少保留一个分析维度") + rates = mapping.get("margin_rates", {}) + if not isinstance(rates, dict) or len(rates) > 500: + raise ValueError("毛利率映射无效") + for key, value in rates.items(): + if not isinstance(key, str) or not 1 <= len(key) <= 256: + raise ValueError("产品分类名称无效") + rate = decimal(value) + if rate is None or not Decimal(0) <= rate <= Decimal(1): + raise ValueError("毛利率必须在 0 到 1 之间") + if rates and "product" not in keys: + raise ValueError("毛利率映射需要 product 维度") + if mapping.get("roi_formula") not in ("profit_over_cost", "gross_less_cost_over_cost"): + raise ValueError("请选择 ROI 口径") + products = mapping.get("self_operated_products", []) + if ( + not isinstance(products, list) + or len(products) > 500 + or any(not isinstance(p, str) or not 1 <= len(p) <= 256 for p in products) + ): + raise ValueError("自营产品范围无效") + if products and "product" not in keys: + raise ValueError("自营产品范围需要 product 维度") + return mapping + + +def decimal(value, required=False): + if value is None or str(value).strip() == "": + if required: + raise ValueError("必填金额为空") + return None + try: + number = Decimal(str(value).replace(",", "").strip()) + if not number.is_finite() or abs(number) >= Decimal("1e16"): + raise ValueError("金额超出范围") + if number != number.quantize(Decimal("1e-24")): + raise ValueError("金额精度超过二十四位小数") + return number + except InvalidOperation as exc: + raise ValueError("不是有效金额") from exc + + +def period_value(values): + if values.get("period"): + value = values["period"] + if isinstance(value, (date, datetime)): + return value.strftime("%Y-%m") + text = str(value).strip().replace("/", "-") + match = re.fullmatch(r"(\d{4})-(\d{1,2})(?:-\d{1,2}(?:T.*)?)?", text) + if not match: + raise ValueError("期间必须是 YYYY-MM 或日期") + year, month = map(int, match.groups()) + else: + year = int(str(values.get("year", "")).removesuffix("年")) + month = int(str(values.get("month", "")).removesuffix("月")) + if not 1900 <= year <= 2200 or not 1 <= month <= 12: + raise ValueError("期间超出有效范围") + return f"{year:04d}-{month:02d}" + + +def process_dataset(dataset_id): + ds = Dataset.objects.get(pk=dataset_id) + mapping = ds.mapping + quality = { + "negative_revenue": 0, + "missing_cost": 0, + "missing_gross": 0, + "missing_customer": 0, + "blank_rows": 0, + "g_l_mismatch": 0, + "i_n_mismatch": 0, + } + names = {c["index"]: c["name"] for c in mapping["columns"]} + reference_layout = names.get(6) == names.get(11) == "应收金额" and names.get(13) == "实际收入" + periods, batch, count = set(), [], 0 + try: + with transaction.atomic(): + if ds.facts.exists(): + raise ValueError("数据版本已包含明细，禁止覆盖") + for row_no, row in enumerate(rows_for(ds.file.path, mapping["sheet"]), 1): + if row_no <= mapping["header"]: + continue + if not any(v is not None and str(v).strip() for v in row): + quality["blank_rows"] += 1 + continue + count += 1 + if count > settings.MAX_IMPORT_ROWS: + raise ValueError("数据行超过配置上限") + values, dims = {}, {} + for col in mapping["columns"]: + v = row[col["index"]] if col["index"] < len(row) else None + if col["role"] == "dimension": + dims[col["key"]] = safe_text(v)[:256] or "（未分类）" + elif col["role"] != "ignore": + values[col["role"]] = v + try: + period = period_value(values) + revenue = decimal(values.get("revenue"), required=True) + nums = { + k: decimal(values.get(k)) + for k in ("cost", "gross", "quantity", "list_amount", "discount") + } + if nums["gross"] is None and dims.get("product") in mapping.get("margin_rates", {}): + nums["gross"] = ( + revenue * Decimal(str(mapping["margin_rates"][dims["product"]])) + ).quantize(Decimal("1e-24")) + customer = safe_text(values.get("customer"))[:256] or None + except (ValueError, TypeError, OverflowError) as exc: + raise ValueError(f"第 {row_no} 行：{exc}") from exc + quality["negative_revenue"] += revenue < 0 + if reference_layout and len(row) >= 14: + quality["g_l_mismatch"] += row[6] != row[11] + quality["i_n_mismatch"] += row[8] != row[13] + quality["missing_cost"] += nums["cost"] is None + quality["missing_gross"] += nums["gross"] is None + quality["missing_customer"] += customer is None + periods.add(period) + batch.append( + Fact( + dataset=ds, + period=period, + revenue=revenue, + dimensions=dims, + customer=customer, + source_row=row_no, + raw=list(map(safe_text, row)), + **nums, + ) + ) + if len(batch) >= 2000: + Fact.objects.bulk_create(batch, batch_size=500) + batch.clear() + if not count: + raise ValueError("工作表没有数据行") + Fact.objects.bulk_create(batch, batch_size=500) + ds.status, ds.row_count, ds.periods, ds.quality = "ready", count, sorted(periods), quality + ds.error, ds.finished_at = "", timezone.now() + ds.save() + except Exception as exc: + Dataset.objects.filter(pk=ds.pk).update( + status="failed", + error=str(exc)[:500] if isinstance(exc, ValueError) else "导入失败，请查看服务端日志", + finished_at=timezone.now(), + ) + if not isinstance(exc, ValueError): + import logging + + logging.getLogger(__name__).exception("Import failed for %s", ds.pk) + return Dataset.objects.get(pk=ds.pk) + + +def file_digest(file): + digest = hashlib.sha256() + for chunk in file.chunks(): + digest.update(chunk) + file.seek(0) + return digest.hexdigest() diff --git a/finance/management/__init__.py b/finance/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/finance/management/commands/__init__.py b/finance/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/finance/management/commands/bootstrap.py b/finance/management/commands/bootstrap.py new file mode 100644 index 0000000..b2e6fda --- /dev/null +++ b/finance/management/commands/bootstrap.py @@ -0,0 +1,29 @@ +import getpass +import os + +from django.contrib.auth.models import User +from django.contrib.auth.password_validation import validate_password +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from finance.models import Membership, Organization + + +class Command(BaseCommand): + help = "Create the initial organization administrator without a default password" + + def add_arguments(self, parser): + parser.add_argument("--username", required=True) + parser.add_argument("--organization", required=True) + + def handle(self, *args, **options): + username = options["username"] + if User.objects.filter(username=username).exists(): + raise CommandError("Username already exists; no changes made") + password = os.environ.get("BOOTSTRAP_PASSWORD") or getpass.getpass("Administrator password: ") + validate_password(password, User(username=username)) + with transaction.atomic(): + org = Organization.objects.create(name=options["organization"]) + user = User.objects.create_user(username=username, password=password) + Membership.objects.create(user=user, organization=org, role="admin") + self.stdout.write("Administrator created") diff --git a/finance/management/commands/cleanup_formula_exports.py b/finance/management/commands/cleanup_formula_exports.py new file mode 100644 index 0000000..692e1aa --- /dev/null +++ b/finance/management/commands/cleanup_formula_exports.py @@ -0,0 +1,65 @@ +import time +from datetime import timedelta +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.db import transaction +from django.utils import timezone + +from finance.models import FormulaExport + + +class Command(BaseCommand): + help = "Remove expired formula workbooks and stale temporary files" + + def add_arguments(self, parser): + parser.add_argument("--loop", action="store_true") + parser.add_argument("--interval", type=int, default=3600) + + def handle(self, *args, **options): + while True: + jobs, files = self.clean_expired() + self.stdout.write(f"Expired {jobs} jobs; removed {files} files") + if not options["loop"]: + return + time.sleep(max(60, options["interval"])) + + def clean_expired(self): + export_root = Path(settings.MEDIA_ROOT) / "formula-exports" + if export_root.is_symlink(): + raise ValueError("公式导出目录不能是符号链接") + root = export_root.resolve() + now = timezone.now() + jobs = files = 0 + + for job in FormulaExport.objects.filter(status="succeeded", expires_at__lte=now).iterator(): + path = Path(job.path) if job.path else None + resolved = path.resolve() if path and path.is_absolute() and not path.is_symlink() else None + in_root = bool(resolved and resolved.is_relative_to(root)) + try: + with transaction.atomic(): + if in_root and resolved.is_file(): + resolved.unlink() + files += 1 + job.status, job.path = "expired", "" + job.save(update_fields=["status", "path"]) + jobs += 1 + except OSError as exc: + self.stderr.write(f"Could not remove expired export {job.pk}: {exc}") + + cutoff = (now - timedelta(hours=24)).timestamp() + if root.is_dir() and not root.is_symlink(): + for user_dir in root.iterdir(): + if user_dir.is_symlink() or not user_dir.is_dir(): + continue + for path in user_dir.glob("*.tmp"): + try: + if path.is_symlink() or not path.is_file() or path.stat().st_mtime > cutoff: + continue + path.unlink() + files += 1 + except OSError as exc: + self.stderr.write(f"Could not remove stale temporary export {path.name}: {exc}") + + return jobs, files diff --git a/finance/management/commands/import_file.py b/finance/management/commands/import_file.py new file mode 100644 index 0000000..e08a7ec --- /dev/null +++ b/finance/management/commands/import_file.py @@ -0,0 +1,54 @@ +from pathlib import Path + +from django.contrib.auth.models import User +from django.core.files import File +from django.core.management.base import BaseCommand, CommandError + +from finance.importing import file_digest, profile_file, suggest_mapping, validate_mapping +from finance.models import Audit, Dataset + + +class Command(BaseCommand): + help = "Import a local source using reviewed field aliases and lookup rates" + + def add_arguments(self, parser): + parser.add_argument("path") + parser.add_argument("--username", required=True) + parser.add_argument("--confirm-mapping", action="store_true") + + def handle(self, *args, **options): + user = User.objects.get(username=options["username"]) + if user.membership.role != "admin" or user.membership.departments: + raise CommandError("Full administrator required") + path = Path(options["path"]) + profile = profile_file(path) + mapping = validate_mapping(suggest_mapping(profile), profile) + if not options["confirm_mapping"]: + self.stdout.write(str(mapping)) + self.stdout.write("Review mapping, then rerun with --confirm-mapping") + return + with path.open("rb") as source: + file = File(source, name=path.name) + digest = file_digest(file) + ds = Dataset.objects.filter(organization=user.membership.organization, sha256=digest).first() + if ds: + self.stdout.write(f"Existing version: {ds.pk} {ds.status}") + return + ds = Dataset.objects.create( + organization=user.membership.organization, + created_by=user, + name=path.name, + file=file, + sha256=digest, + mapping=mapping, + profile=profile, + status="queued", + ) + Audit.objects.create( + organization=user.membership.organization, + user=user, + action="dataset.cli_import", + resource=str(ds.pk), + details={"mapping": mapping}, + ) + self.stdout.write(f"{ds.pk}: queued; import_worker will process this version") diff --git a/finance/management/commands/import_worker.py b/finance/management/commands/import_worker.py new file mode 100644 index 0000000..469b76b --- /dev/null +++ b/finance/management/commands/import_worker.py @@ -0,0 +1,74 @@ +import time + +from django.core.management.base import BaseCommand +from django.db import connection, transaction +from django.utils import timezone + +from finance.exporting import process_formula_export +from finance.importing import process_dataset +from finance.models import Dataset, FormulaExport + + +class Command(BaseCommand): + help = "Process durable import and formula-export jobs" + + def add_arguments(self, parser): + parser.add_argument("--once", action="store_true") + + def handle(self, *args, **options): + postgres = connection.vendor == "postgresql" + if postgres: + with connection.cursor() as cursor: + cursor.execute("SELECT pg_try_advisory_lock(782146291)") + if not cursor.fetchone()[0]: + self.stdout.write("Another worker owns the database lock") + return + try: + self.run_jobs(options, postgres) + finally: + if postgres: + with connection.cursor() as cursor: + cursor.execute("SELECT pg_advisory_unlock(782146291)") + + def run_jobs(self, options, recover): + while True: + processed = self._next_import(recover) + if not processed: + processed = self._next_export(recover) + if options["once"]: + return + if not processed: + time.sleep(2) + + def _claim(self, model, states): + with transaction.atomic(): + qs = model.objects.filter(status__in=states).order_by("created_at") + if connection.features.has_select_for_update_skip_locked: + qs = qs.select_for_update(skip_locked=True) + item = qs.first() + if not item: + return None + updated = model.objects.filter(pk=item.pk, status__in=states).update(status="processing") + return item if updated else None + + def _next_import(self, recover): + states = ["queued", "processing"] if recover else ["queued"] + item = self._claim(Dataset, states) + if not item: + return False + Dataset.objects.filter(pk=item.pk).update(started_at=timezone.now()) + result = process_dataset(item.pk) + self.stdout.write(f"{result.pk}: {result.status}, {result.row_count} rows") + return True + + def _next_export(self, recover): + states = ["queued", "processing"] if recover else ["queued"] + item = self._claim(FormulaExport, states) + if not item: + return False + try: + stats = process_formula_export(item.pk) + self.stdout.write(f"{item.pk}: formula export ready, {stats['rows']} rows") + except Exception as exc: + self.stderr.write(f"Formula export {item.pk} failed: {exc}") + return True diff --git a/finance/middleware.py b/finance/middleware.py new file mode 100644 index 0000000..5b7e8fe --- /dev/null +++ b/finance/middleware.py @@ -0,0 +1,14 @@ +class SecurityHeaders: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + response["Content-Security-Policy"] = ( + "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" + ) + response["Referrer-Policy"] = "same-origin" + response["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()" + if request.path.startswith("/api/"): + response["Cache-Control"] = "no-store" + return response diff --git a/finance/migrations/0001_initial.py b/finance/migrations/0001_initial.py new file mode 100644 index 0000000..d2b2bcb --- /dev/null +++ b/finance/migrations/0001_initial.py @@ -0,0 +1,224 @@ +# Generated by Django 5.2.17 on 2026-09-22 16:35 + +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Organization", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=160)), + ], + ), + migrations.CreateModel( + name="Throttle", + fields=[ + ("key", models.CharField(max_length=64, primary_key=True, serialize=False)), + ("count", models.PositiveIntegerField(default=0)), + ("expires_at", models.DateTimeField()), + ], + ), + migrations.CreateModel( + name="Dataset", + fields=[ + ( + "id", + models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + ("name", models.CharField(max_length=200)), + ("file", models.FileField(upload_to="sources/%Y/%m")), + ("sha256", models.CharField(max_length=64)), + ("status", models.CharField(default="mapping", max_length=16)), + ("profile", models.JSONField(default=dict)), + ("mapping", models.JSONField(default=dict)), + ("quality", models.JSONField(default=dict)), + ("row_count", models.PositiveIntegerField(default=0)), + ("periods", models.JSONField(default=list)), + ("error", models.TextField(blank=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("started_at", models.DateTimeField(null=True)), + ("finished_at", models.DateTimeField(null=True)), + ( + "created_by", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL + ), + ), + ( + "organization", + models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="finance.organization"), + ), + ], + ), + migrations.CreateModel( + name="Membership", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "role", + models.CharField( + choices=[("admin", "管理员"), ("analyst", "分析师"), ("viewer", "查看者")], + max_length=16, + ), + ), + ("departments", models.JSONField(default=list)), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ( + "organization", + models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="finance.organization"), + ), + ], + ), + migrations.CreateModel( + name="Audit", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("action", models.CharField(max_length=64)), + ("resource", models.CharField(blank=True, max_length=100)), + ("details", models.JSONField(default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL + ), + ), + ( + "organization", + models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="finance.organization"), + ), + ], + ), + migrations.CreateModel( + name="Preference", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("value", models.JSONField(default=dict)), + ("revision", models.PositiveIntegerField(default=1)), + ( + "dataset", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="finance.dataset"), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ], + ), + migrations.CreateModel( + name="Fact", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("period", models.CharField(max_length=7)), + ("dimensions", models.JSONField(default=dict)), + ("customer", models.CharField(max_length=256, null=True)), + ("revenue", models.DecimalField(decimal_places=4, max_digits=22)), + ("cost", models.DecimalField(decimal_places=4, max_digits=22, null=True)), + ("gross", models.DecimalField(decimal_places=4, max_digits=22, null=True)), + ("quantity", models.DecimalField(decimal_places=4, max_digits=22, null=True)), + ("list_amount", models.DecimalField(decimal_places=4, max_digits=22, null=True)), + ("discount", models.DecimalField(decimal_places=4, max_digits=22, null=True)), + ("source_row", models.PositiveIntegerField()), + ("raw", models.JSONField(default=list)), + ( + "dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="facts", + to="finance.dataset", + ), + ), + ], + options={ + "indexes": [ + models.Index(fields=["dataset", "period"], name="finance_fac_dataset_69c9ce_idx") + ], + "constraints": [ + models.UniqueConstraint(fields=("dataset", "source_row"), name="unique_source_row") + ], + }, + ), + migrations.CreateModel( + name="Investigation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("key", models.CharField(max_length=64)), + ("context", models.JSONField(default=dict)), + ("status", models.CharField(default="open", max_length=16)), + ("note", models.TextField(blank=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "dataset", + models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="finance.dataset"), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint(fields=("dataset", "user", "key"), name="unique_investigation") + ], + }, + ), + migrations.AddConstraint( + model_name="dataset", + constraint=models.UniqueConstraint(fields=("organization", "sha256"), name="unique_org_source"), + ), + migrations.AddConstraint( + model_name="preference", + constraint=models.UniqueConstraint(fields=("dataset", "user"), name="unique_preference"), + ), + ] diff --git a/finance/migrations/0002_alter_fact_cost_alter_fact_discount_alter_fact_gross_and_more.py b/finance/migrations/0002_alter_fact_cost_alter_fact_discount_alter_fact_gross_and_more.py new file mode 100644 index 0000000..7dc75c3 --- /dev/null +++ b/finance/migrations/0002_alter_fact_cost_alter_fact_discount_alter_fact_gross_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 5.2.17 on 2026-09-22 16:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("finance", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="fact", + name="cost", + field=models.DecimalField(decimal_places=24, max_digits=44, null=True), + ), + migrations.AlterField( + model_name="fact", + name="discount", + field=models.DecimalField(decimal_places=24, max_digits=44, null=True), + ), + migrations.AlterField( + model_name="fact", + name="gross", + field=models.DecimalField(decimal_places=24, max_digits=44, null=True), + ), + migrations.AlterField( + model_name="fact", + name="list_amount", + field=models.DecimalField(decimal_places=24, max_digits=44, null=True), + ), + migrations.AlterField( + model_name="fact", + name="quantity", + field=models.DecimalField(decimal_places=24, max_digits=44, null=True), + ), + migrations.AlterField( + model_name="fact", + name="revenue", + field=models.DecimalField(decimal_places=24, max_digits=44), + ), + ] diff --git a/finance/migrations/0003_formulaexport.py b/finance/migrations/0003_formulaexport.py new file mode 100644 index 0000000..95329bb --- /dev/null +++ b/finance/migrations/0003_formulaexport.py @@ -0,0 +1,50 @@ +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("finance", "0002_alter_fact_cost_alter_fact_discount_alter_fact_gross_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="FormulaExport", + fields=[ + ( + "id", + models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + ("context", models.JSONField(default=dict)), + ("departments", models.JSONField(default=list)), + ("status", models.CharField(default="queued", max_length=16)), + ("path", models.CharField(blank=True, max_length=512)), + ("error", models.CharField(blank=True, max_length=500)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("finished_at", models.DateTimeField(null=True)), + ("expires_at", models.DateTimeField(null=True)), + ( + "dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="formula_exports", + to="finance.dataset", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ], + ), + migrations.AddIndex( + model_name="formulaexport", + index=models.Index(fields=["status", "created_at"], name="finance_for_status_fa1c07_idx"), + ), + ] diff --git a/finance/migrations/0004_formuladefinition.py b/finance/migrations/0004_formuladefinition.py new file mode 100644 index 0000000..45f44c8 --- /dev/null +++ b/finance/migrations/0004_formuladefinition.py @@ -0,0 +1,50 @@ +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("finance", "0003_formulaexport"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="FormulaDefinition", + fields=[ + ( + "id", + models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + ("normalized_name", models.CharField(max_length=320)), + ("definition", models.JSONField()), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="formula_definitions", + to="finance.dataset", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.AddConstraint( + model_name="formuladefinition", + constraint=models.UniqueConstraint( + fields=("dataset", "user", "normalized_name"), + name="unique_formula_definition_name", + ), + ), + ] diff --git a/finance/migrations/__init__.py b/finance/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/finance/models.py b/finance/models.py new file mode 100644 index 0000000..1de5920 --- /dev/null +++ b/finance/models.py @@ -0,0 +1,132 @@ +import uuid + +from django.conf import settings +from django.db import models + + +class Organization(models.Model): + name = models.CharField(max_length=160) + + +class Membership(models.Model): + user = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + organization = models.ForeignKey(Organization, on_delete=models.PROTECT) + role = models.CharField( + max_length=16, choices=[("admin", "管理员"), ("analyst", "分析师"), ("viewer", "查看者")] + ) + departments = models.JSONField(default=list) + + +class Dataset(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + organization = models.ForeignKey(Organization, on_delete=models.PROTECT) + created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT) + name = models.CharField(max_length=200) + file = models.FileField(upload_to="sources/%Y/%m") + sha256 = models.CharField(max_length=64) + status = models.CharField(max_length=16, default="mapping") + profile = models.JSONField(default=dict) + mapping = models.JSONField(default=dict) + quality = models.JSONField(default=dict) + row_count = models.PositiveIntegerField(default=0) + periods = models.JSONField(default=list) + error = models.TextField(blank=True) + created_at = models.DateTimeField(auto_now_add=True) + started_at = models.DateTimeField(null=True) + finished_at = models.DateTimeField(null=True) + + class Meta: + constraints = [models.UniqueConstraint(fields=["organization", "sha256"], name="unique_org_source")] + + +class Fact(models.Model): + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name="facts") + period = models.CharField(max_length=7) + dimensions = models.JSONField(default=dict) + customer = models.CharField(max_length=256, null=True) + revenue = models.DecimalField(max_digits=44, decimal_places=24) + cost = models.DecimalField(max_digits=44, decimal_places=24, null=True) + gross = models.DecimalField(max_digits=44, decimal_places=24, null=True) + quantity = models.DecimalField(max_digits=44, decimal_places=24, null=True) + list_amount = models.DecimalField(max_digits=44, decimal_places=24, null=True) + discount = models.DecimalField(max_digits=44, decimal_places=24, null=True) + source_row = models.PositiveIntegerField() + raw = models.JSONField(default=list) + + class Meta: + indexes = [models.Index(fields=["dataset", "period"])] + constraints = [models.UniqueConstraint(fields=["dataset", "source_row"], name="unique_source_row")] + + +class Preference(models.Model): + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + value = models.JSONField(default=dict) + revision = models.PositiveIntegerField(default=1) + + class Meta: + constraints = [models.UniqueConstraint(fields=["dataset", "user"], name="unique_preference")] + + +class Investigation(models.Model): + dataset = models.ForeignKey(Dataset, on_delete=models.PROTECT) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT) + key = models.CharField(max_length=64) + context = models.JSONField(default=dict) + status = models.CharField(max_length=16, default="open") + note = models.TextField(blank=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["dataset", "user", "key"], name="unique_investigation") + ] + + +class Audit(models.Model): + organization = models.ForeignKey(Organization, on_delete=models.PROTECT) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT) + action = models.CharField(max_length=64) + resource = models.CharField(max_length=100, blank=True) + details = models.JSONField(default=dict) + created_at = models.DateTimeField(auto_now_add=True) + + +class Throttle(models.Model): + key = models.CharField(max_length=64, primary_key=True) + count = models.PositiveIntegerField(default=0) + expires_at = models.DateTimeField() + + +class FormulaExport(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name="formula_exports") + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + context = models.JSONField(default=dict) + departments = models.JSONField(default=list) + status = models.CharField(max_length=16, default="queued") + path = models.CharField(max_length=512, blank=True) + error = models.CharField(max_length=500, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + finished_at = models.DateTimeField(null=True) + expires_at = models.DateTimeField(null=True) + + class Meta: + indexes = [models.Index(fields=["status", "created_at"], name="finance_for_status_fa1c07_idx")] + + +class FormulaDefinition(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name="formula_definitions") + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + normalized_name = models.CharField(max_length=320) + definition = models.JSONField() + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["dataset", "user", "normalized_name"], name="unique_formula_definition_name" + ) + ] diff --git a/finance/views.py b/finance/views.py new file mode 100644 index 0000000..4f78477 --- /dev/null +++ b/finance/views.py @@ -0,0 +1,610 @@ +import csv +import hashlib +import io +import ipaddress +import json +import math +import tempfile +import uuid +from datetime import timedelta +from functools import wraps +from pathlib import Path + +from django.conf import settings +from django.contrib.auth import authenticate, login, logout +from django.contrib.auth.models import User +from django.contrib.auth.password_validation import validate_password +from django.core.exceptions import ValidationError +from django.db import IntegrityError, connection, transaction +from django.http import FileResponse, HttpResponse, JsonResponse +from django.middleware.csrf import get_token +from django.utils import timezone +from openpyxl import Workbook +from openpyxl.cell import WriteOnlyCell + +from .analysis import METRICS, analyze, context, default_preferences, dimensions, scoped_facts +from .importing import file_digest, profile_file, suggest_mapping, validate_mapping +from .models import Audit, Dataset, Investigation, Membership, Preference, Throttle + + +class APIError(Exception): + def __init__(self, message, status=400): + self.message, self.status = message, status + + +def endpoint(methods=("GET",), roles=None, public=False): + def decorate(fn): + @wraps(fn) + def view(request, *args, **kwargs): + try: + if request.method not in methods: + raise APIError("请求方法不允许", 405) + if not public: + if not request.user.is_authenticated: + raise APIError("请先登录", 401) + try: + request.member = request.user.membership + except Membership.DoesNotExist: + raise APIError("账号尚未分配组织", 403) + if roles and request.member.role not in roles: + raise APIError("没有此操作的权限", 403) + return fn(request, *args, **kwargs) + except APIError as exc: + return JsonResponse({"error": exc.message}, status=exc.status) + except (ValueError, ValidationError, json.JSONDecodeError) as exc: + return JsonResponse({"error": str(exc)}, status=400) + + return view + + return decorate + + +def body(request): + if request.content_type != "application/json": + raise APIError("请求必须使用 application/json") + value = json.loads(request.body or b"{}") + if not isinstance(value, dict): + raise APIError("请求必须是 JSON 对象") + return value + + +def audit(request, action, resource="", details=None): + Audit.objects.create( + organization=request.member.organization, + user=request.user, + action=action, + resource=str(resource), + details=details or {}, + ) + + +def limit(key, count=10, seconds=900): + key = hashlib.sha256(key.encode()).hexdigest() + now = timezone.now() + with transaction.atomic(): + item, _ = Throttle.objects.get_or_create( + key=key, defaults={"expires_at": now + timedelta(seconds=seconds)} + ) + item = Throttle.objects.select_for_update().get(pk=key) + if item.expires_at <= now: + item.count, item.expires_at = 0, now + timedelta(seconds=seconds) + if item.count >= count: + raise APIError("请求过于频繁，请稍后重试", 429) + item.count += 1 + item.save() + + +def dataset(request, dataset_id, ready=False): + ds = Dataset.objects.filter(pk=dataset_id, organization=request.member.organization).first() + if not ds: + raise APIError("数据集不存在", 404) + if ready and ds.status != "ready": + raise APIError("数据集尚未导入完成", 409) + return ds + + +def preferences(request, ds): + pref = Preference.objects.filter(dataset=ds, user=request.user).first() + return {**default_preferences(ds), **(pref.value if pref else {})}, pref.revision if pref else 0 + + +def serialize_dataset(ds, detailed=False, restricted=False, raw_access=True): + out = { + "id": str(ds.pk), + "name": ds.name, + "status": ds.status, + "version": ds.sha256, + "row_count": None if restricted else ds.row_count, + "periods": ds.periods, + "created_at": ds.created_at.isoformat(), + "error": ds.error if not restricted else "", + "quality": ds.quality if not restricted else {}, + "dimensions": dimensions(ds) if ds.mapping else [], + } + if detailed: + out.update(mapping=ds.mapping) + if not restricted and raw_access: + out["profile"] = ds.profile + return out + + +@endpoint(public=True) +def session(request): + data = { + "csrf": get_token(request), + "user": None, + "ai_enabled": settings.AI_ENABLED and bool(settings.OPENAI_API_KEY and settings.OPENAI_MODEL), + } + if request.user.is_authenticated: + try: + m = request.user.membership + data["user"] = { + "id": request.user.pk, + "name": request.user.username, + "role": m.role, + "organization": m.organization.name, + "departments": m.departments, + } + except Membership.DoesNotExist: + pass + return JsonResponse(data) + + +@endpoint(("POST",), public=True) +def sign_in(request): + data = body(request) + username, password = data.get("username", ""), data.get("password", "") + if ( + not isinstance(username, str) + or not isinstance(password, str) + or len(username) > 150 + or len(password) > 1024 + ): + raise APIError("登录参数无效") + address = request.META.get("REMOTE_ADDR", "unknown") + if settings.TRUST_PROXY and request.META.get("HTTP_X_REAL_IP"): + try: + address = str(ipaddress.ip_address(request.META["HTTP_X_REAL_IP"])) + except ValueError: + raise APIError("代理来源地址无效") + limit("login-ip:" + address, 30) + limit("login-user:" + username.casefold(), 10) + user = authenticate(request, username=username, password=password) + if not user or not Membership.objects.filter(user=user).exists(): + raise APIError("用户名或密码错误", 401) + login(request, user) + request.member = user.membership + audit(request, "auth.login") + return JsonResponse({"ok": True, "csrf": get_token(request)}) + + +@endpoint(("POST",)) +def sign_out(request): + audit(request, "auth.logout") + logout(request) + return JsonResponse({"ok": True}) + + +@endpoint(("GET", "POST")) +def datasets(request): + if request.method == "GET": + return JsonResponse( + { + "items": [ + serialize_dataset(ds, restricted=bool(request.member.departments)) + for ds in Dataset.objects.filter(organization=request.member.organization).order_by( + "-created_at" + )[:100] + ] + } + ) + if request.member.role not in ("admin", "analyst") or request.member.departments: + raise APIError("仅有完整数据权限的管理员或分析师可导入", 403) + limit(f"upload:{request.user.pk}", 20, 3600) + file = request.FILES.get("file") + if not file or not 0 < file.size <= settings.MAX_UPLOAD_BYTES: + raise APIError("文件为空或超过上传大小限制") + ext = Path(file.name).suffix.lower() + if ext not in (".xlsx", ".csv"): + raise APIError("仅接受 .xlsx 或 UTF-8 CSV") + digest = file_digest(file) + existing = Dataset.objects.filter(organization=request.member.organization, sha256=digest).first() + if existing: + return JsonResponse( + serialize_dataset( + existing, + True, + raw_access=request.member.role == "admin" + or (existing.created_by_id == request.user.pk and existing.status in ("mapping", "failed")), + ), + status=200, + ) + ds = Dataset( + organization=request.member.organization, + created_by=request.user, + name=Path(file.name).name[:200], + sha256=digest, + ) + ds.file.save(str(uuid.uuid4()) + ext, file, save=False) + try: + ds.profile = profile_file(ds.file.path) + ds.mapping = suggest_mapping(ds.profile) + ds.save() + except IntegrityError: + ds.file.delete(save=False) + return JsonResponse( + serialize_dataset( + Dataset.objects.get(organization=request.member.organization, sha256=digest), + True, + raw_access=request.member.role == "admin", + ) + ) + except Exception as exc: + ds.file.delete(save=False) + raise APIError( + "无法读取文件，请检查格式或文件安全限制：" + + (str(exc)[:160] if isinstance(exc, (ValueError, UnicodeError)) else "文件损坏") + ) from exc + audit(request, "dataset.upload", ds.pk, {"sha256": digest}) + return JsonResponse(serialize_dataset(ds, True), status=201) + + +@endpoint() +def dataset_detail(request, dataset_id): + ds = dataset(request, dataset_id) + raw_access = request.member.role == "admin" or ( + ds.created_by_id == request.user.pk and ds.status in ("mapping", "failed") + ) + result = serialize_dataset(ds, True, bool(request.member.departments), raw_access) + if ds.status == "ready": + result["preferences"], result["revision"] = preferences(request, ds) + return JsonResponse(result) + + +@endpoint(("POST",), roles=("admin", "analyst")) +def mapping(request, dataset_id): + ds = dataset(request, dataset_id) + if request.member.departments: + raise APIError("没有修改导入映射的权限", 403) + data = body(request) + if data.get("suggest"): + return JsonResponse(suggest_mapping(ds.profile, data.get("sheet"), data.get("header"))) + value = validate_mapping(data, ds.profile) + updated = Dataset.objects.filter(pk=ds.pk, status__in=["mapping", "failed"]).update( + mapping=value, status="queued", error="", started_at=None, finished_at=None + ) + if not updated: + raise APIError("数据版本已锁定，不能重复导入或修改", 409) + audit(request, "dataset.confirm_mapping", ds.pk, {"mapping": value}) + return JsonResponse({"status": "queued"}, status=202) + + +@endpoint(("POST",)) +def query(request, dataset_id): + ds = dataset(request, dataset_id, True) + pref, _ = preferences(request, ds) + limit(f"analysis:{request.user.pk}", 120, 60) + return JsonResponse(analyze(ds, request.member, body(request), pref)) + + +@endpoint(("POST",)) +def evidence(request, dataset_id): + ds = dataset(request, dataset_id, True) + data = body(request) + ctx = context(ds, data) + offset, size = data.get("offset", 0), data.get("limit", 50) + if type(offset) is not int or type(size) is not int or not 0 <= offset <= 1000000 or not 1 <= size <= 200: + raise APIError("分页参数无效") + side = data.get("side", "current") + if side not in ("current", "baseline"): + raise APIError("期间选择无效") + qs = scoped_facts(ds, request.member, ctx["filters"]).filter(period__in=ctx[side]).order_by("source_row") + rows = list( + qs[offset : offset + size].values( + "source_row", "period", "dimensions", "revenue", "cost", "gross", "raw" + ) + ) + # Raw patient/customer identifiers require an explicit administrator role. + if request.member.role != "admin": + for row in rows: + row.pop("raw") + audit( + request, + "evidence.read", + ds.pk, + {"context": ctx, "side": side, "offset": offset, "limit": size}, + ) + return JsonResponse( + { + "rows": rows, + "total": qs.count(), + "context": ctx, + "version": ds.sha256, + "sheet": ds.mapping["sheet"], + } + ) + + +def csv_safe(value): + text = str(value) if value is not None else "" + if not isinstance(value, str): + return text + return ( + "'" + text + if text.startswith(("\t", "\r", "\n")) or text.lstrip().startswith(("=", "+", "-", "@")) + else text + ) + + +@endpoint(("POST",), roles=("admin", "analyst")) +def export(request, dataset_id): + ds = dataset(request, dataset_id, True) + data = body(request) + ctx = context(ds, data) + limit(f"export:{request.user.pk}", 10, 3600) + kind = data.get("format", "csv") + if kind not in ("csv", "xlsx"): + raise APIError("导出格式无效") + qs = ( + scoped_facts(ds, request.member, ctx["filters"]) + .filter(period__in=ctx["current"] + ctx["baseline"]) + .order_by("source_row") + ) + if qs.count() > 100000: + raise APIError("单次导出上限 100,000 行，请缩小期间或筛选范围") + output = tempfile.TemporaryFile(mode="w+b") + csv_stream = None + if kind == "csv": + output.write(b"\xef\xbb\xbf") + csv_stream = io.TextIOWrapper(output, encoding="utf-8", newline="", write_through=True) + csv_writer = csv.writer(csv_stream) if csv_stream else None + workbook = Workbook(write_only=True) if kind == "xlsx" else None + sheet = workbook.create_sheet("证据明细") if workbook else None + + def write_row(values): + if sheet is None: + csv_writer.writerow(csv_safe(value) for value in values) + else: + cells = [] + for value in values: + # Excel has only 15 significant digits; evidence amounts stay exact text. + cell = WriteOnlyCell(sheet, value=str(value) if value is not None else "") + cell.data_type = "s" + cells.append(cell) + sheet.append(cells) + + try: + write_row( + [ + "TraceIQ 数据版本", + ds.sha256, + ] + ) + write_row(["筛选上下文", json.dumps(ctx, ensure_ascii=False)]) + write_row(["ROI 口径", ds.mapping["roi_formula"]]) + write_row(["导出时间 UTC", timezone.now().isoformat()]) + dims = dimensions(ds) + write_row( + [ + "原始工作表", + "原始行号", + "期间", + *[d["name"] for d in dims], + "实际收入", + "成本", + "毛利贡献", + ] + ) + for row in qs.iterator(chunk_size=1000): + write_row( + [ + ds.mapping["sheet"], + row.source_row, + row.period, + *[row.dimensions.get(d["key"], "") for d in dims], + row.revenue, + row.cost, + row.gross, + ] + ) + if workbook: + workbook.save(output) + content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + else: + csv_stream.flush() + csv_stream.detach() + csv_stream = None + content_type = "text/csv; charset=utf-8" + output.seek(0) + audit(request, "evidence.export", ds.pk, {"context": ctx, "format": kind}) + return FileResponse( + output, + as_attachment=True, + filename=f"traceiq-evidence.{kind}", + content_type=content_type, + ) + except Exception: + if csv_stream is not None: + csv_stream.close() + else: + output.close() + raise + + +@endpoint(("GET", "PUT")) +def settings_view(request, dataset_id): + ds = dataset(request, dataset_id, True) + if request.method == "GET": + value, revision = preferences(request, ds) + return JsonResponse({"value": value, "revision": revision}) + data = body(request) + value = data.get("value") + defaults = default_preferences(ds) + if not isinstance(value, dict) or set(value) != set(defaults): + raise APIError("设置字段不完整") + allowed = {d["key"] for d in dimensions(ds)} + hierarchy = value["hierarchy"] + if ( + not isinstance(hierarchy, list) + or not hierarchy + or any(not isinstance(k, str) or k not in allowed for k in hierarchy) + or len(set(hierarchy)) != len(hierarchy) + ): + raise APIError("下钻层级无效或重复") + if ( + value["entry"] not in ("overview", "anomalies") + or value["comparison"] not in ("yoy", "mom") + or not isinstance(value["metric"], str) + or value["metric"] not in METRICS + ): + raise APIError("默认视图或指标无效") + for field, lo, hi in [("threshold", 0, 10), ("min_impact", 0, 1e15), ("min_samples", 1, 1000000)]: + v = value[field] + if type(v) not in (int, float) or not math.isfinite(v) or not lo <= v <= hi: + raise APIError("异常阈值无效") + if type(value["min_samples"]) is not int: + raise APIError("最小样本数必须为整数") + for field in ["show_heatmap", "show_queue", "show_copilot", "alerts"]: + if type(value[field]) is not bool: + raise APIError("显示设置必须为布尔值") + with transaction.atomic(): + pref = Preference.objects.select_for_update().filter(dataset=ds, user=request.user).first() + if data.get("revision") != (pref.revision if pref else 0): + raise APIError("设置已在其他窗口更新，请重新打开", 409) + if pref: + pref.value, pref.revision = value, pref.revision + 1 + pref.save() + else: + try: + pref = Preference.objects.create(dataset=ds, user=request.user, value=value) + except IntegrityError as exc: + raise APIError("设置发生并发更新，请重新打开", 409) from exc + audit(request, "settings.update", ds.pk, {"revision": pref.revision, "value": value}) + return JsonResponse({"value": pref.value, "revision": pref.revision}) + + +@endpoint(("GET", "POST")) +def investigations(request, dataset_id): + ds = dataset(request, dataset_id, True) + if request.method == "GET": + return JsonResponse( + { + "items": list( + Investigation.objects.filter(dataset=ds, user=request.user).values( + "id", "key", "context", "status", "note", "updated_at" + ) + ) + } + ) + if request.member.role == "viewer": + raise APIError("查看者不可修改调查状态", 403) + data = body(request) + ctx = context(ds, data.get("context", {})) + status, note = data.get("status"), data.get("note", "") + if status not in ("open", "reviewing", "resolved") or not isinstance(note, str) or len(note) > 2000: + raise APIError("调查状态或备注无效") + key = hashlib.sha256(json.dumps(ctx, sort_keys=True).encode()).hexdigest() + item, _ = Investigation.objects.update_or_create( + dataset=ds, user=request.user, key=key, defaults={"context": ctx, "status": status, "note": note} + ) + audit(request, "investigation.update", ds.pk, {"key": key, "status": status}) + return JsonResponse({"id": item.pk, "key": key, "status": status}) + + +@endpoint(("GET", "POST", "PATCH"), roles=("admin",)) +def users(request): + if request.member.departments: + raise APIError("限域管理员不能管理账号", 403) + if request.method == "GET": + return JsonResponse( + { + "items": [ + { + "id": m.user_id, + "username": m.user.username, + "role": m.role, + "active": m.user.is_active, + "departments": m.departments, + } + for m in Membership.objects.filter( + organization=request.member.organization + ).select_related("user") + ] + } + ) + data = body(request) + role, departments = data.get("role"), data.get("departments", []) + if ( + role not in ("admin", "analyst", "viewer") + or not isinstance(departments, list) + or any(not isinstance(d, str) or not d or len(d) > 256 for d in departments) + or len(departments) > 100 + ): + raise APIError("角色或科室范围无效") + with transaction.atomic(): + if request.method == "POST": + username, password = data.get("username"), data.get("password") + if ( + not isinstance(username, str) + or not 1 <= len(username) <= 150 + or not isinstance(password, str) + ): + raise APIError("账号格式无效") + validate_password(password, User(username=username)) + try: + user = User.objects.create_user(username=username, password=password) + except IntegrityError as exc: + raise APIError("用户名已存在", 409) from exc + Membership.objects.create( + user=user, organization=request.member.organization, role=role, departments=departments + ) + else: + if type(data.get("id")) is not int or data["id"] <= 0: + raise APIError("账号 ID 无效") + membership = ( + Membership.objects.filter(user_id=data.get("id"), organization=request.member.organization) + .select_related("user") + .first() + ) + if not membership: + raise APIError("账号不存在", 404) + if membership.user_id == request.user.pk: + raise APIError("不可在此修改自己的权限") + if type(data.get("active")) is not bool: + raise APIError("账号状态无效") + user = membership.user + user.is_active = data["active"] + user.save(update_fields=["is_active"]) + membership.role, membership.departments = role, departments + membership.save() + audit(request, "user.update", user.pk, {"role": role, "departments": departments}) + return JsonResponse({"ok": True}) + + +@endpoint(roles=("admin",)) +def audit_log(request): + if request.member.departments: + raise APIError("没有组织审计权限", 403) + return JsonResponse( + { + "items": list( + Audit.objects.filter(organization=request.member.organization) + .order_by("-id") + .values("id", "user__username", "action", "resource", "details", "created_at")[:200] + ) + } + ) + + +@endpoint(public=True) +def health(request): + with connection.cursor() as cursor: + cursor.execute("SELECT 1") + return JsonResponse({"status": "ok"}) + + +def index(request): + path = settings.BASE_DIR / "frontend/dist/index.html" + if not path.exists(): + return HttpResponse("Frontend not built. Run npm run build.", status=503) + return FileResponse(path.open("rb"), content_type="text/html") diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..9e1037d --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + TraceIQ | 财务分析工作台 + + +
+ + + diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..cd41397 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,1552 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { AgGridReact } from "ag-grid-react"; +import { + themeQuartz, + type ColDef, + type ICellRendererParams, +} from "ag-grid-community"; +import { + Activity, + Calculator, + ArrowDownRight, + ArrowUpRight, + Bell, + BookOpen, + ChevronRight, + Database, + Download, + Eye, + FileSpreadsheet, + LayoutDashboard, + LogOut, + Menu, + MessageSquare, + RefreshCw, + Search, + Settings, + ShieldCheck, + Upload, + Users, + X, +} from "lucide-react"; +import { api, download, setCsrf } from "./api"; +import { Chart } from "./Chart"; +import { + AdminDialog, + FormulaExportDialog, + Copilot, + EvidenceDialog, + ImportDialog, + Modal, + SettingsDialog, +} from "./dialogs"; +import { + format, + labels, + type Analysis, + type Context, + type Dataset, + type Metric, + type Preferences, + type Row, + type Session, +} from "./types"; + +const gridTheme = themeQuartz.withParams({ + accentColor: "#167366", + backgroundColor: "#ffffff", + foregroundColor: "#24332f", + borderColor: "#e0e7e4", + headerBackgroundColor: "#f3f6f5", + fontFamily: "system-ui, sans-serif", + fontSize: 13, + rowHeight: 49, + headerHeight: 43, + borderRadius: 0, +}); +type Screen = "overview" | "anomalies" | "investigate" | "data"; +export function App() { + const [session, setSession] = useState(), + [datasets, setDatasets] = useState([]), + [ds, setDs] = useState(), + [ctx, setCtx] = useState(), + [result, setResult] = useState(), + [screen, setScreen] = useState("overview"), + [path, setPath] = useState<{ label: string; ctx: Context }[]>([]), + [selected, setSelected] = useState(), + [error, setError] = useState(""), + [loading, setLoading] = useState(false), + [refresh, setRefresh] = useState(0), + [modal, setModal] = useState< + | "import" + | "settings" + | "dictionary" + | "users" + | "audit" + | null + | "formula" + >(null), + [evidenceCtx, setEvidenceCtx] = useState(), + [chat, setChat] = useState(false), + [source, setSource] = useState(false), + [search, setSearch] = useState(""), + [mobileNav, setMobileNav] = useState(false), + [note, setNote] = useState(""), + [status, setStatus] = useState("open"), + [saved, setSaved] = useState(false); + const upload = useRef(null), + requestVersion = useRef(0); + const report = useCallback( + (e: unknown) => setError((e as Error).message), + [], + ); + const loadSession = useCallback(async () => { + const s = await api("/session"); + setCsrf(s.csrf); + setSession(s); + }, []); + useEffect(() => { + loadSession().catch(report); + }, [loadSession, report]); + const loadDataset = useCallback( + async (id: string) => { + const ticket = ++requestVersion.current; + setLoading(true); + try { + const d = await api("/datasets/" + id); + if (ticket !== requestVersion.current) return; + setDs(d); + localStorage.setItem("traceiq.dataset", d.id); + setModal(null); + setPath([]); + setSelected(undefined); + setResult(undefined); + setEvidenceCtx(undefined); + setChat(false); + setError(""); + if (d.status === "ready") { + const p = d.preferences; + setCtx({ + start: d.periods.at(-1)!, + end: d.periods.at(-1)!, + comparison: p.comparison, + filters: {}, + group: p.hierarchy[0], + metric: p.metric, + }); + setScreen(p.entry); + } else { + setCtx(undefined); + setScreen("data"); + if (["mapping", "failed"].includes(d.status)) setModal("import"); + } + } catch (e) { + report(e); + } finally { + if (ticket === requestVersion.current) setLoading(false); + } + }, + [report], + ); + const loadList = useCallback(async () => { + const data = await api<{ items: Dataset[] }>("/datasets"); + setDatasets(data.items); + return data.items; + }, []); + useEffect(() => { + if (session?.user) { + loadList() + .then((items) => { + const last = localStorage.getItem("traceiq.dataset"); + if (items.length) + loadDataset( + items.find((item) => item.id === last)?.id || items[0].id, + ); + }) + .catch(report); + } + }, [session?.user?.id, loadList, loadDataset, report]); + useEffect(() => { + if (!ds || !["queued", "processing"].includes(ds.status)) return; + const timer = setInterval(() => { + api("/datasets/" + ds.id) + .then((d) => { + setDs(d); + if (d.status === "ready") { + loadList(); + loadDataset(d.id); + } + }) + .catch(report); + }, 2000); + return () => clearInterval(timer); + }, [ds?.id, ds?.status, loadList, loadDataset, report]); + useEffect(() => { + if (!ds || !ctx || ds.status !== "ready") return; + let active = true; + setLoading(true); + setError(""); + setResult(undefined); + setSelected(undefined); + api(`/datasets/${ds.id}/query`, "POST", ctx) + .then((data) => { + if (active) setResult(data); + }) + .catch((e) => { + if (active) report(e); + }) + .finally(() => { + if (active) setLoading(false); + }); + return () => { + active = false; + }; + }, [ds?.id, ctx, refresh, report]); + useEffect(() => { + setNote(""); + setStatus("open"); + setSaved(false); + if (!ds || !result || screen !== "investigate") return; + let active = true; + const current = result.context; + api<{ items: { context: Context; status: string; note: string }[] }>( + `/datasets/${ds.id}/investigations`, + ) + .then(({ items }) => { + if (!active) return; + const match = items.find( + ({ context: c }) => + c.start === current.start && + c.end === current.end && + c.baseline_end === current.baseline_end && + c.comparison === current.comparison && + c.metric === current.metric && + c.group === current.group && + Object.keys(c.filters).length === + Object.keys(current.filters).length && + Object.entries(c.filters).every( + ([k, v]) => current.filters[k] === v, + ), + ); + if (match) { + setNote(match.note); + setStatus(match.status); + setSaved(true); + } + }) + .catch(report); + return () => { + active = false; + }; + }, [ds?.id, result, screen, report]); + async function doUpload(file: File) { + setLoading(true); + setError(""); + try { + const form = new FormData(); + form.append("file", file); + const d = await api("/datasets", "POST", form); + setSource(true); + await loadList(); + await loadDataset(d.id); + } catch (e) { + report(e); + } finally { + setLoading(false); + if (upload.current) upload.current.value = ""; + } + } + function home(target: Screen = "overview") { + if (ds?.status === "ready" && ctx) { + setCtx({ ...ctx, filters: {}, group: ds.preferences.hierarchy[0] }); + setPath([]); + } + setScreen(target); + setMobileNav(false); + } + function drill(row: Row) { + if (!ctx || !ds) return; + const hierarchy = ds.preferences.hierarchy; + const index = hierarchy.indexOf(ctx.group); + const next = hierarchy[index + 1]; + if (!next) { + setEvidenceCtx({ ...ctx, filters: row.filters }); + return; + } + setPath([...path, { label: row.label, ctx: { ...ctx } }]); + setCtx({ ...ctx, filters: row.filters, group: next }); + setScreen("investigate"); + setSelected(undefined); + } + function crumb(index: number) { + if (!ctx || !ds) return; + if (index === 0) { + home(); + return; + } + const previous = path[index]; + if (previous) { + setCtx(previous.ctx); + setPath(path.slice(0, index)); + setScreen("investigate"); + } + } + function savePreferences(p: Preferences, revision: number) { + if (!ds || !ctx) return; + setDs({ ...ds, preferences: p, revision }); + setCtx({ + ...ctx, + metric: p.metric, + comparison: p.comparison, + filters: {}, + group: p.hierarchy[0], + }); + setScreen(p.entry); + setPath([]); + setModal(null); + } + const columns = useMemo[]>( + () => [ + { + field: "label", + headerName: + ds?.dimensions.find((d) => d.key === ctx?.group)?.name || "分组", + pinned: "left", + minWidth: 160, + flex: 1, + cellRenderer: (p: ICellRendererParams) => ( + + ), + }, + ...[ + ...new Set([ + "revenue", + "gross", + "cost", + "roi", + "customers", + ctx?.metric || "revenue", + ]), + ].map((metric) => ({ + colId: metric, + headerName: labels[metric], + minWidth: 130, + valueGetter: (p: { data?: Row }) => + p.data?.values[metric].current == null + ? null + : Number(p.data.values[metric].current), + valueFormatter: (p: { value: unknown }) => + format(p.value as number, metric), + cellClass: metric === ctx?.metric ? "metric-column" : "", + type: "numericColumn", + })), + { + colId: "change", + headerName: "重点指标变化", + minWidth: 145, + valueGetter: (p) => + p.data?.values[ctx?.metric || "revenue"].change == null + ? null + : Number(p.data.values[ctx?.metric || "revenue"].change), + valueFormatter: (p) => format(p.value, "change"), + cellClassRules: { + negative: (p) => p.value != null && p.value < 0, + positive: (p) => p.value != null && p.value > 0, + }, + }, + { + colId: "actions", + headerName: "调查", + width: 155, + pinned: "right", + sortable: false, + filter: false, + cellRenderer: (p: ICellRendererParams) => ( +
+ + +
+ ), + }, + ], + [ds, ctx, path], + ); + const rows = (result?.rows || []).filter( + (r) => (screen !== "anomalies" || r.anomaly) && r.label.includes(search), + ); + const metric = ctx?.metric || "revenue"; + const trend = useMemo( + () => ({ + color: ["#197568", "#5794c9"], + tooltip: { trigger: "axis" }, + legend: { bottom: 0 }, + grid: { top: 25, left: 65, right: 25, bottom: 65 }, + xAxis: { + type: "category", + data: result?.trend.map((r) => r.period), + axisLabel: { color: "#73827b" }, + }, + yAxis: { type: "value", splitLine: { lineStyle: { color: "#ecf0ee" } } }, + series: [ + { + name: "实际收入", + type: "line", + showSymbol: true, + symbolSize: 7, + connectNulls: false, + data: result?.trend.map((r) => + r.revenue == null ? null : Number(r.revenue), + ), + }, + { + name: "毛利贡献", + type: "line", + showSymbol: true, + symbolSize: 6, + connectNulls: false, + data: result?.trend.map((r) => + r.gross == null ? null : Number(r.gross), + ), + }, + ], + }), + [result], + ); + const contribution = useMemo( + () => ({ + tooltip: { trigger: "axis" }, + grid: { top: 15, left: 95, right: 30, bottom: 30 }, + xAxis: { type: "value", splitLine: { lineStyle: { color: "#ecf0ee" } } }, + yAxis: { + type: "category", + inverse: true, + data: result?.rows.slice(0, 8).map((r) => r.label), + }, + series: [ + { + type: "bar", + barMaxWidth: 20, + data: result?.rows.slice(0, 8).map((r) => ({ + value: + r.values[metric].delta == null + ? null + : Number(r.values[metric].delta), + itemStyle: { + color: Number(r.values[metric].delta) < 0 ? "#cd665a" : "#308c7c", + }, + })), + }, + ], + }), + [result, metric], + ); + const enterprise = !!import.meta.env.VITE_AG_GRID_LICENSE_KEY; + if (!session) + return ( +
+ +

{error || "正在连接工作台"}

+ {error && ( + + )} +
+ ); + if (!session.user) return ; + const user = session.user; + const canUpload = user.role !== "viewer" && !user.departments.length; + return ( +
+ +
+
+
+ + 工作台 + + + {screen === "data" + ? "数据管理" + : screen === "overview" + ? "经营总览" + : screen === "anomalies" + ? "异常追踪" + : "异常调查"} + +
+
+ + + 已连接 + + + {ds?.preferences?.alerts && ( + + )} +
+
+
+
+
+
FINANCIAL INTELLIGENCE
+

+ {screen === "overview" + ? "集团经营总览" + : screen === "anomalies" + ? "异常追踪" + : screen === "data" + ? "数据管理" + : `${path.at(-1)?.label || "当前范围"} · 异常调查`} +

+

+ {screen === "data" + ? "数据源、导入版本与质量状态" + : `${user.organization} / ${ds?.name || "尚未选择数据集"}`} +

+
+
+ {screen !== "overview" && screen !== "data" && ( + + )} + {canUpload && ( + + )} + { + if (e.target.files?.[0]) doUpload(e.target.files[0]); + }} + /> +
+
+ {error && ( +
+ {error} + +
+ )} + {!datasets.length && !loading && ( +
+ +

尚无财务数据

+ {canUpload && ( + + )} +
+ )} + {ds && ( +
+ + {ctx && ds.status === "ready" && ( + <> + + +
+ {[ + ["yoy", "同比"], + ["mom", "环比"], + ["custom", "自定义"], + ].map(([k, v]) => ( + + ))} +
+ {ctx.comparison === "custom" && ( + + )} + + )} + +
+ )} + {ds && source && ( +
+ +
+ {ds.name} + + {ds.row_count?.toLocaleString() ?? "受限"} 行 ·{" "} + {ds.periods.length} 个期间 · 版本 {ds.version.slice(0, 12)} + +
+ + +
+ )} + {screen === "data" && ( +
+
+

导入版本

+ {datasets.length} 个数据集 +
+
+ + + + + + + + + + + + + {datasets.map((d) => ( + + + + + + + + + ))} + +
文件状态明细行可用期间版本操作
+ + {d.name} + + + { + { + mapping: "待确认", + queued: "等待导入", + processing: "正在导入", + ready: "已就绪", + failed: "导入失败", + }[d.status] + } + + {d.row_count?.toLocaleString() ?? "受限"}{d.periods.join(" / ") || "等待解析"} + {d.version.slice(0, 12)} + + +
+
+ {ds && ( + <> +

数据质量

+
+ {Object.entries(ds.quality).map(([k, v]) => ( + + {( + { + negative_revenue: "负收入记录", + missing_cost: "成本缺失", + missing_gross: "毛利缺失", + missing_customer: "客户标识缺失", + blank_rows: "空白行", + } as Record + )[k] || k} + {v.toLocaleString()} + + ))} +
+ {ds.error &&

{ds.error}

} + {["mapping", "failed"].includes(ds.status) && canUpload && ( + + )} + {["queued", "processing"].includes(ds.status) && ( +

+ 导入任务 + {ds.status === "queued" ? "等待后台处理" : "处理中"} + ，页面会自动更新。 +

+ )} + + )} +
+ )} + {screen !== "data" && ds && ctx && ( + <> + {path.length > 0 && ( + + )} + {loading && ( +
+ + 正在计算分析结果 +
+ )} + {result && ( + <> + {!result.comparable && ( +
+ 缺少期间：{result.context.missing.join("、")} + 。比较与异常结论不可用。 +
+ )} +
+ {( + [ + "revenue", + "gross", + "cost", + "roi", + "customers", + ] as Metric[] + ).map((m) => ( + + ))} +
+ {screen === "overview" ? ( + <> +
+
+
+

收入与毛利走势

+ 单位：元 +
+ +
+
+
+

变化贡献

+ {labels[metric]} +
+ { + const row = result.rows.find( + (r) => r.label === name, + ); + if (row) drill(row); + }} + /> +
+
+ {ds.preferences.show_heatmap && ( +
+
+

经营指标矩阵

+ 本期与基期变化 +
+
+ + + + + {( + [ + "revenue", + "gross", + "cost", + "roi", + ] as Metric[] + ).map((m) => ( + + ))} + + + + + {result.rows.slice(0, 12).map((r) => ( + + + {( + [ + "revenue", + "gross", + "cost", + "roi", + ] as Metric[] + ).map((m) => { + const v = r.values[m].change; + return ( + + ); + })} + + + ))} + +
+ { + ds.dimensions.find( + (d) => d.key === ctx.group, + )?.name + } + {labels[m]}状态
{r.label} + + + + {r.anomaly ? "待调查" : "未触发"} + +
+
+
+ )} + + ) : screen === "investigate" ? ( +
+
+
+

{labels[metric]}变化分解

+ + {ctx.start} 至 {ctx.end} + +
+ +
+
+ INVESTIGATION +

+ {Object.values(ctx.filters).join(" / ") || "当前范围"} +

+

+ 本期 {format(result.summary[metric].current, metric)}{" "} + / 基期{" "} + {format(result.summary[metric].baseline, metric)} +

+

{result.notice}

+ +