This commit is contained in:
Jayway Liu
@@ -0,0 +1,378 @@
|
||||
import csv
|
||||
import hashlib
|
||||
import re
|
||||
import zipfile
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation, getcontext
|
||||
from itertools import islice
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from .formula_columns import RESERVED_DATA_HEADERS
|
||||
from .models import Dataset, Fact
|
||||
|
||||
getcontext().prec = 60
|
||||
|
||||
ROLES = {
|
||||
"period",
|
||||
"year",
|
||||
"month",
|
||||
"revenue",
|
||||
"cost",
|
||||
"gross",
|
||||
"quantity",
|
||||
"customer",
|
||||
"list_amount",
|
||||
"discount",
|
||||
"dimension",
|
||||
"ignore",
|
||||
}
|
||||
ALIASES = {
|
||||
"period": ["期间", "日期", "period", "date"],
|
||||
"year": ["年份", "年", "year"],
|
||||
"month": ["月份", "月", "month"],
|
||||
"revenue": ["实际收入", "收入", "revenue", "actual_revenue"],
|
||||
"cost": ["成本", "cost"],
|
||||
"gross": ["毛利", "毛利贡献", "gross_profit"],
|
||||
"quantity": ["数量", "quantity"],
|
||||
"customer": ["客户姓名", "客户编码", "customer_id"],
|
||||
"list_amount": ["应收金额", "list_amount"],
|
||||
"discount": ["折扣金额（去税）", "discount"],
|
||||
}
|
||||
DIMENSIONS = {
|
||||
"科室（重分类）": "department",
|
||||
"科室": "department",
|
||||
"department": "department",
|
||||
"学科分类": "subject",
|
||||
"学科": "subject",
|
||||
"医生等级": "doctor_level",
|
||||
"医生": "doctor",
|
||||
"医生ID": "doctor",
|
||||
"产品一级分类": "product",
|
||||
"产品二级分类": "product_subcategory",
|
||||
"区域": "region",
|
||||
"顾客来源": "source",
|
||||
}
|
||||
|
||||
|
||||
def safe_text(value):
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
return str(value)[:4000]
|
||||
|
||||
|
||||
def validate_archive(file):
|
||||
file.seek(0)
|
||||
with zipfile.ZipFile(file) as archive:
|
||||
infos = archive.infolist()
|
||||
if len(infos) > 5000 or sum(i.file_size for i in infos) > 1024 * 1024 * 1024:
|
||||
raise ValueError("工作簿解压体积超过安全限制")
|
||||
if any(i.file_size / max(i.compress_size, 1) > 1000 for i in infos):
|
||||
raise ValueError("工作簿压缩比异常")
|
||||
if any("vbaProject" in i.filename or "externalLinks/" in i.filename for i in infos):
|
||||
raise ValueError("不接受宏或外部链接工作簿")
|
||||
file.seek(0)
|
||||
|
||||
|
||||
def rows_for(path, sheet=None, formulas=False):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
with open(path, encoding="utf-8-sig", newline="") as file:
|
||||
yield from csv.reader(file)
|
||||
else:
|
||||
wb = load_workbook(path, read_only=True, data_only=not formulas, keep_links=False)
|
||||
try:
|
||||
yield from wb[sheet].iter_rows(values_only=True)
|
||||
finally:
|
||||
wb.close()
|
||||
|
||||
|
||||
def profile_file(path):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
sheets = [{"name": "CSV", "rows": None, "columns": None}]
|
||||
else:
|
||||
with open(path, "rb") as file:
|
||||
validate_archive(file)
|
||||
wb = load_workbook(path, read_only=True, data_only=True, keep_links=False)
|
||||
sheets = [{"name": s.title, "rows": s.max_row, "columns": s.max_column} for s in wb]
|
||||
wb.close()
|
||||
previews = []
|
||||
for sheet in sheets:
|
||||
preview_count = 600 if sheet["name"] == "产品毛利率" else 8
|
||||
rows = [list(map(safe_text, r)) for r in islice(rows_for(path, sheet["name"]), preview_count)]
|
||||
if any(len(r) > 200 for r in rows):
|
||||
raise ValueError("最多支持 200 列")
|
||||
previews.append({**sheet, "preview": rows})
|
||||
return {"sheets": previews}
|
||||
|
||||
|
||||
def suggest_mapping(profile, sheet=None, header=None):
|
||||
s = next((s for s in profile["sheets"] if s["name"] == sheet), None)
|
||||
s = s or next((s for s in profile["sheets"] if s["name"] == "明细"), profile["sheets"][0])
|
||||
rows = s["preview"]
|
||||
if not rows:
|
||||
raise ValueError("工作表为空")
|
||||
if header is None:
|
||||
|
||||
def score(row):
|
||||
names = set(row)
|
||||
return sum(any(a in names for a in aliases) for aliases in ALIASES.values())
|
||||
|
||||
header = max(range(min(5, len(rows))), key=lambda i: score(rows[i])) + 1
|
||||
if not isinstance(header, int) or not 1 <= header <= len(rows):
|
||||
raise ValueError("表头行超出预览范围")
|
||||
columns, used = [], set()
|
||||
for i, name in enumerate(rows[header - 1]):
|
||||
role = next(
|
||||
(r for r, names in ALIASES.items() if name in names),
|
||||
"ignore" if name in {"单价", "折扣金额", "实收金额"} else "dimension",
|
||||
)
|
||||
if role != "dimension" and role in used:
|
||||
role = "ignore"
|
||||
# The reference workbook's L column is the agreed list-amount source.
|
||||
if name == "应收金额" and i == 11:
|
||||
for c in columns:
|
||||
if c["role"] == "list_amount":
|
||||
c["role"] = "ignore"
|
||||
role = "list_amount"
|
||||
used.add(role)
|
||||
columns.append(
|
||||
{
|
||||
"index": i,
|
||||
"name": name or f"列{i + 1}",
|
||||
"description": rows[header - 2][i] if header > 1 and i < len(rows[header - 2]) else "",
|
||||
"role": role,
|
||||
"key": DIMENSIONS.get(name, f"field_{i}"),
|
||||
"confidence": 1 if name in DIMENSIONS or role not in ("dimension", "ignore") else 0.5,
|
||||
}
|
||||
)
|
||||
rates = {}
|
||||
for lookup in profile["sheets"]:
|
||||
if lookup["name"] != "产品毛利率":
|
||||
continue
|
||||
name_index = rate_index = None
|
||||
for row in lookup["preview"]:
|
||||
if "产品分类" in row and "毛利率" in row:
|
||||
name_index, rate_index = row.index("产品分类"), row.index("毛利率")
|
||||
elif name_index is not None and len(row) > max(name_index, rate_index) and row[name_index]:
|
||||
try:
|
||||
rate = decimal(row[rate_index])
|
||||
if rate is not None and 0 <= rate <= 1:
|
||||
rates[row[name_index]] = str(rate)
|
||||
except ValueError:
|
||||
pass
|
||||
return {
|
||||
"sheet": s["name"],
|
||||
"header": header,
|
||||
"columns": columns,
|
||||
"margin_rates": rates,
|
||||
"self_operated_products": [p for p in ("保健品", "检验检查", "医疗服务", "中药") if p in rates],
|
||||
"roi_formula": "profit_over_cost",
|
||||
"description": "",
|
||||
}
|
||||
|
||||
|
||||
def validate_mapping(mapping, profile):
|
||||
if not isinstance(mapping, dict):
|
||||
raise ValueError("映射格式错误")
|
||||
sheets = {s["name"]: s for s in profile["sheets"]}
|
||||
if (
|
||||
not isinstance(mapping.get("sheet"), str)
|
||||
or mapping["sheet"] not in sheets
|
||||
or type(mapping.get("header")) is not int
|
||||
or not 1 <= mapping["header"] <= 8
|
||||
):
|
||||
raise ValueError("工作表或表头无效")
|
||||
columns = mapping.get("columns", [])
|
||||
if not isinstance(columns, list) or not 1 <= len(columns) <= 200:
|
||||
raise ValueError("映射列数无效")
|
||||
roles, keys, indexes = [], [], []
|
||||
for c in columns:
|
||||
if (
|
||||
not isinstance(c, dict)
|
||||
or not isinstance(c.get("role"), str)
|
||||
or c.get("role") not in ROLES
|
||||
or type(c.get("index")) is not int
|
||||
or not 0 <= c["index"] < 200
|
||||
or not isinstance(c.get("name"), str)
|
||||
or not 1 <= len(c["name"]) <= 256
|
||||
or not isinstance(c.get("description", ""), str)
|
||||
or len(c.get("description", "")) > 4000
|
||||
):
|
||||
raise ValueError("字段映射无效")
|
||||
indexes.append(c["index"])
|
||||
if c["role"] not in ("ignore", "dimension"):
|
||||
roles.append(c["role"])
|
||||
if c["role"] == "dimension":
|
||||
key = c.get("key", "")
|
||||
if not isinstance(key, str) or not re.fullmatch(r"[a-z][a-z0-9_]{0,49}", key) or "__" in key:
|
||||
raise ValueError("维度标识只允许小写字母、数字和单下划线")
|
||||
keys.append(key)
|
||||
if len(set(roles)) != len(roles) or len(set(keys)) != len(keys) or len(set(indexes)) != len(indexes):
|
||||
raise ValueError("指标、维度标识或列索引重复")
|
||||
if RESERVED_DATA_HEADERS.intersection(key.casefold() for key in keys):
|
||||
raise ValueError("维度标识与公式工作簿保留列名冲突")
|
||||
if "revenue" not in roles or not ("period" in roles or {"year", "month"}.issubset(roles)):
|
||||
raise ValueError("必须映射实际收入以及期间（或年份和月份）")
|
||||
if not keys:
|
||||
raise ValueError("至少保留一个分析维度")
|
||||
rates = mapping.get("margin_rates", {})
|
||||
if not isinstance(rates, dict) or len(rates) > 500:
|
||||
raise ValueError("毛利率映射无效")
|
||||
for key, value in rates.items():
|
||||
if not isinstance(key, str) or not 1 <= len(key) <= 256:
|
||||
raise ValueError("产品分类名称无效")
|
||||
rate = decimal(value)
|
||||
if rate is None or not Decimal(0) <= rate <= Decimal(1):
|
||||
raise ValueError("毛利率必须在 0 到 1 之间")
|
||||
if rates and "product" not in keys:
|
||||
raise ValueError("毛利率映射需要 product 维度")
|
||||
if mapping.get("roi_formula") not in ("profit_over_cost", "gross_less_cost_over_cost"):
|
||||
raise ValueError("请选择 ROI 口径")
|
||||
products = mapping.get("self_operated_products", [])
|
||||
if (
|
||||
not isinstance(products, list)
|
||||
or len(products) > 500
|
||||
or any(not isinstance(p, str) or not 1 <= len(p) <= 256 for p in products)
|
||||
):
|
||||
raise ValueError("自营产品范围无效")
|
||||
if products and "product" not in keys:
|
||||
raise ValueError("自营产品范围需要 product 维度")
|
||||
return mapping
|
||||
|
||||
|
||||
def decimal(value, required=False):
|
||||
if value is None or str(value).strip() == "":
|
||||
if required:
|
||||
raise ValueError("必填金额为空")
|
||||
return None
|
||||
try:
|
||||
number = Decimal(str(value).replace(",", "").strip())
|
||||
if not number.is_finite() or abs(number) >= Decimal("1e16"):
|
||||
raise ValueError("金额超出范围")
|
||||
if number != number.quantize(Decimal("1e-24")):
|
||||
raise ValueError("金额精度超过二十四位小数")
|
||||
return number
|
||||
except InvalidOperation as exc:
|
||||
raise ValueError("不是有效金额") from exc
|
||||
|
||||
|
||||
def period_value(values):
|
||||
if values.get("period"):
|
||||
value = values["period"]
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.strftime("%Y-%m")
|
||||
text = str(value).strip().replace("/", "-")
|
||||
match = re.fullmatch(r"(\d{4})-(\d{1,2})(?:-\d{1,2}(?:T.*)?)?", text)
|
||||
if not match:
|
||||
raise ValueError("期间必须是 YYYY-MM 或日期")
|
||||
year, month = map(int, match.groups())
|
||||
else:
|
||||
year = int(str(values.get("year", "")).removesuffix("年"))
|
||||
month = int(str(values.get("month", "")).removesuffix("月"))
|
||||
if not 1900 <= year <= 2200 or not 1 <= month <= 12:
|
||||
raise ValueError("期间超出有效范围")
|
||||
return f"{year:04d}-{month:02d}"
|
||||
|
||||
|
||||
def process_dataset(dataset_id):
|
||||
ds = Dataset.objects.get(pk=dataset_id)
|
||||
mapping = ds.mapping
|
||||
quality = {
|
||||
"negative_revenue": 0,
|
||||
"missing_cost": 0,
|
||||
"missing_gross": 0,
|
||||
"missing_customer": 0,
|
||||
"blank_rows": 0,
|
||||
"g_l_mismatch": 0,
|
||||
"i_n_mismatch": 0,
|
||||
}
|
||||
names = {c["index"]: c["name"] for c in mapping["columns"]}
|
||||
reference_layout = names.get(6) == names.get(11) == "应收金额" and names.get(13) == "实际收入"
|
||||
periods, batch, count = set(), [], 0
|
||||
try:
|
||||
with transaction.atomic():
|
||||
if ds.facts.exists():
|
||||
raise ValueError("数据版本已包含明细，禁止覆盖")
|
||||
for row_no, row in enumerate(rows_for(ds.file.path, mapping["sheet"]), 1):
|
||||
if row_no <= mapping["header"]:
|
||||
continue
|
||||
if not any(v is not None and str(v).strip() for v in row):
|
||||
quality["blank_rows"] += 1
|
||||
continue
|
||||
count += 1
|
||||
if count > settings.MAX_IMPORT_ROWS:
|
||||
raise ValueError("数据行超过配置上限")
|
||||
values, dims = {}, {}
|
||||
for col in mapping["columns"]:
|
||||
v = row[col["index"]] if col["index"] < len(row) else None
|
||||
if col["role"] == "dimension":
|
||||
dims[col["key"]] = safe_text(v)[:256] or "（未分类）"
|
||||
elif col["role"] != "ignore":
|
||||
values[col["role"]] = v
|
||||
try:
|
||||
period = period_value(values)
|
||||
revenue = decimal(values.get("revenue"), required=True)
|
||||
nums = {
|
||||
k: decimal(values.get(k))
|
||||
for k in ("cost", "gross", "quantity", "list_amount", "discount")
|
||||
}
|
||||
if nums["gross"] is None and dims.get("product") in mapping.get("margin_rates", {}):
|
||||
nums["gross"] = (
|
||||
revenue * Decimal(str(mapping["margin_rates"][dims["product"]]))
|
||||
).quantize(Decimal("1e-24"))
|
||||
customer = safe_text(values.get("customer"))[:256] or None
|
||||
except (ValueError, TypeError, OverflowError) as exc:
|
||||
raise ValueError(f"第 {row_no} 行：{exc}") from exc
|
||||
quality["negative_revenue"] += revenue < 0
|
||||
if reference_layout and len(row) >= 14:
|
||||
quality["g_l_mismatch"] += row[6] != row[11]
|
||||
quality["i_n_mismatch"] += row[8] != row[13]
|
||||
quality["missing_cost"] += nums["cost"] is None
|
||||
quality["missing_gross"] += nums["gross"] is None
|
||||
quality["missing_customer"] += customer is None
|
||||
periods.add(period)
|
||||
batch.append(
|
||||
Fact(
|
||||
dataset=ds,
|
||||
period=period,
|
||||
revenue=revenue,
|
||||
dimensions=dims,
|
||||
customer=customer,
|
||||
source_row=row_no,
|
||||
raw=list(map(safe_text, row)),
|
||||
**nums,
|
||||
)
|
||||
)
|
||||
if len(batch) >= 2000:
|
||||
Fact.objects.bulk_create(batch, batch_size=500)
|
||||
batch.clear()
|
||||
if not count:
|
||||
raise ValueError("工作表没有数据行")
|
||||
Fact.objects.bulk_create(batch, batch_size=500)
|
||||
ds.status, ds.row_count, ds.periods, ds.quality = "ready", count, sorted(periods), quality
|
||||
ds.error, ds.finished_at = "", timezone.now()
|
||||
ds.save()
|
||||
except Exception as exc:
|
||||
Dataset.objects.filter(pk=ds.pk).update(
|
||||
status="failed",
|
||||
error=str(exc)[:500] if isinstance(exc, ValueError) else "导入失败，请查看服务端日志",
|
||||
finished_at=timezone.now(),
|
||||
)
|
||||
if not isinstance(exc, ValueError):
|
||||
import logging
|
||||
|
||||
logging.getLogger(__name__).exception("Import failed for %s", ds.pk)
|
||||
return Dataset.objects.get(pk=ds.pk)
|
||||
|
||||
|
||||
def file_digest(file):
|
||||
digest = hashlib.sha256()
|
||||
for chunk in file.chunks():
|
||||
digest.update(chunk)
|
||||
file.seek(0)
|
||||
return digest.hexdigest()
|
||||
Reference in New Issue
Block a user