/** * 端侧 ASR 资产获取:pnpm fetch:asr * * 下载 sherpa-onnx WASM 运行时 + SenseVoiceSmall int8 模型,产物落到 public/asr/ * (该目录在 .gitignore 里,模型不进 git;换机器/删了目录就重跑本脚本)。 * * 资产来源(2026-10-04 核实,大小为当天下载实测字节数): * - WASM 运行时:HuggingFace 官方 space k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc * https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/resolve/main/<文件名> * (glue 95,308 B / wasm 11,722,172 B / api 47,391 B) * 注意:普通 wasm/asr 运行时(GitHub release 的 zipformer/paraformer demo)只导出 * Online(流式)API,Offline 系列不可用;vad-asr 构建才导出 OfflineRecognizer * (wasm/vad-asr/CMakeLists.txt 的 exported_functions),而 SenseVoice 是离线模型, * 必须用这一份。文件逐个下载,无需解压 tarball。 * - 模型:GitHub release asr-models * https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 * (163,002,883 B;内含 model.int8.onnx 239,233,841 B + tokens.txt 315,894 B) * * 关键改造:官方 glue(sherpa-onnx-wasm-main-vad-asr.js)内嵌的 Emscripten 打包清单指向 * zipformer-ctc demo 的 silero_vad + zipformer 模型(367MB .data)。我们把清单改写为只含 * /model.int8.onnx + /tokens.txt 两个文件,并据此生成自己的 .data(两文件直接拼接, * Emscripten preload 格式 = 原始字节流 + JS 侧清单)。这样运行时一次请求拿到模型包, * 模型出现在 MEMFS 根路径,OfflineRecognizer 按绝对路径读取。 * * 产物(public/asr/): * sherpa-onnx-wasm-main-vad-asr.js glue(清单已改写) * sherpa-onnx-wasm-main-vad-asr.wasm 运行时 wasm(11,722,172 B,含 VAD + Offline API) * sherpa-onnx-wasm-main-vad-asr.data 模型预加载包(≈239,549,735 B) * sherpa-onnx-asr.js 官方 JS API 封装(OfflineRecognizer) * sense-voice-LICENSE.txt 模型许可(FunASR 模型许可,允许商用需署名) * manifest.json 资产清单 + 字节数(引擎下载进度与完整性自检用) */ import { spawnSync } from 'node:child_process'; import { createWriteStream } from 'node:fs'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const OUT_DIR = path.join(ROOT, 'public', 'asr'); const FORCE = process.argv.includes('--force'); const HF_SPACE = 'https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/resolve/main'; const GLUE_NAME = 'sherpa-onnx-wasm-main-vad-asr.js'; const WASM_NAME = 'sherpa-onnx-wasm-main-vad-asr.wasm'; const API_NAME = 'sherpa-onnx-asr.js'; const DATA_NAME = 'sherpa-onnx-wasm-main-vad-asr.data'; const RUNTIME_FILES: Array<{ name: string; bytes: number }> = [ { name: GLUE_NAME, bytes: 95_308 }, { name: WASM_NAME, bytes: 11_722_172 }, { name: API_NAME, bytes: 47_391 }, ]; const MODEL_URL = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2'; const MODEL_BYTES = 163_002_883; // 模型 tarball 内 const MODEL_ONNX_NAME = 'model.int8.onnx'; const TOKENS_NAME = 'tokens.txt'; const LICENSE_NAME = 'sense-voice-LICENSE.txt'; async function download(url: string, dest: string, expectedBytes: number): Promise { for (let attempt = 1; attempt <= 3; attempt++) { try { const res = await fetch(url); if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`); const reader = res.body.getReader(); const out = createWriteStream(dest); let received = 0; let logged = 0; for (;;) { const { done, value } = await reader.read(); if (done) break; out.write(value); received += value.byteLength; if (received - logged >= 20 * 1024 * 1024) { logged = received; console.log(` … ${(received / 1e6).toFixed(0)} MB`); } } await new Promise((resolve, reject) => out.end(resolve as () => void, reject)); if (received !== expectedBytes) { throw new Error(`大小不符:期望 ${expectedBytes},实际 ${received}(上游资产可能已更新,请核对后改脚本常量)`); } return; } catch (err) { console.log(` 下载失败(第 ${attempt} 次):${err instanceof Error ? err.message : err}`); if (attempt === 3) throw err; } } } async function extract(tarball: string, dir: string): Promise { const r = spawnSync('tar', ['-xf', tarball, '-C', dir], { stdio: 'inherit' }); if (r.status !== 0) throw new Error(`解压失败:${tarball}`); } /** 只含一个顶层目录时返回该目录路径,否则返回 dir 本身 */ async function onlySubdir(dir: string): Promise { const entries = await fs.readdir(dir, { withFileTypes: true }); const dirs = entries.filter((e) => e.isDirectory()); return dirs.length === 1 && entries.length === 1 ? path.join(dir, dirs[0].name) : dir; } /** 把 glue 里 loadPackage({...}) 的实参整体替换为新的打包清单(括号配平扫描,兼容 key 带不带引号两种生成风格) */ export function patchGlueMetadata(glue: string, files: Array<{ filename: string; start: number; end: number }>, total: number): string { const anchor = 'loadPackage({'; const start = glue.indexOf(anchor); if (start === -1) throw new Error('glue 中找不到 loadPackage 调用'); let depth = 0; for (let i = start + 'loadPackage('.length; i < glue.length; i++) { const ch = glue[i]; if (ch === '{') depth++; else if (ch === '}') { depth--; if (depth === 0) { const meta = `{files:[${files.map((f) => `{filename:"${f.filename}",start:${f.start},end:${f.end}}`).join(',')}],remote_package_size:${total}}`; return glue.slice(0, start) + 'loadPackage(' + meta + glue.slice(i + 1); } } } throw new Error('glue 中 loadPackage 清单括号不配平'); } async function main() { await fs.mkdir(OUT_DIR, { recursive: true }); const manifestPath = path.join(OUT_DIR, 'manifest.json'); if (!FORCE && (await fs.stat(manifestPath).catch(() => null)) !== null) { console.log('public/asr/ 已存在(--force 可强制重新下载)'); return; } const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'jev-asr-')); console.log(`临时目录:${tmp}`); console.log('\n1/2 下载 WASM 运行时(vad-asr 构建,含 Offline API)'); for (const f of RUNTIME_FILES) { console.log(` ${HF_SPACE}/${f.name}`); await download(`${HF_SPACE}/${f.name}`, path.join(tmp, f.name), f.bytes); } console.log(`\n2/2 下载 SenseVoiceSmall int8 模型(${(MODEL_BYTES / 1e6).toFixed(0)} MB)\n ${MODEL_URL}`); const modelTar = path.join(tmp, 'model.tar.bz2'); await download(MODEL_URL, modelTar, MODEL_BYTES); const modelDir = path.join(tmp, 'model'); await fs.mkdir(modelDir); await extract(modelTar, modelDir); const model = await onlySubdir(modelDir); const glue = await fs.readFile(path.join(tmp, GLUE_NAME), 'utf8'); const wasm = await fs.readFile(path.join(tmp, WASM_NAME)); const api = await fs.readFile(path.join(tmp, API_NAME)); const modelOnnx = await fs.readFile(path.join(model, MODEL_ONNX_NAME)); const tokens = await fs.readFile(path.join(model, TOKENS_NAME)); const licenseText = await fs.readFile(path.join(model, 'LICENSE')).catch(() => null); // 生成 .data:model.int8.onnx 拼 tokens.txt;清单路径用绝对路径(MEMFS 根) const data = Buffer.concat([modelOnnx, tokens]); const patched = patchGlueMetadata( glue, [ { filename: `/${MODEL_ONNX_NAME}`, start: 0, end: modelOnnx.byteLength }, { filename: `/${TOKENS_NAME}`, start: modelOnnx.byteLength, end: data.byteLength }, ], data.byteLength, ); if (patched === glue) throw new Error('清单改写失败'); const assets: Array<{ name: string; bytes: number }> = [ { name: GLUE_NAME, bytes: Buffer.byteLength(patched) }, { name: WASM_NAME, bytes: wasm.byteLength }, { name: DATA_NAME, bytes: data.byteLength }, { name: API_NAME, bytes: api.byteLength }, ]; await fs.writeFile(path.join(OUT_DIR, GLUE_NAME), patched); await fs.writeFile(path.join(OUT_DIR, WASM_NAME), wasm); await fs.writeFile(path.join(OUT_DIR, DATA_NAME), data); await fs.writeFile(path.join(OUT_DIR, API_NAME), api); if (licenseText) await fs.writeFile(path.join(OUT_DIR, LICENSE_NAME), licenseText); await fs.writeFile( path.join(OUT_DIR, 'manifest.json'), JSON.stringify( { generatedAt: new Date().toISOString(), runtime: { source: `${HF_SPACE} (k2-fsa 官方 space,vad-asr 构建)`, note: 'wasm/asr 构建不含 Offline API,SenseVoice 必须用 vad-asr 运行时' }, model: { url: MODEL_URL, name: 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17' }, assets, }, null, 2, ), ); // 自检:清单字节数与落盘文件一致 for (const a of assets) { const actual = (await fs.stat(path.join(OUT_DIR, a.name))).size; if (actual !== a.bytes) throw new Error(`自检失败:${a.name} 大小 ${actual} != 清单 ${a.bytes}`); } await fs.rm(tmp, { recursive: true, force: true }); console.log(`\n完成。产物在 public/asr/(共 ${(assets.reduce((n, a) => n + a.bytes, 0) / 1e6).toFixed(1)} MB):`); for (const a of assets) console.log(` ${(a.bytes / 1e6).toFixed(2).padStart(8)} MB ${a.name}`); console.log('\n首次在浏览器/桌面端使用时懒加载到 CacheStorage;见 src/asr/ondevice.ts 头注释。'); } main().catch((err) => { console.error(err); process.exit(1); });