diff --git a/.env.example b/.env.example index bf9e24e..b73a74d 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,6 @@ VITE_JEV_PROXY_URL= # ASR（语音转写）代理地址。Tauri/不支持 SpeechRecognition 的环境走录音上传通路； # 服务端未配置真实 ASR provider 时返回占位结果，输入框兜底为打字。 VITE_ASR_PROXY_URL=http://localhost:8787 + +# 端侧小模型转写（sherpa-onnx WASM + SenseVoiceSmall）默认开；'0' 关闭 +# VITE_ONDEVICE_ASR=0 diff --git a/.gitignore b/.gitignore index 6e9f2c0..1c84b38 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,8 @@ dist-ssr # 生成物：评审包（pnpm script:review 重新生成） docs/script/review-bundle.md +# 端侧 ASR 资产（pnpm fetch:asr 重新下载，模型体积大不进 git） +public/asr/ + # 工具产物 .codegraph/ diff --git a/package.json b/package.json index d106a10..94956b4 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "preview": "vite preview", "tauri": "tauri", "verify": "tsx scripts/verify.ts", + "fetch:asr": "tsx scripts/fetch-ondevice-asr.ts", "verify:live": "tsx scripts/verify-live.ts", "art:prompts": "tsx scripts/extract-art-prompts.ts", "script:review": "tsx scripts/build-script-review.ts" diff --git a/scripts/fetch-ondevice-asr.ts b/scripts/fetch-ondevice-asr.ts new file mode 100644 index 0000000..470bd40 --- /dev/null +++ b/scripts/fetch-ondevice-asr.ts @@ -0,0 +1,213 @@ +/** + * 端侧 ASR 资产获取：pnpm fetch:asr + * + * 下载 sherpa-onnx WASM 运行时 + SenseVoiceSmall int8 模型，产物落到 public/asr/ + * （该目录在 .gitignore 里，模型不进 git；换机器/删了目录就重跑本脚本）。 + * + * 资产来源（2026-10-04 核实，大小为当天下载实测字节数）： + * - WASM 运行时：HuggingFace 官方 space k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc + * https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/resolve/main/<文件名> + * （glue 95,308 B / wasm 11,722,172 B / api 47,391 B） + * 注意：普通 wasm/asr 运行时（GitHub release 的 zipformer/paraformer demo）只导出 + * Online（流式）API，Offline 系列不可用；vad-asr 构建才导出 OfflineRecognizer + * （wasm/vad-asr/CMakeLists.txt 的 exported_functions），而 SenseVoice 是离线模型， + * 必须用这一份。文件逐个下载，无需解压 tarball。 + * - 模型：GitHub release asr-models + * https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 + * （163,002,883 B；内含 model.int8.onnx 239,233,841 B + tokens.txt 315,894 B） + * + * 关键改造：官方 glue（sherpa-onnx-wasm-main-vad-asr.js）内嵌的 Emscripten 打包清单指向 + * zipformer-ctc demo 的 silero_vad + zipformer 模型（367MB .data）。我们把清单改写为只含 + * /model.int8.onnx + /tokens.txt 两个文件，并据此生成自己的 .data（两文件直接拼接， + * Emscripten preload 格式 = 原始字节流 + JS 侧清单）。这样运行时一次请求拿到模型包， + * 模型出现在 MEMFS 根路径，OfflineRecognizer 按绝对路径读取。 + * + * 产物（public/asr/）： + * sherpa-onnx-wasm-main-vad-asr.js glue（清单已改写） + * sherpa-onnx-wasm-main-vad-asr.wasm 运行时 wasm（11,722,172 B，含 VAD + Offline API） + * sherpa-onnx-wasm-main-vad-asr.data 模型预加载包（≈239,549,735 B） + * sherpa-onnx-asr.js 官方 JS API 封装（OfflineRecognizer） + * sense-voice-LICENSE.txt 模型许可（FunASR 模型许可，允许商用需署名） + * manifest.json 资产清单 + 字节数（引擎下载进度与完整性自检用） + */ +import { spawnSync } from 'node:child_process'; +import { createWriteStream } from 'node:fs'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const OUT_DIR = path.join(ROOT, 'public', 'asr'); +const FORCE = process.argv.includes('--force'); + +const HF_SPACE = + 'https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-ctc/resolve/main'; +const GLUE_NAME = 'sherpa-onnx-wasm-main-vad-asr.js'; +const WASM_NAME = 'sherpa-onnx-wasm-main-vad-asr.wasm'; +const API_NAME = 'sherpa-onnx-asr.js'; +const DATA_NAME = 'sherpa-onnx-wasm-main-vad-asr.data'; +const RUNTIME_FILES: Array<{ name: string; bytes: number }> = [ + { name: GLUE_NAME, bytes: 95_308 }, + { name: WASM_NAME, bytes: 11_722_172 }, + { name: API_NAME, bytes: 47_391 }, +]; + +const MODEL_URL = + 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2'; +const MODEL_BYTES = 163_002_883; +// 模型 tarball 内 +const MODEL_ONNX_NAME = 'model.int8.onnx'; +const TOKENS_NAME = 'tokens.txt'; +const LICENSE_NAME = 'sense-voice-LICENSE.txt'; + +async function download(url: string, dest: string, expectedBytes: number): Promise { + for (let attempt = 1; attempt <= 3; attempt++) { + try { + const res = await fetch(url); + if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`); + const reader = res.body.getReader(); + const out = createWriteStream(dest); + let received = 0; + let logged = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + out.write(value); + received += value.byteLength; + if (received - logged >= 20 * 1024 * 1024) { + logged = received; + console.log(` … ${(received / 1e6).toFixed(0)} MB`); + } + } + await new Promise((resolve, reject) => out.end(resolve as () => void, reject)); + if (received !== expectedBytes) { + throw new Error(`大小不符：期望 ${expectedBytes}，实际 ${received}（上游资产可能已更新，请核对后改脚本常量）`); + } + return; + } catch (err) { + console.log(` 下载失败（第 ${attempt} 次）：${err instanceof Error ? err.message : err}`); + if (attempt === 3) throw err; + } + } +} + +async function extract(tarball: string, dir: string): Promise { + const r = spawnSync('tar', ['-xf', tarball, '-C', dir], { stdio: 'inherit' }); + if (r.status !== 0) throw new Error(`解压失败：${tarball}`); +} + +/** 只含一个顶层目录时返回该目录路径，否则返回 dir 本身 */ +async function onlySubdir(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + const dirs = entries.filter((e) => e.isDirectory()); + return dirs.length === 1 && entries.length === 1 ? path.join(dir, dirs[0].name) : dir; +} + +/** 把 glue 里 loadPackage({...}) 的实参整体替换为新的打包清单（括号配平扫描，兼容 key 带不带引号两种生成风格） */ +export function patchGlueMetadata(glue: string, files: Array<{ filename: string; start: number; end: number }>, total: number): string { + const anchor = 'loadPackage({'; + const start = glue.indexOf(anchor); + if (start === -1) throw new Error('glue 中找不到 loadPackage 调用'); + let depth = 0; + for (let i = start + 'loadPackage('.length; i < glue.length; i++) { + const ch = glue[i]; + if (ch === '{') depth++; + else if (ch === '}') { + depth--; + if (depth === 0) { + const meta = `{files:[${files.map((f) => `{filename:"${f.filename}",start:${f.start},end:${f.end}}`).join(',')}],remote_package_size:${total}}`; + return glue.slice(0, start) + 'loadPackage(' + meta + glue.slice(i + 1); + } + } + } + throw new Error('glue 中 loadPackage 清单括号不配平'); +} + +async function main() { + await fs.mkdir(OUT_DIR, { recursive: true }); + + const manifestPath = path.join(OUT_DIR, 'manifest.json'); + if (!FORCE && (await fs.stat(manifestPath).catch(() => null)) !== null) { + console.log('public/asr/ 已存在（--force 可强制重新下载）'); + return; + } + + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'jev-asr-')); + console.log(`临时目录：${tmp}`); + + console.log('\n1/2 下载 WASM 运行时（vad-asr 构建，含 Offline API）'); + for (const f of RUNTIME_FILES) { + console.log(` ${HF_SPACE}/${f.name}`); + await download(`${HF_SPACE}/${f.name}`, path.join(tmp, f.name), f.bytes); + } + + console.log(`\n2/2 下载 SenseVoiceSmall int8 模型（${(MODEL_BYTES / 1e6).toFixed(0)} MB）\n ${MODEL_URL}`); + const modelTar = path.join(tmp, 'model.tar.bz2'); + await download(MODEL_URL, modelTar, MODEL_BYTES); + const modelDir = path.join(tmp, 'model'); + await fs.mkdir(modelDir); + await extract(modelTar, modelDir); + const model = await onlySubdir(modelDir); + + const glue = await fs.readFile(path.join(tmp, GLUE_NAME), 'utf8'); + const wasm = await fs.readFile(path.join(tmp, WASM_NAME)); + const api = await fs.readFile(path.join(tmp, API_NAME)); + const modelOnnx = await fs.readFile(path.join(model, MODEL_ONNX_NAME)); + const tokens = await fs.readFile(path.join(model, TOKENS_NAME)); + const licenseText = await fs.readFile(path.join(model, 'LICENSE')).catch(() => null); + + // 生成 .data：model.int8.onnx 拼 tokens.txt；清单路径用绝对路径（MEMFS 根） + const data = Buffer.concat([modelOnnx, tokens]); + const patched = patchGlueMetadata( + glue, + [ + { filename: `/${MODEL_ONNX_NAME}`, start: 0, end: modelOnnx.byteLength }, + { filename: `/${TOKENS_NAME}`, start: modelOnnx.byteLength, end: data.byteLength }, + ], + data.byteLength, + ); + if (patched === glue) throw new Error('清单改写失败'); + + const assets: Array<{ name: string; bytes: number }> = [ + { name: GLUE_NAME, bytes: Buffer.byteLength(patched) }, + { name: WASM_NAME, bytes: wasm.byteLength }, + { name: DATA_NAME, bytes: data.byteLength }, + { name: API_NAME, bytes: api.byteLength }, + ]; + + await fs.writeFile(path.join(OUT_DIR, GLUE_NAME), patched); + await fs.writeFile(path.join(OUT_DIR, WASM_NAME), wasm); + await fs.writeFile(path.join(OUT_DIR, DATA_NAME), data); + await fs.writeFile(path.join(OUT_DIR, API_NAME), api); + if (licenseText) await fs.writeFile(path.join(OUT_DIR, LICENSE_NAME), licenseText); + await fs.writeFile( + path.join(OUT_DIR, 'manifest.json'), + JSON.stringify( + { + generatedAt: new Date().toISOString(), + runtime: { source: `${HF_SPACE} （k2-fsa 官方 space，vad-asr 构建）`, note: 'wasm/asr 构建不含 Offline API，SenseVoice 必须用 vad-asr 运行时' }, + model: { url: MODEL_URL, name: 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17' }, + assets, + }, + null, + 2, + ), + ); + + // 自检：清单字节数与落盘文件一致 + for (const a of assets) { + const actual = (await fs.stat(path.join(OUT_DIR, a.name))).size; + if (actual !== a.bytes) throw new Error(`自检失败：${a.name} 大小 ${actual} != 清单 ${a.bytes}`); + } + + await fs.rm(tmp, { recursive: true, force: true }); + console.log(`\n完成。产物在 public/asr/（共 ${(assets.reduce((n, a) => n + a.bytes, 0) / 1e6).toFixed(1)} MB）：`); + for (const a of assets) console.log(` ${(a.bytes / 1e6).toFixed(2).padStart(8)} MB ${a.name}`); + console.log('\n首次在浏览器/桌面端使用时懒加载到 CacheStorage；见 src/asr/ondevice.ts 头注释。'); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/verify-ondevice-asr.ts b/scripts/verify-ondevice-asr.ts new file mode 100644 index 0000000..ca51eb5 --- /dev/null +++ b/scripts/verify-ondevice-asr.ts @@ -0,0 +1,175 @@ +/** + * 端侧小模型转写（sherpa-onnx WASM + SenseVoiceSmall）的 node 侧校验。 + * + * WASM 实跑只能在浏览器/WKWebView 里做，这里不断言识别结果，只断言： + * 1. 引擎接口形状：ready/init/transcribe 齐全；无浏览器能力时 init 优雅返回 false； + * 2. mock 引擎跑通接口契约（verify 与真机共用同一形状）； + * 3. 长音频切分纯函数：≤28s、连续无重叠无遗漏、优先落静音谷； + * 4. 重采样 48k→16k 的长度与插值正确； + * 5. fetch 脚本的 glue 打包清单改写（括号配平，兼容带/不带引号两种生成风格）。 + * 运行：pnpm verify（由 verify.ts 末尾挂链引入）。 + */ +import assert from 'node:assert'; + +import { resampleTo16k } from '../src/asr/capture'; +import { SENSE_VOICE_MAX_SEGMENT_SEC, segmentForSenseVoice } from '../src/asr/segment'; +import { createMockOndeviceEngine, getOndeviceEngine, ondeviceAsrEnabled } from '../src/asr/ondevice'; +import { patchGlueMetadata } from './fetch-ondevice-asr'; + +let failures = 0; +function check(ok: boolean, msg: string) { + if (!ok) { + failures++; + console.error(`✗ ${msg}`); + } else { + console.log(`✓ ${msg}`); + } +} + +const SR = 16000; + +// --- 1. 引擎接口形状与优雅降级 --- +{ + const engine = getOndeviceEngine(); + check( + typeof engine.ready === 'function' && typeof engine.init === 'function' && typeof engine.transcribe === 'function', + '端侧引擎接口形状：ready()/init()/transcribe() 齐全', + ); + check(engine.ready() === false, '端侧引擎初始未就绪'); + check(ondeviceAsrEnabled() === true, '端侧引擎默认开启（VITE_ONDEVICE_ASR 未设）'); + const ok = await engine.init(() => undefined); // node 无 Worker/CacheStorage + check(ok === false && engine.ready() === false, '无浏览器能力时 init 返回 false（静默降级，不抛错）'); + let threw = false; + try { + await engine.transcribe(new Float32Array(SR)); + } catch { + threw = true; + } + check(threw, '未就绪时 transcribe 抛错（调用方捕获后走降级通路）'); +} + +// --- 2. mock 引擎契约 --- +{ + const mock = createMockOndeviceEngine({ text: '端侧 mock 转写' }); + check(mock.ready() === true, 'mock 引擎 ready'); + check((await mock.init()) === true, 'mock 引擎 init'); + const t = await mock.transcribe(new Float32Array(SR)); + check(t.text === '端侧 mock 转写', 'mock 引擎 transcribe 返回文本'); + const tagged = createMockOndeviceEngine({ text: 'x', emotion: '<|HAPPY|>', event: '<|BGM|>', lang: '<|zh|>' }); + const t2 = await tagged.transcribe(new Float32Array(0)); + check(t2.emotion === '<|HAPPY|>' && t2.event === '<|BGM|>' && t2.lang === '<|zh|>', '情感/事件/语种标签可透传（SenseVoice 经 sherpa-onnx 暴露）'); +} + +// --- 3. 长音频切分 --- +function sine(len: number, freq = 440): Float32Array { + const a = new Float32Array(len); + for (let i = 0; i < len; i++) a[i] = Math.sin((2 * Math.PI * freq * i) / SR); + return a; +} + +function concat(segs: Float32Array[]): Float32Array { + const n = segs.reduce((s, x) => s + x.length, 0); + const out = new Float32Array(n); + let o = 0; + for (const s of segs) { + out.set(s, o); + o += s.length; + } + return out; +} + +{ + const short = sine(10 * SR); + const segs = segmentForSenseVoice(short); + check(segs.length === 1 && segs[0].length === short.length, '10 秒音频不切分（单段原样）'); + check(segmentForSenseVoice(new Float32Array(0)).length === 0, '空音频返回空数组'); + + // 60 秒"讲述"：每说 9 秒停 1.5 秒（口语停顿节奏），切点都应落在停顿带里 + const SPEECH = 9 * SR; + const PAUSE = 1.5 * SR; + const pcm = new Float32Array(60 * SR); + let cursor = 0; + let utterance = 0; + while (cursor < pcm.length) { + const n = Math.min(SPEECH, pcm.length - cursor); + pcm.set(sine(n, 440 + utterance * 100), cursor); // 不同音高，避免能量巧合 + cursor += n; + utterance++; + if (cursor < pcm.length) cursor = Math.min(cursor + PAUSE, pcm.length); + } + const segs2 = segmentForSenseVoice(pcm); + const maxLen = SENSE_VOICE_MAX_SEGMENT_SEC * SR; + check(segs2.length >= 2, `60 秒音频切为 ${segs2.length} 段（≥2）`); + check(segs2.every((s) => s.length <= maxLen), `每段 ≤ ${SENSE_VOICE_MAX_SEGMENT_SEC}s`); + check(concat(segs2).length === pcm.length, '分段拼接后长度等于原音频（无遗漏无重叠）'); + let silentCut = true; + let offset = 0; + for (const s of segs2.slice(0, -1)) { + offset += s.length; + // 切点应在停顿带内：向后 200ms 必须仍是静音（切点取停顿起始帧） + let sum = 0; + let n = 0; + for (let i = offset; i < Math.min(pcm.length, offset + 0.2 * SR); i++) { + sum += pcm[i] * pcm[i]; + n++; + } + if (n > 0 && Math.sqrt(sum / n) > 0.01) silentCut = false; + } + check(silentCut, '切点落在静音谷（不是词中间）'); + + // 无静音可找时硬切，仍满足长度与覆盖约束 + const solid = sine(90 * SR); + const segs3 = segmentForSenseVoice(solid); + check(segs3.length >= 4 && segs3.every((s) => s.length <= maxLen) && concat(segs3).length === solid.length, '连续无静音音频硬切：段数/长度/覆盖全部满足'); +} + +// --- 4. 重采样 --- +{ + const from = new Float32Array(48000); + for (let i = 0; i < from.length; i++) from[i] = i / 48000; // 线性 ramp，插值可精确验证 + const out = resampleTo16k(from, 48000); + check(out.length === 16000, '48k→16k 长度比例正确'); + check(Math.abs(out[100] - 100 * 3 / 48000) < 1e-6, '线性插值数值正确'); + const same = new Float32Array([0.1, 0.2]); + check(resampleTo16k(same, 16000) === same, '已是 16k 时原样返回（零拷贝）'); +} + +// --- 5. glue 打包清单改写 --- +{ + // v1.13.x glue 风格：key 不带引号 + const glueV1 = 'aa loadPackage({files:[{filename:"/.gitignore",start:0,end:0},{filename:"/encoder.onnx",start:0,end:195}],remote_package_size:195}); bb'; + // 旧 HF space glue 风格：key 带引号 + const glueV2 = 'aa loadPackage({"files":[{"filename":"/a","start":0,"end":1}],"remote_package_size":1})(); bb'; + const files = [ + { filename: '/model.int8.onnx', start: 0, end: 100 }, + { filename: '/tokens.txt', start: 100, end: 120 }, + ]; + for (const [name, src] of [['无引号', glueV1], ['带引号', glueV2]] as const) { + const out = patchGlueMetadata(src, files, 120); + check(out.includes('/model.int8.onnx') && out.includes('/tokens.txt') && out.includes('remote_package_size:120'), `glue 清单改写（${name} 风格）：文件名与总大小替换成功`); + check(out.includes('/encoder.onnx') === false && out.includes('"/a"') === false, `glue 清单改写（${name} 风格）：旧清单清除`); + // 括号配平：loadPackage( 后能找到配平的收尾 + const tail = out.slice(out.indexOf('loadPackage(') + 'loadPackage('.length); + let depth = 0; + let balanced = false; + for (const ch of tail) { + if (ch === '{') depth++; + else if (ch === '}') { + depth--; + if (depth === 0) { + balanced = true; + break; + } + } + } + check(balanced, `glue 清单改写（${name} 风格）：替换后括号配平`); + } + assert.throws(() => patchGlueMetadata('no anchor here', [], 0), /loadPackage/, '无锚点时抛错（上游 glue 结构变了会及时发现）'); + console.log('✓ glue 清单改写：无锚点时显式抛错'); +} + +if (failures > 0) { + console.error(`\n${failures} 项端侧 ASR 校验失败`); + process.exit(1); +} +console.log('\n端侧 ASR 接口形状校验通过（WASM 实跑需真机验证，见 README 已知缺口）'); diff --git a/src/asr/capture.ts b/src/asr/capture.ts new file mode 100644 index 0000000..f121e02 --- /dev/null +++ b/src/asr/capture.ts @@ -0,0 +1,163 @@ +/** + * 16kHz 单声道 PCM 采集器（端侧转写专用）。 + * + * AudioContext({sampleRate: 16000}) 只是采样率提示——Safari/WKWebView 经常忽略， + * 实际可能是 48kHz，所以统一按 context 实际采样率收原始 Float32，stop 时再线性 + * 重采样到 16k（与官方 wasm demo 的 downsampleBuffer 同款均值/线性思路，ASR 够用）。 + * + * 优先 AudioWorklet（不连 destination 会被部分浏览器挂起处理，这里经零增益 + * 节点接到 destination，标准做法）；不支持时退 ScriptProcessor。 + * 只产出 Float32Array（[-1,1]），不落地任何音频文件。 + */ + +export interface Pcm16kRecorder { + start(): Promise; + /** 停止采集并返回整段 16kHz 单声道 PCM；未在录音时抛错 */ + stop(): Float32Array; + /** 放弃已采集数据（权限失败/用户取消时的清理） */ + abort(): void; +} + +export const TARGET_SAMPLE_RATE = 16000; + +// 8 行 plain JS，用 Blob 喂给 addModule，绕开打包器对 AudioWorklet entry 的差异处理 +const WORKLET_SOURCE = ` +class PcmCaptureProcessor extends AudioWorkletProcessor { + process(inputs) { + const ch = inputs[0] && inputs[0][0]; + if (ch && ch.length > 0) this.port.postMessage(ch.slice()); + return true; + } +} +registerProcessor('pcm-capture', PcmCaptureProcessor); +`; + +/** 线性插值重采样（单声道，目标 16kHz） */ +export function resampleTo16k(pcm: Float32Array, fromRate: number): Float32Array { + if (fromRate === TARGET_SAMPLE_RATE || pcm.length === 0) return pcm; + const ratio = fromRate / TARGET_SAMPLE_RATE; + const outLen = Math.floor(pcm.length / ratio); + const out = new Float32Array(outLen); + for (let i = 0; i < outLen; i++) { + const pos = i * ratio; + const i0 = Math.floor(pos); + const i1 = Math.min(i0 + 1, pcm.length - 1); + const frac = pos - i0; + out[i] = pcm[i0] * (1 - frac) + pcm[i1] * frac; + } + return out; +} + +interface ActiveRecording { + ctx: AudioContext; + stream: MediaStream; + nativeRate: number; + chunks: Float32Array[]; + onChunk(chunk: Float32Array): void; + teardown(): void; +} + +let active: ActiveRecording | null = null; + +function collect(rec: ActiveRecording, chunk: Float32Array): void { + // AudioBuffer 的底层数组会被复用，必须拷贝 + rec.chunks.push(chunk.slice()); +} + +export function createPcm16kRecorder(): Pcm16kRecorder { + return { + async start() { + if (active) throw new Error('已有进行中的录音'); + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, + }); + const ctx = new AudioContext({ sampleRate: TARGET_SAMPLE_RATE }); + await ctx.resume(); + const source = ctx.createMediaStreamSource(stream); + const rec: ActiveRecording = { + ctx, + stream, + nativeRate: ctx.sampleRate, + chunks: [], + onChunk: () => undefined, + teardown: () => undefined, + }; + rec.onChunk = (c) => collect(rec, c); + + let disconnected = false; + const disconnect = () => { + if (disconnected) return; + disconnected = true; + try { + source.disconnect(); + } catch { + // 节点可能已被浏览器回收 + } + stream.getTracks().forEach((t) => t.stop()); + void ctx.close(); + }; + rec.teardown = disconnect; + + if (ctx.audioWorklet) { + try { + await ctx.audioWorklet.addModule( + URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'text/javascript' })), + ); + const node = new AudioWorkletNode(ctx, 'pcm-capture'); + node.port.onmessage = (e: MessageEvent) => rec.onChunk(e.data); + const sink = ctx.createGain(); + sink.gain.value = 0; // 不监听回放，只防止处理链被挂起 + source.connect(node).connect(sink).connect(ctx.destination); + active = rec; + return; + } catch { + // addModule/WorkletNode 失败 → ScriptProcessor 兜底 + } + } + + const bufferSize = 4096; + const legacy = ctx as AudioContext & { + createScriptProcessor?: (buf: number, inp: number, out: number) => ScriptProcessorNode; + createJavaScriptNode?: (buf: number, inp: number, out: number) => ScriptProcessorNode; + }; + const node = + legacy.createScriptProcessor?.(bufferSize, 1, 1) ?? legacy.createJavaScriptNode?.(bufferSize, 1, 1); + if (!node) { + disconnect(); + throw new Error('当前环境不支持音频采集'); + } + node.onaudioprocess = (e: AudioProcessingEvent) => { + const ch = e.inputBuffer.getChannelData(0); + if (ch.length > 0) rec.onChunk(ch); + }; + const sink = ctx.createGain(); + sink.gain.value = 0; + source.connect(node).connect(sink).connect(ctx.destination); + active = rec; + }, + + stop() { + const rec = active; + if (!rec) throw new Error('未在录音'); + active = null; + rec.teardown(); + const n = rec.chunks.reduce((s, c) => s + c.length, 0); + const flat = new Float32Array(n); + let offset = 0; + for (const c of rec.chunks) { + flat.set(c, offset); + offset += c.length; + } + rec.chunks = []; + return resampleTo16k(flat, rec.nativeRate); + }, + + abort() { + const rec = active; + if (!rec) return; + active = null; + rec.chunks = []; + rec.teardown(); + }, + }; +} diff --git a/src/asr/ondevice-asr.worker.ts b/src/asr/ondevice-asr.worker.ts new file mode 100644 index 0000000..901e6d4 --- /dev/null +++ b/src/asr/ondevice-asr.worker.ts @@ -0,0 +1,166 @@ +/** + * 端侧识别 Web Worker（classic worker）：在后台线程加载 sherpa-onnx WASM 运行时， + * 主线程通过 postMessage 送 16kHz PCM，这里用 OfflineRecognizer 整段解码。 + * + * 资产传递方式：主线程已把 glue/api JS 文本与 .wasm/.data 二进制 fetch 好， + * Worker 里用 Blob URL 喂给 Emscripten： + * - glue（sherpa-onnx-wasm-main-vad-asr.js）在加载时立即拉取 .data 并解包到 MEMFS + * （.data 由 fetch 脚本生成 = model.int8.onnx 拼 tokens.txt，glue 内嵌打包清单 + * 已在下载时被改写为对应这两个文件）； + * - Module.locateFile 把 .wasm/.data 指向 Blob URL，避免 Worker 内二次网络请求； + * - 模型因此出现在 MEMFS 的 /model.int8.onnx 与 /tokens.txt，识别器直接按路径读。 + * + * 注意保持 classic worker（Vite 默认格式）：importScripts 需要它。 + */ +import { segmentForSenseVoice } from './segment'; +import type { OndeviceTranscript } from './ondevice'; + +type WorkerInMessage = + | { type: 'init'; glueJs: string; apiJs: string; wasm: ArrayBuffer; data: ArrayBuffer } + | { type: 'transcribe'; id: number; pcm: Float32Array }; + +type WorkerOutMessage = + | { type: 'ready' } + | { type: 'error'; stage: string; message: string } + | { type: 'result'; id: number } & OndeviceTranscript; + +declare function importScripts(...urls: string[]): void; + +interface SherpaStream { + acceptWaveform(sampleRate: number, samples: Float32Array): void; + free(): void; +} + +interface SherpaResult { + text?: string; + lang?: string; + emotion?: string; + event?: string; +} + +interface SherpaOfflineRecognizer { + createStream(): SherpaStream; + decode(stream: SherpaStream): void; + getResult(stream: SherpaStream): SherpaResult; + free(): void; +} + +declare const OfflineRecognizer: new (config: unknown, module: unknown) => SherpaOfflineRecognizer; + +/** Emscripten Module 的全局形状（只声明我们用到的字段） */ +interface SherpaModuleGlobal { + locateFile?(path: string, prefix: string): string; + onRuntimeInitialized?(): void; + print?(msg: string): void; + printErr?(msg: string): void; +} + +const scope = self as unknown as { + Module?: SherpaModuleGlobal; + onmessage: ((e: MessageEvent) => void) | null; + postMessage(msg: WorkerOutMessage): void; + close(): void; +}; + +function fail(stage: string, err: unknown): void { + scope.postMessage({ type: 'error', stage, message: err instanceof Error ? err.message : String(err) }); +} + +let recognizer: SherpaOfflineRecognizer | null = null; + +function init(msg: Extract): Promise { + const wasmUrl = URL.createObjectURL(new Blob([msg.wasm], { type: 'application/wasm' })); + const dataUrl = URL.createObjectURL(new Blob([msg.data], { type: 'application/octet-stream' })); + + const ready = new Promise((resolve, reject) => { + scope.Module = { + locateFile: (path: string) => { + if (path.endsWith('.wasm')) return wasmUrl; + if (path.endsWith('.data')) return dataUrl; + return path; + }, + onRuntimeInitialized: () => resolve(), + print: () => undefined, // 官方 glue 很吵，静默 + printErr: () => undefined, + }; + // 180 秒还没初始化成功（解包 240MB + 编译 wasm）就放弃 + setTimeout(() => reject(new Error('WASM 运行时初始化超时')), 180_000); + }); + + const glueUrl = URL.createObjectURL(new Blob([msg.glueJs], { type: 'text/javascript' })); + const apiUrl = URL.createObjectURL(new Blob([msg.apiJs], { type: 'text/javascript' })); + importScripts(glueUrl, apiUrl); + + return ready.then(() => { + const config = { + featConfig: { sampleRate: 16000, featureDim: 80 }, + modelConfig: { + senseVoice: { + model: '/model.int8.onnx', + language: 'auto', + useInverseTextNormalization: 1, + }, + tokens: '/tokens.txt', + numThreads: 1, + provider: 'cpu', + debug: 0, + }, + decodingMethod: 'greedy_search', + }; + recognizer = new OfflineRecognizer(config, scope.Module); + }); +} + +function transcribe(id: number, pcm: Float32Array): void { + if (!recognizer) throw new Error('识别器未初始化'); + const segments = segmentForSenseVoice(pcm); + const texts: string[] = []; + const votes = { lang: new Map(), emotion: new Map(), event: new Map() }; + + for (const seg of segments) { + const stream = recognizer.createStream(); + try { + stream.acceptWaveform(16000, seg); + recognizer.decode(stream); + const r = recognizer.getResult(stream); + texts.push(r.text ?? ''); + for (const key of ['lang', 'emotion', 'event'] as const) { + const value = r[key]; + if (value) votes[key].set(value, (votes[key].get(value) ?? 0) + 1); + } + } finally { + stream.free(); + } + } + + const pick = (m: Map): string | undefined => + [...m.entries()].sort((a, b) => b[1] - a[1])[0]?.[0]; + + scope.postMessage({ + type: 'result', + id, + text: texts.join(''), + lang: pick(votes.lang), + emotion: pick(votes.emotion), + event: pick(votes.event), + }); +} + +scope.onmessage = (e) => { + const msg = e.data; + if (msg.type === 'init') { + init(msg).then( + () => scope.postMessage({ type: 'ready' }), + (err) => { + fail('init', err); + scope.close(); + }, + ); + return; + } + try { + transcribe(msg.id, msg.pcm); + } catch (err) { + fail('transcribe', err); + } +}; diff --git a/src/asr/ondevice.ts b/src/asr/ondevice.ts new file mode 100644 index 0000000..2d391fa --- /dev/null +++ b/src/asr/ondevice.ts @@ -0,0 +1,278 @@ +/** + * 端侧小模型转写引擎（ADR-0004 最强"即焚"：音频不出设备，只留转写文本）。 + * + * 选型：sherpa-onnx WASM + SenseVoiceSmall int8（docs/research/asr-options.md 1.2 / 4.1 / 4.4）。 + * 运行时在 Web Worker 里加载（避免阻塞 UI），主线程只负责把 16kHz 单声道 PCM + * 整体送进去解码——SenseVoice 是离线（Offline）识别器，录完一次性转写。 + * + * 资产（`public/asr/`，由 `pnpm fetch:asr` 下载，不进 git）： + * - sherpa-onnx-wasm-main-vad-asr.js / .wasm：官方 WASM 运行时，取自 k2-fsa 的 + * vad-asr space。注意必须是 vad-asr 构建：普通 wasm/asr 运行时只导出 Online + * （流式）API，而 SenseVoice 是离线模型，Offline 系列（CreateOfflineRecognizer + * 等）只有 vad-asr 构建导出（见 wasm/vad-asr/CMakeLists.txt 的 exported_functions）。 + * glue 的打包清单已在下载时被改写为只含 model.int8.onnx + tokens.txt 两个文件； + * - sherpa-onnx-wasm-main-vad-asr.data：Emscripten 预加载包 = model.int8.onnx 拼 tokens.txt； + * - sherpa-onnx-asr.js：官方 JS API 封装（OfflineRecognizer）； + * - manifest.json：文件名 + 字节数，用于下载进度统计与完整性自检。 + * + * 情感/事件标签：SenseVoice 经由 sherpa-onnx 的离线结果 JSON 暴露 lang / emotion / + * event 字段（sherpa-onnx csrc/offline-recognizer-sense-voice-impl.h），引擎原样透传， + * 不自行编造。VoiceInput 第一版只回填文本，标签留给讲述事件接线。 + * + * 用 `VITE_ONDEVICE_ASR=0` 关闭（默认开）。任何一步失败（资产缺失、Worker 挂了、 + * 浏览器不支持）都表现为 init() 返回 false，调用方静默退回录音上传通路。 + */ + +import { segmentForSenseVoice } from './segment'; + +export interface OndeviceTranscript { + text: string; + /** SenseVoice 输出，形如 "<|zh|>" / "<|HAPPY|>" / "<|BGM|>"，无则缺省 */ + lang?: string; + emotion?: string; + event?: string; +} + +export interface OndeviceAsrEngine { + /** 模型与运行时是否已就绪（init 成功过后恒为 true） */ + ready(): boolean; + /** 懒加载资产（fetch → CacheStorage，带进度）并启动 Worker 里的 WASM 运行时 */ + init(onProgress?: (loaded: number, total: number) => void): Promise; + /** 整段 16kHz 单声道 PCM（[-1,1] Float32）→ 本地转写文本 */ + transcribe(pcm16k: Float32Array): Promise; +} + +/** public/asr/ 下的资产清单（与 scripts/fetch-ondevice-asr.ts 产出的 manifest 对应） */ +// Vite 注入 import.meta.env；node（verify）下不存在，读法要兼容两种环境 +const viteEnv = (import.meta.env as unknown as { BASE_URL?: string; VITE_ONDEVICE_ASR?: string } | undefined) ?? {}; +const ASSET_BASE = `${viteEnv.BASE_URL ?? '/'}asr/`; +const CACHE_NAME = 'jev-ondevice-asr-v1'; + +interface AssetEntry { + name: string; + bytes: number; +} + +interface Manifest { + generatedAt: string; + assets: AssetEntry[]; +} + +// Worker 协议（与 ondevice-asr.worker.ts 对应） +type WorkerInMessage = + | { type: 'init'; glueJs: string; apiJs: string; wasm: ArrayBuffer; data: ArrayBuffer } + | { type: 'transcribe'; id: number; pcm: Float32Array }; + +type WorkerOutMessage = + | { type: 'ready' } + | { type: 'error'; stage: string; message: string } + | { type: 'result'; id: number; text: string; lang?: string; emotion?: string; event?: string }; + +export function ondeviceAsrEnabled(): boolean { + return viteEnv.VITE_ONDEVICE_ASR !== '0'; +} + +// 最近一次懒加载进度（组件可能多次挂载，让后来的 VoiceInput 直接看到） +let lastLoadProgress: { loaded: number; total: number } | null = null; + +export function getOndeviceLoadProgress(): { loaded: number; total: number } | null { + return lastLoadProgress; +} + +function supportsOndeviceAsr(): boolean { + return ( + typeof Worker !== 'undefined' && + typeof fetch !== 'undefined' && + typeof Blob !== 'undefined' && + typeof URL !== 'undefined' && + typeof URL.createObjectURL === 'function' + ); +} + +/** fetch（带下载进度）→ CacheStorage；缓存命中则不重复下载。缓存不可用时退化为普通 fetch。 */ +async function fetchAsset( + entry: AssetEntry, + cache: Cache | null, + onBytes: (loaded: number) => void, +): Promise { + const url = ASSET_BASE + entry.name; + if (cache) { + const hit = await cache.match(url); + if (hit) { + const buf = await hit.arrayBuffer(); + onBytes(buf.byteLength); + return buf; + } + } + + const res = await fetch(url); + if (!res.ok || !res.body) throw new Error(`下载端侧识别资产失败：${entry.name} (${res.status})`); + + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let received = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + received += value.byteLength; + onBytes(value.byteLength); + } + const buf = new Uint8Array(received); + let offset = 0; + for (const c of chunks) { + buf.set(c, offset); + offset += c.byteLength; + } + if (cache && received === entry.bytes) { + // 只缓存完整文件；Content-Type 不影响 ArrayBuffer 消费 + await cache.put(url, new Response(buf.slice().buffer, { headers: { 'Content-Type': 'application/octet-stream' } })); + } + return buf.buffer; +} + +class OndeviceAsrEngineImpl implements OndeviceAsrEngine { + private worker: Worker | null = null; + private isReady = false; + private initPromise: Promise | null = null; + private nextId = 1; + private readonly pending = new Map< + number, + { resolve: (t: OndeviceTranscript) => void; reject: (e: Error) => void; timer: ReturnType } + >(); + + ready(): boolean { + return this.isReady; + } + + init(onProgress?: (loaded: number, total: number) => void): Promise { + this.initPromise ??= this.doInit(onProgress).catch(() => false); + return this.initPromise; + } + + private async doInit(onProgress?: (loaded: number, total: number) => void): Promise { + if (!ondeviceAsrEnabled() || !supportsOndeviceAsr()) return false; + + let manifest: Manifest; + try { + const res = await fetch(ASSET_BASE + 'manifest.json'); + if (!res.ok) return false; // 资产未获取（未跑 pnpm fetch:asr），静默降级 + manifest = (await res.json()) as Manifest; + } catch { + return false; + } + + const cache = typeof caches !== 'undefined' ? await caches.open(CACHE_NAME).catch(() => null) : null; + const total = manifest.assets.reduce((n, a) => n + a.bytes, 0); + let loaded = 0; + const report = () => { + lastLoadProgress = { loaded, total }; + onProgress?.(loaded, total); + }; + + const buffers = new Map(); + try { + for (const entry of manifest.assets) { + const before = loaded; + buffers.set(entry.name, await fetchAsset(entry, cache, (n) => { + loaded = before + n; + report(); + })); + } + } catch { + return false; // 下载失败（网络/磁盘），降级到录音上传通路 + } + + const glueJs = new TextDecoder().decode(buffers.get('sherpa-onnx-wasm-main-vad-asr.js')); + const apiJs = new TextDecoder().decode(buffers.get('sherpa-onnx-asr.js')); + const wasm = buffers.get('sherpa-onnx-wasm-main-vad-asr.wasm'); + const data = buffers.get('sherpa-onnx-wasm-main-vad-asr.data'); + if (!glueJs || !apiJs || !wasm || !data) return false; + + const worker = new Worker(new URL('./ondevice-asr.worker.ts', import.meta.url)); + this.worker = worker; + worker.onmessage = (e: MessageEvent) => this.onMessage(e.data); + worker.onerror = (e) => this.failAll(new Error(`端侧识别 Worker 异常：${e.message}`)); + + const ready = new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), 180_000); // WASM 初始化（含 data 解包）超时 + this.readyResolve = (ok: boolean) => { + clearTimeout(timer); + resolve(ok); + }; + }); + + const msg: WorkerInMessage = { type: 'init', glueJs, apiJs, wasm, data }; + worker.postMessage(msg, [wasm, data]); + this.isReady = await ready; + return this.isReady; + } + + private readyResolve: ((ok: boolean) => void) | null = null; + + private onMessage(msg: WorkerOutMessage): void { + if (msg.type === 'ready') { + this.isReady = true; + this.readyResolve?.(true); + this.readyResolve = null; + return; + } + if (msg.type === 'error') { + const err = new Error(`${msg.stage}: ${msg.message}`); + if (this.readyResolve) { + this.readyResolve?.(false); + this.readyResolve = null; + } else { + this.failAll(err); + } + return; + } + const entry = this.pending.get(msg.id); + if (!entry) return; + clearTimeout(entry.timer); + this.pending.delete(msg.id); + entry.resolve({ text: msg.text, lang: msg.lang, emotion: msg.emotion, event: msg.event }); + } + + private failAll(err: Error): void { + for (const [, entry] of this.pending) { + clearTimeout(entry.timer); + entry.reject(err); + } + this.pending.clear(); + } + + async transcribe(pcm16k: Float32Array): Promise { + if (!this.worker || !this.isReady) throw new Error('端侧识别引擎未就绪'); + const id = this.nextId++; + const msg: WorkerInMessage = { type: 'transcribe', id, pcm: pcm16k }; + this.worker.postMessage(msg, [pcm16k.buffer]); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new Error('端侧转写超时')); + }, 600_000); + this.pending.set(id, { resolve, reject, timer }); + }); + } +} + +let singleton: OndeviceAsrEngine | null = null; + +/** 进程内单例：VoiceInput 每次挂载复用同一个引擎（重复 init 是幂等的） */ +export function getOndeviceEngine(): OndeviceAsrEngine { + singleton ??= new OndeviceAsrEngineImpl(); + return singleton; +} + +/** 测试/verify 用：以假实现校验接口形状（真实 WASM 只能在浏览器里跑） */ +export function createMockOndeviceEngine(result: OndeviceTranscript = { text: 'mock 转写文本' }): OndeviceAsrEngine { + return { + ready: () => true, + init: async () => true, + transcribe: async (pcm) => { + void segmentForSenseVoice(pcm); // mock 也走一遍切分，保持形状一致 + return result; + }, + }; +} diff --git a/src/asr/segment.ts b/src/asr/segment.ts new file mode 100644 index 0000000..54c365e --- /dev/null +++ b/src/asr/segment.ts @@ -0,0 +1,61 @@ +/** + * SenseVoice 单次推理上限约 30 秒音频（见 docs/research/asr-options.md 4.1）， + * 更长的讲述必须在送 OfflineRecognizer 之前切分。 + * + * 纯函数、无 Web API 依赖，verify 在 node 下直接测： + * - 短音频原样返回单个分段； + * - 长音频按 ≤ MAX_SEGMENT_SEC 切分，切点优先选边界附近的静音谷 + * （避免把词拦腰截断），找不到静音谷才硬切； + * - 分段连续、无重叠、无遗漏（concat 后等于原序列）。 + */ + +export const SENSE_VOICE_MAX_SEGMENT_SEC = 28; +const SEARCH_BACK_SEC = 8; +const FRAME_MS = 20; + +function frameRms(pcm: Float32Array, start: number, len: number): number { + let sum = 0; + const end = Math.min(start + len, pcm.length); + for (let i = start; i < end; i++) sum += pcm[i] * pcm[i]; + return end > start ? Math.sqrt(sum / (end - start)) : 0; +} + +/** 在 [limit - searchBack, limit] 内找 RMS 最小的帧起点作为切分点。 */ +function findSplitPoint(pcm: Float32Array, limit: number, frameLen: number, searchBack: number): number { + const start = Math.max(0, limit - searchBack); + let best = start; + let bestRms = Infinity; + for (let i = start; i + frameLen <= limit; i += frameLen) { + const rms = frameRms(pcm, i, frameLen); + if (rms < bestRms) { + bestRms = rms; + best = i; + } + } + return best; +} + +export function segmentForSenseVoice( + pcm: Float32Array, + sampleRate = 16000, +): Float32Array[] { + if (pcm.length === 0) return []; + const maxLen = Math.floor(SENSE_VOICE_MAX_SEGMENT_SEC * sampleRate); + if (pcm.length <= maxLen) return [pcm]; + + const frameLen = Math.max(1, Math.floor((FRAME_MS / 1000) * sampleRate)); + const searchBack = Math.floor(SEARCH_BACK_SEC * sampleRate); + + const segments: Float32Array[] = []; + let offset = 0; + while (offset < pcm.length) { + if (pcm.length - offset <= maxLen) { + segments.push(pcm.subarray(offset)); + break; + } + const split = findSplitPoint(pcm, offset + maxLen, frameLen, searchBack); + segments.push(pcm.subarray(offset, split)); + offset = split; + } + return segments; +} diff --git a/src/ui/VoiceInput.tsx b/src/ui/VoiceInput.tsx index a00d389..6064e1f 100644 --- a/src/ui/VoiceInput.tsx +++ b/src/ui/VoiceInput.tsx @@ -1,8 +1,15 @@ import { useEffect, useMemo, useRef, useState } from 'react'; +import { createPcm16kRecorder } from '../asr/capture'; +import type { Pcm16kRecorder } from '../asr/capture'; +import { getOndeviceEngine, getOndeviceLoadProgress, ondeviceAsrEnabled } from '../asr/ondevice'; + /** - * 语音讲述输入。三条通路，按能力自动选择，文本框永远可用： + * 语音讲述输入。四条通路，按能力自动选择，文本框永远可用： * + * 0. 端侧转写（第一优先，模型就绪时）：sherpa-onnx WASM + SenseVoiceSmall 在本地 + * 转写，音频不出设备（ADR-0004 最强"即焚"，只留转写文本）。模型首次使用懒加载 + * （进度见下方提示），未就绪/失败时静默退回后面的通路。`VITE_ONDEVICE_ASR=0` 关闭； * 1. SpeechRecognition（桌面 Chrome/Edge）：点按开始/结束，实时转写，沉默自动续录； * 2. MediaRecorder 录音上传（Tauri WKWebView / 不支持 SR 的环境）： * 录音 → 上传 server 的 /api/asr → 转写文本回填进可编辑文本框； @@ -61,20 +68,88 @@ export function VoiceInput({ onSubmit, disabled }: Props) { const SR = useMemo(() => getSRCtor(), []); const [recording, setRecording] = useState(false); // SR 识别中 const [recAudio, setRecAudio] = useState(false); // MediaRecorder 录音中 + const [recOndevice, setRecOndevice] = useState(false); // 端侧录音中 const [interim, setInterim] = useState(''); const [text, setText] = useState(''); const [hint, setHint] = useState(null); const [busy, setBusy] = useState(false); + // 端侧引擎状态：loading（首次懒加载，含下载进度）/ ready / failed / disabled + const [ondevice, setOndevice] = useState<'loading' | 'ready' | 'failed' | 'disabled'>(() => + ondeviceAsrEnabled() ? 'loading' : 'disabled', + ); + const [dlProgress, setDlProgress] = useState<{ loaded: number; total: number } | null>(() => + getOndeviceLoadProgress(), + ); const recRef = useRef(null); const wantRef = useRef(false); const fatalRef = useRef(false); const mediaRecRef = useRef(null); const chunksRef = useRef([]); + const pcmRecRef = useRef(null); /** SR 识别引擎不可用（无 SR 或已致命失败）时，改用录音上传通路 */ const useAudioPath = !SR || fatalRef.current; + // ---- 通路 0：端侧转写（sherpa-onnx WASM + SenseVoiceSmall，音频不出设备） ---- + + useEffect(() => { + if (!ondeviceAsrEnabled()) return; + let alive = true; + getOndeviceEngine() + .init((loaded, total) => { + if (alive) setDlProgress({ loaded, total }); + }) + .then((ok) => { + if (alive) setOndevice(ok ? 'ready' : 'failed'); + }); + return () => { + alive = false; + }; + }, []); + + async function startOndevice() { + setHint(null); + const rec = createPcm16kRecorder(); + try { + await rec.start(); + } catch { + setHint('麦克风不可用或未授权——直接打字也完全没问题。'); + return; + } + pcmRecRef.current = rec; + setRecOndevice(true); + } + + async function stopOndevice() { + const rec = pcmRecRef.current; + if (!rec) return; + pcmRecRef.current = null; + setRecOndevice(false); + setBusy(true); + setHint(null); + try { + const pcm = rec.stop(); + if (pcm.length < 16000 * 0.3) { + setHint('这段太短没听清——可以重说一遍，或直接打字。'); + return; + } + const t = await getOndeviceEngine().transcribe(pcm); + const transcribed = t.text.trim(); + if (transcribed) { + // SenseVoice 的 emotion/event 标签已随结果返回（t.emotion/t.event）， + // 等讲述事件接线后再进画像；回填文本框第一版只用文本。 + setText((prev) => (prev ? prev + transcribed : transcribed)); + } else { + setHint('端侧转写没听清这段——可以重说一遍，或直接打字。'); + } + } catch { + setHint('端侧转写失败——可以改用录音上传，或直接打字。'); + } finally { + setBusy(false); + } + } + // ---- 通路 1：SpeechRecognition ---- function launchSr() { @@ -205,6 +280,7 @@ export function VoiceInput({ onSubmit, disabled }: Props) { } catch { // 卸载清理 } + pcmRecRef.current?.abort(); }, [], ); @@ -240,6 +316,16 @@ export function VoiceInput({ onSubmit, disabled }: Props) { } />
+ {ondevice === 'ready' && + (recOndevice ? ( + + ) : ( + + ))} {!useAudioPath && (recording ? ( ))}
+ {ondevice === 'loading' && dlProgress && dlProgress.total > 0 && ( +

+ 端侧识别模型首次加载中 {Math.floor((dlProgress.loaded / dlProgress.total) * 100)}%（之后不再下载）——等不及可以先用下面的录音或打字。 +

+ )} + {recOndevice &&

端侧讲述中——音频只在本机转写，说完点「停止并转写」。

} {recording &&

讲述中——停顿会自动续录，说完点「结束讲述」。

} {recAudio &&

录音中——说完点「停止并转写」。

} {hint &&

{hint}

} diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 23ba92e..ade304d 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -3,6 +3,8 @@ interface ImportMetaEnv { /** TypeSafe AI System One (Jev) API key；缺失时自动降级 MockJevClient */ readonly VITE_TYPESAFE_API_KEY?: string; + /** 端侧小模型转写（sherpa-onnx WASM + SenseVoiceSmall）开关；'0' 关闭，默认开 */ + readonly VITE_ONDEVICE_ASR?: string; } interface ImportMeta {