feat: 端侧转写通路（sherpa-onnx WASM + SenseVoiceSmall，音频不出设备）+ 模型获取脚本与接口校验
This commit is contained in:
Jayway Liu
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* 端侧小模型转写（sherpa-onnx WASM + SenseVoiceSmall）的 node 侧校验。
|
||||
*
|
||||
* WASM 实跑只能在浏览器/WKWebView 里做，这里不断言识别结果，只断言：
|
||||
* 1. 引擎接口形状：ready/init/transcribe 齐全；无浏览器能力时 init 优雅返回 false；
|
||||
* 2. mock 引擎跑通接口契约（verify 与真机共用同一形状）；
|
||||
* 3. 长音频切分纯函数：≤28s、连续无重叠无遗漏、优先落静音谷；
|
||||
* 4. 重采样 48k→16k 的长度与插值正确；
|
||||
* 5. fetch 脚本的 glue 打包清单改写（括号配平，兼容带/不带引号两种生成风格）。
|
||||
* 运行：pnpm verify（由 verify.ts 末尾挂链引入）。
|
||||
*/
|
||||
import assert from 'node:assert';
|
||||
|
||||
import { resampleTo16k } from '../src/asr/capture';
|
||||
import { SENSE_VOICE_MAX_SEGMENT_SEC, segmentForSenseVoice } from '../src/asr/segment';
|
||||
import { createMockOndeviceEngine, getOndeviceEngine, ondeviceAsrEnabled } from '../src/asr/ondevice';
|
||||
import { patchGlueMetadata } from './fetch-ondevice-asr';
|
||||
|
||||
let failures = 0;
|
||||
function check(ok: boolean, msg: string) {
|
||||
if (!ok) {
|
||||
failures++;
|
||||
console.error(`✗ ${msg}`);
|
||||
} else {
|
||||
console.log(`✓ ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
const SR = 16000;
|
||||
|
||||
// --- 1. 引擎接口形状与优雅降级 ---
|
||||
{
|
||||
const engine = getOndeviceEngine();
|
||||
check(
|
||||
typeof engine.ready === 'function' && typeof engine.init === 'function' && typeof engine.transcribe === 'function',
|
||||
'端侧引擎接口形状：ready()/init()/transcribe() 齐全',
|
||||
);
|
||||
check(engine.ready() === false, '端侧引擎初始未就绪');
|
||||
check(ondeviceAsrEnabled() === true, '端侧引擎默认开启（VITE_ONDEVICE_ASR 未设）');
|
||||
const ok = await engine.init(() => undefined); // node 无 Worker/CacheStorage
|
||||
check(ok === false && engine.ready() === false, '无浏览器能力时 init 返回 false（静默降级，不抛错）');
|
||||
let threw = false;
|
||||
try {
|
||||
await engine.transcribe(new Float32Array(SR));
|
||||
} catch {
|
||||
threw = true;
|
||||
}
|
||||
check(threw, '未就绪时 transcribe 抛错（调用方捕获后走降级通路）');
|
||||
}
|
||||
|
||||
// --- 2. mock 引擎契约 ---
|
||||
{
|
||||
const mock = createMockOndeviceEngine({ text: '端侧 mock 转写' });
|
||||
check(mock.ready() === true, 'mock 引擎 ready');
|
||||
check((await mock.init()) === true, 'mock 引擎 init');
|
||||
const t = await mock.transcribe(new Float32Array(SR));
|
||||
check(t.text === '端侧 mock 转写', 'mock 引擎 transcribe 返回文本');
|
||||
const tagged = createMockOndeviceEngine({ text: 'x', emotion: '<|HAPPY|>', event: '<|BGM|>', lang: '<|zh|>' });
|
||||
const t2 = await tagged.transcribe(new Float32Array(0));
|
||||
check(t2.emotion === '<|HAPPY|>' && t2.event === '<|BGM|>' && t2.lang === '<|zh|>', '情感/事件/语种标签可透传（SenseVoice 经 sherpa-onnx 暴露）');
|
||||
}
|
||||
|
||||
// --- 3. 长音频切分 ---
|
||||
function sine(len: number, freq = 440): Float32Array {
|
||||
const a = new Float32Array(len);
|
||||
for (let i = 0; i < len; i++) a[i] = Math.sin((2 * Math.PI * freq * i) / SR);
|
||||
return a;
|
||||
}
|
||||
|
||||
function concat(segs: Float32Array[]): Float32Array {
|
||||
const n = segs.reduce((s, x) => s + x.length, 0);
|
||||
const out = new Float32Array(n);
|
||||
let o = 0;
|
||||
for (const s of segs) {
|
||||
out.set(s, o);
|
||||
o += s.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
{
|
||||
const short = sine(10 * SR);
|
||||
const segs = segmentForSenseVoice(short);
|
||||
check(segs.length === 1 && segs[0].length === short.length, '10 秒音频不切分（单段原样）');
|
||||
check(segmentForSenseVoice(new Float32Array(0)).length === 0, '空音频返回空数组');
|
||||
|
||||
// 60 秒"讲述"：每说 9 秒停 1.5 秒（口语停顿节奏），切点都应落在停顿带里
|
||||
const SPEECH = 9 * SR;
|
||||
const PAUSE = 1.5 * SR;
|
||||
const pcm = new Float32Array(60 * SR);
|
||||
let cursor = 0;
|
||||
let utterance = 0;
|
||||
while (cursor < pcm.length) {
|
||||
const n = Math.min(SPEECH, pcm.length - cursor);
|
||||
pcm.set(sine(n, 440 + utterance * 100), cursor); // 不同音高，避免能量巧合
|
||||
cursor += n;
|
||||
utterance++;
|
||||
if (cursor < pcm.length) cursor = Math.min(cursor + PAUSE, pcm.length);
|
||||
}
|
||||
const segs2 = segmentForSenseVoice(pcm);
|
||||
const maxLen = SENSE_VOICE_MAX_SEGMENT_SEC * SR;
|
||||
check(segs2.length >= 2, `60 秒音频切为 ${segs2.length} 段（≥2）`);
|
||||
check(segs2.every((s) => s.length <= maxLen), `每段 ≤ ${SENSE_VOICE_MAX_SEGMENT_SEC}s`);
|
||||
check(concat(segs2).length === pcm.length, '分段拼接后长度等于原音频（无遗漏无重叠）');
|
||||
let silentCut = true;
|
||||
let offset = 0;
|
||||
for (const s of segs2.slice(0, -1)) {
|
||||
offset += s.length;
|
||||
// 切点应在停顿带内：向后 200ms 必须仍是静音（切点取停顿起始帧）
|
||||
let sum = 0;
|
||||
let n = 0;
|
||||
for (let i = offset; i < Math.min(pcm.length, offset + 0.2 * SR); i++) {
|
||||
sum += pcm[i] * pcm[i];
|
||||
n++;
|
||||
}
|
||||
if (n > 0 && Math.sqrt(sum / n) > 0.01) silentCut = false;
|
||||
}
|
||||
check(silentCut, '切点落在静音谷（不是词中间）');
|
||||
|
||||
// 无静音可找时硬切，仍满足长度与覆盖约束
|
||||
const solid = sine(90 * SR);
|
||||
const segs3 = segmentForSenseVoice(solid);
|
||||
check(segs3.length >= 4 && segs3.every((s) => s.length <= maxLen) && concat(segs3).length === solid.length, '连续无静音音频硬切：段数/长度/覆盖全部满足');
|
||||
}
|
||||
|
||||
// --- 4. 重采样 ---
|
||||
{
|
||||
const from = new Float32Array(48000);
|
||||
for (let i = 0; i < from.length; i++) from[i] = i / 48000; // 线性 ramp，插值可精确验证
|
||||
const out = resampleTo16k(from, 48000);
|
||||
check(out.length === 16000, '48k→16k 长度比例正确');
|
||||
check(Math.abs(out[100] - 100 * 3 / 48000) < 1e-6, '线性插值数值正确');
|
||||
const same = new Float32Array([0.1, 0.2]);
|
||||
check(resampleTo16k(same, 16000) === same, '已是 16k 时原样返回（零拷贝）');
|
||||
}
|
||||
|
||||
// --- 5. glue 打包清单改写 ---
|
||||
{
|
||||
// v1.13.x glue 风格：key 不带引号
|
||||
const glueV1 = 'aa loadPackage({files:[{filename:"/.gitignore",start:0,end:0},{filename:"/encoder.onnx",start:0,end:195}],remote_package_size:195}); bb';
|
||||
// 旧 HF space glue 风格：key 带引号
|
||||
const glueV2 = 'aa loadPackage({"files":[{"filename":"/a","start":0,"end":1}],"remote_package_size":1})(); bb';
|
||||
const files = [
|
||||
{ filename: '/model.int8.onnx', start: 0, end: 100 },
|
||||
{ filename: '/tokens.txt', start: 100, end: 120 },
|
||||
];
|
||||
for (const [name, src] of [['无引号', glueV1], ['带引号', glueV2]] as const) {
|
||||
const out = patchGlueMetadata(src, files, 120);
|
||||
check(out.includes('/model.int8.onnx') && out.includes('/tokens.txt') && out.includes('remote_package_size:120'), `glue 清单改写（${name} 风格）：文件名与总大小替换成功`);
|
||||
check(out.includes('/encoder.onnx') === false && out.includes('"/a"') === false, `glue 清单改写（${name} 风格）：旧清单清除`);
|
||||
// 括号配平：loadPackage( 后能找到配平的收尾
|
||||
const tail = out.slice(out.indexOf('loadPackage(') + 'loadPackage('.length);
|
||||
let depth = 0;
|
||||
let balanced = false;
|
||||
for (const ch of tail) {
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
balanced = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
check(balanced, `glue 清单改写（${name} 风格）：替换后括号配平`);
|
||||
}
|
||||
assert.throws(() => patchGlueMetadata('no anchor here', [], 0), /loadPackage/, '无锚点时抛错（上游 glue 结构变了会及时发现）');
|
||||
console.log('✓ glue 清单改写：无锚点时显式抛错');
|
||||
}
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`\n${failures} 项端侧 ASR 校验失败`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('\n端侧 ASR 接口形状校验通过（WASM 实跑需真机验证，见 README 已知缺口）');
|
||||
Reference in New Issue
Block a user