feat: 실시간 음성대화 GPU 엔진(whisper+XTTS) + Ollama 쿼터 자동 폴백
- voice_engine.py: faster-whisper(STT)+XTTS-v2(TTS)를 상시 로드해 로컬에서 서빙하는 WebSocket 엔진 (배치/스트리밍 프로토콜) - routes-voice-realtime.ts: 브라우저 WS를 voice_engine.py로 그대로 프록시 - voice-call.js/worklets: 실시간 연속 대화 모드(VAD 기반 발화 감지, barge-in), 마크다운/표/URL을 정리하고 읽는 sanitizeForSpeech 포함 - tts.ts/stt.ts: xtts_gpu/whisper_gpu provider 분기 추가 - ollama-client.ts/factory.ts: Ollama Cloud 세션 쿼터 초과 시 설정된 fallback 모델로 자동 재시도 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -232,16 +232,22 @@
|
||||
"path": "/hooks"
|
||||
},
|
||||
"voice": {
|
||||
"enabled": false,
|
||||
"enabled": true,
|
||||
"stt": {
|
||||
"provider": "whisper_cpp",
|
||||
"provider": "whisper_gpu",
|
||||
"model": "ggml-base.bin",
|
||||
"language": "ko"
|
||||
},
|
||||
"tts": {
|
||||
"provider": "edge_tts",
|
||||
"provider": "xtts_gpu",
|
||||
"model": "ko_KR-kss-medium",
|
||||
"voice": "ko-KR-SunHiNeural"
|
||||
},
|
||||
"engine": {
|
||||
"port": 8765,
|
||||
"sttModel": "medium",
|
||||
"speakerWav": "/home/kim/homeclaw/.smallclaw/voice/speaker_ko.wav",
|
||||
"device": "cuda"
|
||||
}
|
||||
},
|
||||
"telegram": {
|
||||
|
||||
@@ -9,10 +9,19 @@
|
||||
* resetProvider() from the settings API). No other files need touching.
|
||||
*/
|
||||
|
||||
import { getProvider, getModelForRole, getPrimaryModel, resetProvider } from '../providers/factory';
|
||||
import { getProvider, getModelForRole, getPrimaryModel, getFallbackModel, resetProvider } from '../providers/factory';
|
||||
import type { LLMProvider, TokenUsage } from '../providers/LLMProvider';
|
||||
import { AgentRole } from '../types';
|
||||
|
||||
// Ollama Cloud's per-account session quota error. Matched case-insensitively
|
||||
// against the thrown error message to decide whether a fallback retry makes
|
||||
// sense (as opposed to e.g. a malformed request, which retrying won't fix).
|
||||
const QUOTA_ERROR_RE = /session usage limit/i;
|
||||
|
||||
function isQuotaError(err: any): boolean {
|
||||
return QUOTA_ERROR_RE.test(String(err?.message || err || ''));
|
||||
}
|
||||
|
||||
export interface GenerateOutput {
|
||||
response: string;
|
||||
thinking?: string;
|
||||
@@ -45,14 +54,25 @@ export class OllamaClient {
|
||||
}
|
||||
): Promise<ChatOutput> {
|
||||
const model = String(options?.model || '').trim() || getModelForRole(role);
|
||||
const result = await this.provider.chat(messages, model, {
|
||||
const chatOpts = {
|
||||
temperature: options?.temperature,
|
||||
max_tokens: options?.num_predict,
|
||||
num_ctx: options?.num_ctx,
|
||||
tools: options?.tools,
|
||||
think: options?.think,
|
||||
});
|
||||
return { message: result.message, thinking: result.thinking };
|
||||
};
|
||||
try {
|
||||
const result = await this.provider.chat(messages, model, chatOpts);
|
||||
return { message: result.message, thinking: result.thinking };
|
||||
} catch (err: any) {
|
||||
const fallback = getFallbackModel();
|
||||
if (fallback && fallback !== model && isQuotaError(err)) {
|
||||
console.warn(`[OllamaClient] ${model} hit quota limit, retrying with fallback model ${fallback}`);
|
||||
const result = await this.provider.chat(messages, fallback, chatOpts);
|
||||
return { message: result.message, thinking: result.thinking };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Streaming variant: yields text tokens via onToken callback as they arrive.
|
||||
@@ -91,8 +111,22 @@ export class OllamaClient {
|
||||
}
|
||||
}
|
||||
|
||||
// Non-streaming fallback
|
||||
const result = await this.provider.chat(messages, model, chatOpts);
|
||||
// Non-streaming fallback (also the retry path when the streaming attempt
|
||||
// above failed for a reason unrelated to quota — quota is handled below).
|
||||
const fallbackModel = getFallbackModel();
|
||||
let modelToUse = model;
|
||||
let result;
|
||||
try {
|
||||
result = await this.provider.chat(messages, modelToUse, chatOpts);
|
||||
} catch (err: any) {
|
||||
if (fallbackModel && fallbackModel !== modelToUse && isQuotaError(err)) {
|
||||
console.warn(`[OllamaClient] ${modelToUse} hit quota limit, retrying with fallback model ${fallbackModel}`);
|
||||
modelToUse = fallbackModel;
|
||||
result = await this.provider.chat(messages, modelToUse, chatOpts);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// Deliver the complete text as a single token callback for consistency
|
||||
const content = typeof result.message?.content === 'string' ? result.message.content : '';
|
||||
if (content && onToken) onToken(content);
|
||||
|
||||
@@ -275,6 +275,12 @@ export const DEFAULT_CONFIG: SmallClawConfig = {
|
||||
provider: 'piper' as const,
|
||||
model: 'ko_KR-kss-medium',
|
||||
},
|
||||
engine: {
|
||||
port: 8765,
|
||||
sttModel: 'medium',
|
||||
speakerWav: '',
|
||||
device: 'cuda',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -417,6 +423,10 @@ export class ConfigManager {
|
||||
...((DEFAULT_CONFIG.voice as any)?.tts || {}),
|
||||
...((loaded.voice as any)?.tts || {}),
|
||||
},
|
||||
engine: {
|
||||
...((DEFAULT_CONFIG.voice as any)?.engine || {}),
|
||||
...((loaded.voice as any)?.engine || {}),
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import * as http from 'http';
|
||||
import * as net from 'net';
|
||||
import express from 'express';
|
||||
import { ensureEngineRunning, getEnginePort, isEngineReady, isVoiceEngineConfigured } from '../tools/voice-engine-client.js';
|
||||
|
||||
export function registerVoiceRealtimeRoutes(app: express.Application): void {
|
||||
app.get('/api/voice/engine/status', (_req, res) => {
|
||||
res.json({
|
||||
configured: isVoiceEngineConfigured(),
|
||||
ready: isEngineReady(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Raw WebSocket upgrade proxy from the browser to the local GPU voice_engine.py
|
||||
// process (see attachAndroidWsProxy in routes-android.ts for the same pattern).
|
||||
// We don't need to interpret WS frames on the Node side — control messages (JSON
|
||||
// text frames) and audio (binary frames) both pass through untouched — so a raw
|
||||
// TCP pipe avoids re-implementing WS framing twice.
|
||||
export function attachVoiceRealtimeWsProxy(server: http.Server, getSessionUser: (req: any) => any): void {
|
||||
server.on('upgrade', (req, clientSocket, head) => {
|
||||
const url = req.url || '';
|
||||
if (!url.startsWith('/ws/voice-realtime')) return; // not ours — let other upgrade listeners handle it
|
||||
|
||||
const user = getSessionUser(req);
|
||||
if (!user) {
|
||||
clientSocket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
||||
clientSocket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
ensureEngineRunning()
|
||||
.then(() => {
|
||||
const port = getEnginePort();
|
||||
const backendSocket = net.connect({ host: '127.0.0.1', port }, () => {
|
||||
const headerLines = ['GET / HTTP/1.1'];
|
||||
for (let i = 0; i < req.rawHeaders.length; i += 2) {
|
||||
const key = req.rawHeaders[i];
|
||||
const val = key.toLowerCase() === 'host' ? `127.0.0.1:${port}` : req.rawHeaders[i + 1];
|
||||
headerLines.push(`${key}: ${val}`);
|
||||
}
|
||||
backendSocket.write(headerLines.join('\r\n') + '\r\n\r\n');
|
||||
if (head && head.length) backendSocket.write(head);
|
||||
|
||||
clientSocket.pipe(backendSocket);
|
||||
backendSocket.pipe(clientSocket);
|
||||
|
||||
const cleanup = () => {
|
||||
try { clientSocket.destroy(); } catch {}
|
||||
try { backendSocket.destroy(); } catch {}
|
||||
};
|
||||
clientSocket.on('close', cleanup);
|
||||
clientSocket.on('error', cleanup);
|
||||
backendSocket.on('close', cleanup);
|
||||
backendSocket.on('error', cleanup);
|
||||
});
|
||||
backendSocket.on('error', () => { try { clientSocket.destroy(); } catch {} });
|
||||
})
|
||||
.catch(() => {
|
||||
clientSocket.write('HTTP/1.1 503 Service Unavailable\r\n\r\n');
|
||||
clientSocket.destroy();
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -37,7 +37,28 @@ app.post('/api/voice/tts', async (req: express.Request, res: express.Response) =
|
||||
try {
|
||||
const os = await import('os');
|
||||
const { spawn: spawnProc } = await import('child_process');
|
||||
if (provider === 'edge_tts') {
|
||||
if (provider === 'xtts_gpu') {
|
||||
const { synthesizeFullGPU } = await import('../tools/voice-engine-client.js');
|
||||
const { pcm, sampleRate } = await synthesizeFullGPU(text);
|
||||
const pcmPath = path.join(os.tmpdir(), `tts_gpu_${Date.now()}.pcm`);
|
||||
const wavPath = path.join(os.tmpdir(), `tts_gpu_${Date.now()}.wav`);
|
||||
fs.writeFileSync(pcmPath, pcm);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawnProc('ffmpeg', [
|
||||
'-y', '-f', 's16le', '-ar', String(sampleRate), '-ac', '1', '-i', pcmPath,
|
||||
wavPath,
|
||||
]);
|
||||
let stderr = '';
|
||||
ffmpeg.stderr.on('data', (d: Buffer) => { stderr += d.toString('utf8'); });
|
||||
ffmpeg.on('close', (code: number | null) => code === 0 ? resolve() : reject(new Error(`ffmpeg exited ${code}: ${stderr}`)));
|
||||
ffmpeg.on('error', reject);
|
||||
});
|
||||
const wav = fs.readFileSync(wavPath);
|
||||
try { fs.unlinkSync(pcmPath); } catch {}
|
||||
try { fs.unlinkSync(wavPath); } catch {}
|
||||
res.set('Content-Type', 'audio/wav');
|
||||
res.send(wav);
|
||||
} else if (provider === 'edge_tts') {
|
||||
const voice = cfg.tts.voice || 'ko-KR-SunHiNeural';
|
||||
const mp3Path = path.join(os.tmpdir(), `tts_${Date.now()}.mp3`);
|
||||
const scriptPath = path.join(__dirname, '../../src/tools/edge_tts_synth.py');
|
||||
|
||||
@@ -20,6 +20,7 @@ import { registerArduinoRoutes } from './routes-arduino';
|
||||
import { registerAndroidRoutes, attachAndroidWsProxy } from './routes-android';
|
||||
import { registerPptxRoutes } from './routes-pptx';
|
||||
import { registerVoiceRoutes } from './routes-voice';
|
||||
import { registerVoiceRealtimeRoutes, attachVoiceRealtimeWsProxy } from './routes-voice-realtime';
|
||||
import { registerMusicRoutes } from './routes-music';
|
||||
import { registerMCPRoutes } from './routes-mcp';
|
||||
import { registerLanguageRoutes } from './routes-language';
|
||||
@@ -8994,6 +8995,23 @@ function extractAuthToken(req: express.Request | http.IncomingMessage): string |
|
||||
return cookies[AUTH_COOKIE] || null;
|
||||
}
|
||||
|
||||
function getSessionUserFromUpgradeReq(req: http.IncomingMessage): SessionInfo | null {
|
||||
// Browser WebSocket() can't set an Authorization header — accept a ?token=
|
||||
// query param too (same fallback the main /ws connection handler uses),
|
||||
// falling back to the auth cookie for same-origin connections.
|
||||
let token: string | null = null;
|
||||
try {
|
||||
const u = new URL(req.url || '/', `http://${req.headers.host || 'x'}`);
|
||||
token = u.searchParams.get('token');
|
||||
} catch {}
|
||||
if (!token) {
|
||||
const cookies = parseCookies(req as any);
|
||||
token = cookies[AUTH_COOKIE] || null;
|
||||
}
|
||||
if (!token) return null;
|
||||
return activeSessions.get(token) ?? null;
|
||||
}
|
||||
|
||||
function getSessionUser(req: express.Request): SessionInfo | null {
|
||||
const token = extractAuthToken(req);
|
||||
if (!token) return null;
|
||||
@@ -9741,6 +9759,33 @@ app.get('/api/admin/context-viewer', async (req, res) => {
|
||||
res.json({ sessions: withOnline });
|
||||
});
|
||||
|
||||
app.get('/api/admin/banned-ips', async (req, res) => {
|
||||
const session = getSessionUser(req);
|
||||
if (!session || session.role !== 'admin') return res.status(403).json({ error: 'Admin required' });
|
||||
try {
|
||||
const raw = fs.readFileSync('/var/log/fail2ban.log', 'utf-8');
|
||||
const banRe = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),\d+\s+fail2ban\.actions\s+\S+:\s+NOTICE\s+\[([\w.-]+)\]\s+(Ban|Unban)\s+(\S+)/;
|
||||
const current = new Map<string, { jail: string; ip: string; bannedAt: string }>();
|
||||
let totalBans = 0;
|
||||
for (const line of raw.split('\n')) {
|
||||
const m = banRe.exec(line);
|
||||
if (!m) continue;
|
||||
const [, ts, jail, action, ip] = m;
|
||||
const key = `${jail}:${ip}`;
|
||||
if (action === 'Ban') {
|
||||
totalBans++;
|
||||
current.set(key, { jail, ip, bannedAt: ts });
|
||||
} else {
|
||||
current.delete(key);
|
||||
}
|
||||
}
|
||||
const banned = [...current.values()].sort((a, b) => b.bannedAt.localeCompare(a.bannedAt));
|
||||
res.json({ banned, totalBans });
|
||||
} catch (e: any) {
|
||||
res.status(500).json({ error: e?.message || String(e) });
|
||||
}
|
||||
});
|
||||
|
||||
app.delete('/api/admin/sessions/:username/:id', async (req, res) => {
|
||||
const session = getSessionUser(req);
|
||||
if (!session || session.role !== 'admin') return res.status(403).json({ error: 'Admin required' });
|
||||
@@ -10292,6 +10337,7 @@ app.put('/api/files/{*filePath}', (req: express.Request, res: express.Response)
|
||||
});
|
||||
|
||||
registerVoiceRoutes(app);
|
||||
registerVoiceRealtimeRoutes(app);
|
||||
|
||||
// ── Canvas code execution ──────────────────────────────────────────────────
|
||||
const CANVAS_EXEC_EXT: Record<string, string> = {
|
||||
@@ -14050,11 +14096,12 @@ const server = http.createServer(app);
|
||||
// path dispatch means each handler only touches sockets meant for it.
|
||||
wss = new WebSocketServer({ noServer: true });
|
||||
server.on('upgrade', (req, socket, head) => {
|
||||
if ((req.url || '').startsWith('/ws')) {
|
||||
if ((req.url || '').startsWith('/ws') && !(req.url || '').startsWith('/ws/voice-realtime')) {
|
||||
wss.handleUpgrade(req, socket, head, (ws) => wss.emit('connection', ws, req));
|
||||
}
|
||||
});
|
||||
attachAndroidWsProxy(server, getSessionUser);
|
||||
attachVoiceRealtimeWsProxy(server, getSessionUserFromUpgradeReq);
|
||||
wss.on('error', (err: any) => {
|
||||
if (err?.code === 'EADDRINUSE') {
|
||||
console.error(`[Gateway] Port ${HOST}:${PORT} is already in use.`);
|
||||
@@ -14259,6 +14306,16 @@ server.listen(PORT, HOST, async () => {
|
||||
// Auto-connect enabled MCP servers
|
||||
getMCPManager().startEnabledServers().catch(err => console.warn('[MCP] Startup error:', err?.message));
|
||||
|
||||
// GPU voice engine (faster-whisper + XTTS) — only if configured as the active provider,
|
||||
// starts lazily in the background so a slow/failed GPU load never blocks gateway boot.
|
||||
const voiceCfg = (liveConfig as any).voice;
|
||||
if (voiceCfg?.enabled && (voiceCfg?.stt?.provider === 'whisper_gpu' || voiceCfg?.tts?.provider === 'xtts_gpu')) {
|
||||
import('../tools/voice-engine-client.js')
|
||||
.then((m) => m.ensureEngineRunning())
|
||||
.then(() => console.log('[VoiceEngine] GPU voice engine ready.'))
|
||||
.catch((err: any) => console.warn('[VoiceEngine] Startup error (will retry on first request):', err?.message));
|
||||
}
|
||||
|
||||
cronScheduler.start();
|
||||
console.log('[CronScheduler] Tick loop started — heartbeat:', cronScheduler.getConfig().enabled ? 'ON' : 'OFF');
|
||||
initializeAgentSchedules();
|
||||
|
||||
@@ -205,3 +205,10 @@ export function getModelForRole(role: 'manager' | 'executor' | 'verifier' | 'bac
|
||||
export function getPrimaryModel(): string {
|
||||
return getModelForRole('executor');
|
||||
}
|
||||
|
||||
/** Local/cheap model to retry against when the active model hits a
|
||||
* provider-side quota/rate limit (e.g. Ollama Cloud's session usage limit). */
|
||||
export function getFallbackModel(): string {
|
||||
const raw = getConfig().getConfig() as any;
|
||||
return String(raw.models?.fallback || '').trim();
|
||||
}
|
||||
|
||||
@@ -62,6 +62,13 @@ export async function transcribeAudio(
|
||||
sourceFormat: string = 'ogg',
|
||||
languageOverride?: string
|
||||
): Promise<{ text: string; language?: string }> {
|
||||
const cfg = getConfig().getConfig() as any;
|
||||
const provider = cfg?.voice?.stt?.provider;
|
||||
if (provider === 'whisper_gpu') {
|
||||
const { transcribeFullGPU } = await import('./voice-engine-client.js');
|
||||
return transcribeFullGPU(audioBuffer, sourceFormat, languageOverride || cfg?.voice?.stt?.language);
|
||||
}
|
||||
|
||||
const config = getSTTConfig();
|
||||
|
||||
if (!config.modelPath) {
|
||||
|
||||
+31
-1
@@ -31,10 +31,40 @@ export function isTTSAvailable(): boolean {
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(text: string): Promise<Buffer> {
|
||||
const cfg = getConfig().getConfig() as any;
|
||||
const provider = cfg?.voice?.tts?.provider;
|
||||
const truncatedText = text.slice(0, 4000);
|
||||
|
||||
if (provider === 'xtts_gpu') {
|
||||
const tempDir = cfg?.voice?.tempDir || os.tmpdir();
|
||||
const ffmpegPath = cfg?.voice?.ffmpegPath || 'ffmpeg';
|
||||
const { synthesizeFullGPU } = await import('./voice-engine-client.js');
|
||||
const { pcm, sampleRate } = await synthesizeFullGPU(truncatedText);
|
||||
const pcmPath = path.join(tempDir, `tts_gpu_${Date.now()}.pcm`);
|
||||
const oggPath = path.join(tempDir, `tts_gpu_${Date.now()}.ogg`);
|
||||
fs.writeFileSync(pcmPath, pcm);
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn(ffmpegPath, [
|
||||
'-y', '-f', 's16le', '-ar', String(sampleRate), '-ac', '1', '-i', pcmPath,
|
||||
'-c:a', 'libopus', '-b:a', '48k', '-vbr', 'on', '-compression_level', '10',
|
||||
oggPath,
|
||||
]);
|
||||
let stderr = '';
|
||||
ffmpeg.stderr.on('data', (d: Buffer) => { stderr += d.toString('utf8'); });
|
||||
ffmpeg.on('close', (code) => code === 0 ? resolve() : reject(new Error(`ffmpeg exited ${code}: ${stderr}`)));
|
||||
ffmpeg.on('error', reject);
|
||||
});
|
||||
return fs.readFileSync(oggPath);
|
||||
} finally {
|
||||
try { fs.unlinkSync(pcmPath); } catch {}
|
||||
try { fs.unlinkSync(oggPath); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const config = getTTSConfig();
|
||||
if (!config) throw new Error('TTS not configured');
|
||||
|
||||
const truncatedText = text.slice(0, 4000);
|
||||
const tempDir = config.tempDir;
|
||||
const wavPath = path.join(tempDir, `tts_output_${Date.now()}.wav`);
|
||||
const oggPath = path.join(tempDir, `tts_output_${Date.now()}.ogg`);
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import { spawn, ChildProcess } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { WebSocket } from 'ws';
|
||||
import { getConfig } from '../config/config.js';
|
||||
|
||||
interface EngineConfig {
|
||||
port: number;
|
||||
sttModel: string;
|
||||
speakerWav: string;
|
||||
device: string;
|
||||
}
|
||||
|
||||
function getEngineConfig(): EngineConfig {
|
||||
const cfg = getConfig().getConfig() as any;
|
||||
const engine = cfg?.voice?.engine || {};
|
||||
return {
|
||||
port: engine.port || 8765,
|
||||
sttModel: engine.sttModel || 'medium',
|
||||
speakerWav: engine.speakerWav || path.join(REPO_ROOT, '.smallclaw', 'voice', 'speaker_ko.wav'),
|
||||
device: engine.device || 'cuda',
|
||||
};
|
||||
}
|
||||
|
||||
const REPO_ROOT = path.join(__dirname, '..', '..');
|
||||
const VENV_PYTHON = path.join(REPO_ROOT, '.smallclaw', 'voice-venv', 'bin', 'python');
|
||||
const ENGINE_SCRIPT = path.join(REPO_ROOT, 'src', 'tools', 'voice_engine.py');
|
||||
|
||||
const STARTUP_TIMEOUT_MS = 180_000; // first load of whisper+XTTS onto GPU can take a while
|
||||
const PING_INTERVAL_MS = 2_000;
|
||||
|
||||
let proc: ChildProcess | null = null;
|
||||
let socket: WebSocket | null = null;
|
||||
let startPromise: Promise<void> | null = null;
|
||||
let restartAttempts = 0;
|
||||
|
||||
type Pending = { resolve: (v: any) => void; reject: (e: any) => void };
|
||||
const pending = new Map<string, Pending>();
|
||||
let nextId = 1;
|
||||
|
||||
function log(...args: any[]) {
|
||||
console.log('[voice-engine-client]', ...args);
|
||||
}
|
||||
|
||||
function resetConnectionState(reason: string) {
|
||||
for (const [, p] of pending) p.reject(new Error(`voice engine connection lost: ${reason}`));
|
||||
pending.clear();
|
||||
socket = null;
|
||||
}
|
||||
|
||||
function spawnEngine(): ChildProcess {
|
||||
const cfg = getEngineConfig();
|
||||
if (!fs.existsSync(VENV_PYTHON)) {
|
||||
throw new Error(`Voice engine venv not found at ${VENV_PYTHON}. Run: python3 -m venv .smallclaw/voice-venv && .smallclaw/voice-venv/bin/pip install faster-whisper coqui-tts g2pkk websockets`);
|
||||
}
|
||||
const args = [
|
||||
ENGINE_SCRIPT,
|
||||
'--port', String(cfg.port),
|
||||
'--stt-model', cfg.sttModel,
|
||||
'--speaker-wav', cfg.speakerWav,
|
||||
'--device', cfg.device,
|
||||
];
|
||||
log('spawning', VENV_PYTHON, args.join(' '));
|
||||
const venvSitePackages = path.join(REPO_ROOT, '.smallclaw', 'voice-venv', 'lib', 'python3.14', 'site-packages');
|
||||
// ctranslate2's GPU wheel dynamically links CUDA 12 libs that aren't present system-wide
|
||||
// (this host runs CUDA 13 + torch's bundled cu130 runtime) — point it at the pip-installed
|
||||
// nvidia-cublas-cu12 / nvidia-cudnn-cu12 packages instead.
|
||||
const cudaLibDirs = [
|
||||
path.join(venvSitePackages, 'nvidia', 'cublas', 'lib'),
|
||||
path.join(venvSitePackages, 'nvidia', 'cudnn', 'lib'),
|
||||
].filter(fs.existsSync);
|
||||
const env = {
|
||||
...process.env,
|
||||
LD_LIBRARY_PATH: [...cudaLibDirs, process.env.LD_LIBRARY_PATH].filter(Boolean).join(':'),
|
||||
COQUI_TOS_AGREED: '1',
|
||||
};
|
||||
const child = spawn(VENV_PYTHON, args, { stdio: ['ignore', 'pipe', 'pipe'], env });
|
||||
child.stdout?.on('data', (d: Buffer) => log('[stdout]', d.toString('utf8').trim()));
|
||||
child.stderr?.on('data', (d: Buffer) => log('[stderr]', d.toString('utf8').trim()));
|
||||
child.on('exit', (code, signal) => {
|
||||
log(`engine process exited code=${code} signal=${signal}`);
|
||||
proc = null;
|
||||
resetConnectionState('process exited');
|
||||
scheduleRestart();
|
||||
});
|
||||
return child;
|
||||
}
|
||||
|
||||
function scheduleRestart() {
|
||||
restartAttempts++;
|
||||
const delay = Math.min(30_000, 2_000 * restartAttempts);
|
||||
log(`scheduling restart in ${delay}ms (attempt ${restartAttempts})`);
|
||||
setTimeout(() => {
|
||||
ensureEngineRunning().catch(e => log('restart failed:', e.message));
|
||||
}, delay);
|
||||
}
|
||||
|
||||
function connectSocket(port: number): Promise<WebSocket> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ws = new WebSocket(`ws://127.0.0.1:${port}`);
|
||||
const timer = setTimeout(() => { ws.terminate(); reject(new Error('WS connect timeout')); }, STARTUP_TIMEOUT_MS);
|
||||
ws.once('open', () => { clearTimeout(timer); resolve(ws); });
|
||||
ws.once('error', (e) => { clearTimeout(timer); reject(e); });
|
||||
});
|
||||
}
|
||||
|
||||
function wireSocket(ws: WebSocket) {
|
||||
ws.on('message', (data: Buffer, isBinary: boolean) => {
|
||||
if (isBinary) return; // phase 2: streaming binary audio frames
|
||||
let msg: any;
|
||||
try { msg = JSON.parse(data.toString('utf8')); } catch { return; }
|
||||
const p = msg.id != null ? pending.get(String(msg.id)) : undefined;
|
||||
if (!p) return;
|
||||
pending.delete(String(msg.id));
|
||||
if (msg.type === 'error') p.reject(new Error(msg.message || 'voice engine error'));
|
||||
else p.resolve(msg);
|
||||
});
|
||||
ws.on('close', () => resetConnectionState('socket closed'));
|
||||
ws.on('error', (e) => log('socket error:', e.message));
|
||||
}
|
||||
|
||||
export async function ensureEngineRunning(): Promise<void> {
|
||||
if (socket && socket.readyState === WebSocket.OPEN) return;
|
||||
if (startPromise) return startPromise;
|
||||
|
||||
startPromise = (async () => {
|
||||
const cfg = getEngineConfig();
|
||||
if (!proc) proc = spawnEngine();
|
||||
|
||||
let lastErr: any;
|
||||
const deadline = Date.now() + STARTUP_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const ws = await connectSocket(cfg.port);
|
||||
wireSocket(ws);
|
||||
socket = ws;
|
||||
restartAttempts = 0;
|
||||
log('connected to voice engine on port', cfg.port);
|
||||
return;
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
await new Promise(r => setTimeout(r, 1_000));
|
||||
}
|
||||
}
|
||||
throw new Error(`voice engine failed to become ready: ${lastErr?.message || lastErr}`);
|
||||
})();
|
||||
|
||||
try {
|
||||
await startPromise;
|
||||
} finally {
|
||||
startPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function request(msg: Record<string, any>, timeoutMs = 60_000): Promise<any> {
|
||||
await ensureEngineRunning();
|
||||
if (!socket) throw new Error('voice engine socket not connected');
|
||||
const id = String(nextId++);
|
||||
const payload = JSON.stringify({ ...msg, id });
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
pending.delete(id);
|
||||
reject(new Error('voice engine request timed out'));
|
||||
}, timeoutMs);
|
||||
pending.set(id, {
|
||||
resolve: (v) => { clearTimeout(timer); resolve(v); },
|
||||
reject: (e) => { clearTimeout(timer); reject(e); },
|
||||
});
|
||||
socket!.send(payload);
|
||||
});
|
||||
}
|
||||
|
||||
export async function transcribeFullGPU(audioBuffer: Buffer, format: string, language?: string): Promise<{ text: string; language?: string }> {
|
||||
const res = await request({
|
||||
type: 'stt_transcribe_full',
|
||||
audioBase64: audioBuffer.toString('base64'),
|
||||
format,
|
||||
language: language || 'ko',
|
||||
}, 120_000);
|
||||
return { text: res.text, language: res.language };
|
||||
}
|
||||
|
||||
export async function synthesizeFullGPU(text: string): Promise<{ pcm: Buffer; sampleRate: number }> {
|
||||
const res = await request({ type: 'tts_synthesize_full', text }, 120_000);
|
||||
return { pcm: Buffer.from(res.audioBase64, 'base64'), sampleRate: res.sampleRate };
|
||||
}
|
||||
|
||||
export function isVoiceEngineConfigured(): boolean {
|
||||
return fs.existsSync(VENV_PYTHON) && fs.existsSync(ENGINE_SCRIPT);
|
||||
}
|
||||
|
||||
export function getEnginePort(): number {
|
||||
return getEngineConfig().port;
|
||||
}
|
||||
|
||||
export function isEngineReady(): boolean {
|
||||
return socket !== null && socket.readyState === WebSocket.OPEN;
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Persistent GPU voice engine: loads faster-whisper (STT) and XTTS-v2 (TTS)
|
||||
once and serves them over a localhost WebSocket so Node never pays model
|
||||
load latency per request.
|
||||
|
||||
Batch protocol (used by Telegram + the existing single-shot web TTS button):
|
||||
-> {"type":"ping"}
|
||||
<- {"type":"pong","ready":true}
|
||||
|
||||
-> {"type":"stt_transcribe_full","audioBase64":"...","format":"webm","language":"ko"}
|
||||
<- {"type":"stt_final","text":"...","language":"ko"}
|
||||
|
||||
-> {"type":"tts_synthesize_full","text":"..."}
|
||||
<- {"type":"tts_result","audioBase64":"...","sampleRate":24000}
|
||||
|
||||
Streaming protocol (used by the web UI continuous-conversation mode):
|
||||
-> {"type":"stt_start","sampleRate":16000,"language":"ko"}
|
||||
-> <binary PCM16 mono frames at sampleRate, repeated>
|
||||
<- {"type":"stt_partial","text":"..."} (emitted opportunistically as audio accumulates)
|
||||
-> {"type":"stt_stop"}
|
||||
<- {"type":"stt_final","text":"..."}
|
||||
|
||||
-> {"type":"tts_start","text":"...","id":"..."}
|
||||
<- {"type":"tts_stream_start","sampleRate":24000,"id":"..."}
|
||||
<- <binary PCM16 mono frames, in order>
|
||||
<- {"type":"tts_end","id":"..."}
|
||||
-> {"type":"tts_cancel"} (aborts the in-flight stream started by tts_start)
|
||||
|
||||
<- {"type":"error","message":"..."} on failure (echoes "id" when the request had one)
|
||||
|
||||
One WS connection == one call session; STT and TTS state live in ConnectionState
|
||||
(handle_connection is invoked fresh per client by websockets.serve).
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
import numpy as np
|
||||
import websockets
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='[voice_engine] %(asctime)s %(levelname)s %(message)s')
|
||||
log = logging.getLogger('voice_engine')
|
||||
# faster-whisper logs "Processing audio with duration ..." / "VAD filter
|
||||
# removed ..." at INFO level on every partial decode during streaming STT —
|
||||
# multiple times per second during a live conversation. Keep our own INFO
|
||||
# logs but quiet this one down to WARNING.
|
||||
logging.getLogger('faster_whisper').setLevel(logging.WARNING)
|
||||
|
||||
MIN_PARTIAL_BYTES = int(16000 * 2 * 0.6) # ~0.6s of 16kHz mono PCM16 before we bother decoding
|
||||
|
||||
|
||||
def is_unrecoverable_cuda_error(e: Exception) -> bool:
|
||||
# A CUDA "device-side assert" (seen from XTTS inference_stream on very short
|
||||
# text) poisons the whole CUDA context for the rest of the process — every
|
||||
# GPU call after it fails identically, whisper included. There is no
|
||||
# in-process recovery; the only fix is a fresh CUDA context, i.e. a restart.
|
||||
msg = str(e)
|
||||
return 'CUDA' in msg and ('assert' in msg.lower() or 'device-side' in msg.lower() or 'failed' in msg.lower())
|
||||
|
||||
|
||||
def crash_and_restart(where: str, e: Exception):
|
||||
log.error('Unrecoverable CUDA error in %s: %s — exiting so the supervisor restarts with a fresh CUDA context.', where, e)
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def load_models(args):
|
||||
log.info('Loading faster-whisper model=%s device=%s ...', args.stt_model, args.device)
|
||||
from faster_whisper import WhisperModel
|
||||
stt_model = WhisperModel(args.stt_model, device=args.device, compute_type='float16' if args.device == 'cuda' else 'int8')
|
||||
log.info('faster-whisper ready.')
|
||||
|
||||
log.info('Loading XTTS-v2 ...')
|
||||
from TTS.api import TTS
|
||||
tts_wrapper = TTS('tts_models/multilingual/multi-dataset/xtts_v2').to(args.device)
|
||||
xtts = tts_wrapper.synthesizer.tts_model # underlying TTS.tts.models.xtts.Xtts — has inference_stream()
|
||||
log.info('Computing speaker conditioning latents from %s ...', args.speaker_wav)
|
||||
gpt_cond_latent, speaker_embedding = xtts.get_conditioning_latents(audio_path=[args.speaker_wav])
|
||||
log.info('XTTS-v2 ready.')
|
||||
|
||||
return stt_model, tts_wrapper, xtts, gpt_cond_latent, speaker_embedding
|
||||
|
||||
|
||||
def pcm16_from_float(audio: np.ndarray) -> bytes:
|
||||
clipped = np.clip(audio, -1.0, 1.0)
|
||||
return (clipped * 32767.0).astype('<i2').tobytes()
|
||||
|
||||
|
||||
def pcm16_bytes_to_float(pcm: bytes) -> np.ndarray:
|
||||
return np.frombuffer(pcm, dtype='<i2').astype(np.float32) / 32768.0
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.stt_model = None
|
||||
self.tts_model = None # TTS.api.TTS wrapper (used for the batch .tts() call)
|
||||
self.xtts = None # underlying Xtts model (used for inference_stream)
|
||||
self.gpt_cond_latent = None
|
||||
self.speaker_embedding = None
|
||||
|
||||
def ready(self) -> bool:
|
||||
return self.stt_model is not None and self.xtts is not None
|
||||
|
||||
# ── Batch STT/TTS (Telegram, single-shot web button) ──────────────────
|
||||
def transcribe_full(self, audio_bytes: bytes, fmt: str, language: str):
|
||||
suffix = '.' + (fmt or 'webm').lstrip('.')
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f:
|
||||
f.write(audio_bytes)
|
||||
path = f.name
|
||||
try:
|
||||
segments, info = self.stt_model.transcribe(path, language=language or None, beam_size=5, vad_filter=True)
|
||||
text = ''.join(seg.text for seg in segments).strip()
|
||||
return text, (info.language if info else language)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def synthesize_full(self, text: str):
|
||||
wav = self.tts_model.tts(
|
||||
text=text[:4000],
|
||||
speaker_wav=self.args.speaker_wav,
|
||||
language='ko',
|
||||
)
|
||||
audio = np.asarray(wav, dtype=np.float32)
|
||||
pcm = pcm16_from_float(audio)
|
||||
sample_rate = int(self.tts_model.synthesizer.output_sample_rate)
|
||||
return pcm, sample_rate
|
||||
|
||||
# ── Streaming STT (partial decode of an accumulating PCM buffer) ──────
|
||||
def transcribe_pcm(self, pcm: bytes, language: str):
|
||||
audio = pcm16_bytes_to_float(pcm)
|
||||
segments, info = self.stt_model.transcribe(audio, language=language or None, beam_size=1, vad_filter=True)
|
||||
text = ''.join(seg.text for seg in segments).strip()
|
||||
return text, (info.language if info else language)
|
||||
|
||||
# ── Streaming TTS (XTTS inference_stream, chunk-by-chunk) ─────────────
|
||||
def synthesize_stream(self, text: str, is_cancelled):
|
||||
for chunk in self.xtts.inference_stream(
|
||||
text=text[:4000],
|
||||
language='ko',
|
||||
gpt_cond_latent=self.gpt_cond_latent,
|
||||
speaker_embedding=self.speaker_embedding,
|
||||
stream_chunk_size=20,
|
||||
):
|
||||
if is_cancelled():
|
||||
return
|
||||
audio = chunk.squeeze().detach().cpu().numpy().astype(np.float32)
|
||||
yield pcm16_from_float(audio)
|
||||
|
||||
@property
|
||||
def tts_sample_rate(self) -> int:
|
||||
return int(self.tts_model.synthesizer.output_sample_rate)
|
||||
|
||||
|
||||
class ConnectionState:
|
||||
def __init__(self):
|
||||
self.stt_buffer = bytearray()
|
||||
self.stt_decoding = False
|
||||
self.stt_language = 'ko'
|
||||
self.tts_cancel_flags = {} # id -> bool, checked by the producer thread
|
||||
|
||||
|
||||
async def handle_stt_partial(ws, engine: Engine, state: ConnectionState):
|
||||
if state.stt_decoding or len(state.stt_buffer) < MIN_PARTIAL_BYTES:
|
||||
return
|
||||
state.stt_decoding = True
|
||||
try:
|
||||
buf = bytes(state.stt_buffer)
|
||||
loop = asyncio.get_event_loop()
|
||||
text, _lang = await loop.run_in_executor(None, engine.transcribe_pcm, buf, state.stt_language)
|
||||
if text:
|
||||
await ws.send(json.dumps({'type': 'stt_partial', 'text': text}))
|
||||
except Exception as e:
|
||||
log.exception('stt_partial failed')
|
||||
if is_unrecoverable_cuda_error(e):
|
||||
crash_and_restart('handle_stt_partial', e)
|
||||
finally:
|
||||
state.stt_decoding = False
|
||||
|
||||
|
||||
async def handle_tts_start(ws, engine: Engine, state: ConnectionState, msg: dict):
|
||||
text = msg.get('text', '')
|
||||
req_id = msg.get('id')
|
||||
state.tts_cancel_flags[req_id] = False
|
||||
loop = asyncio.get_event_loop()
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
def is_cancelled():
|
||||
return state.tts_cancel_flags.get(req_id, False)
|
||||
|
||||
def produce():
|
||||
try:
|
||||
for chunk in engine.synthesize_stream(text, is_cancelled):
|
||||
loop.call_soon_threadsafe(queue.put_nowait, ('chunk', chunk))
|
||||
loop.call_soon_threadsafe(queue.put_nowait, ('done', None))
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.exception('tts stream failed')
|
||||
loop.call_soon_threadsafe(queue.put_nowait, ('error', str(e)))
|
||||
if is_unrecoverable_cuda_error(e):
|
||||
crash_and_restart('handle_tts_start', e)
|
||||
|
||||
threading.Thread(target=produce, daemon=True).start()
|
||||
await ws.send(json.dumps({'type': 'tts_stream_start', 'sampleRate': engine.tts_sample_rate, 'id': req_id}))
|
||||
try:
|
||||
while True:
|
||||
kind, payload = await queue.get()
|
||||
if kind == 'chunk':
|
||||
await ws.send(payload)
|
||||
elif kind == 'error':
|
||||
await ws.send(json.dumps({'type': 'error', 'message': payload, 'id': req_id}))
|
||||
break
|
||||
else:
|
||||
await ws.send(json.dumps({'type': 'tts_end', 'id': req_id}))
|
||||
break
|
||||
finally:
|
||||
state.tts_cancel_flags.pop(req_id, None)
|
||||
|
||||
|
||||
async def handle_connection(ws, engine: Engine):
|
||||
state = ConnectionState()
|
||||
async for raw in ws:
|
||||
try:
|
||||
if isinstance(raw, (bytes, bytearray)):
|
||||
state.stt_buffer.extend(raw)
|
||||
asyncio.create_task(handle_stt_partial(ws, engine, state))
|
||||
continue
|
||||
|
||||
msg = json.loads(raw)
|
||||
mtype = msg.get('type')
|
||||
req_id = msg.get('id')
|
||||
|
||||
if mtype == 'ping':
|
||||
await ws.send(json.dumps({'type': 'pong', 'ready': engine.ready(), 'id': req_id}))
|
||||
|
||||
elif mtype == 'stt_transcribe_full':
|
||||
audio_bytes = base64.b64decode(msg['audioBase64'])
|
||||
fmt = msg.get('format', 'webm')
|
||||
language = msg.get('language', 'ko')
|
||||
loop = asyncio.get_event_loop()
|
||||
text, lang = await loop.run_in_executor(None, engine.transcribe_full, audio_bytes, fmt, language)
|
||||
await ws.send(json.dumps({'type': 'stt_final', 'text': text, 'language': lang, 'id': req_id}))
|
||||
|
||||
elif mtype == 'tts_synthesize_full':
|
||||
text = msg.get('text', '')
|
||||
loop = asyncio.get_event_loop()
|
||||
pcm, sample_rate = await loop.run_in_executor(None, engine.synthesize_full, text)
|
||||
await ws.send(json.dumps({
|
||||
'type': 'tts_result',
|
||||
'audioBase64': base64.b64encode(pcm).decode('ascii'),
|
||||
'sampleRate': sample_rate,
|
||||
'id': req_id,
|
||||
}))
|
||||
|
||||
elif mtype == 'stt_start':
|
||||
state.stt_buffer = bytearray()
|
||||
state.stt_language = msg.get('language', 'ko')
|
||||
|
||||
elif mtype == 'stt_stop':
|
||||
buf = bytes(state.stt_buffer)
|
||||
state.stt_buffer = bytearray()
|
||||
loop = asyncio.get_event_loop()
|
||||
text, lang = ('', state.stt_language)
|
||||
if len(buf) >= 320: # at least ~10ms, skip empty stop
|
||||
text, lang = await loop.run_in_executor(None, engine.transcribe_pcm, buf, state.stt_language)
|
||||
await ws.send(json.dumps({'type': 'stt_final', 'text': text, 'language': lang, 'id': req_id}))
|
||||
|
||||
elif mtype == 'tts_start':
|
||||
asyncio.create_task(handle_tts_start(ws, engine, state, msg))
|
||||
|
||||
elif mtype == 'tts_cancel':
|
||||
target_id = msg.get('id')
|
||||
if target_id is not None:
|
||||
state.tts_cancel_flags[target_id] = True
|
||||
else:
|
||||
for k in state.tts_cancel_flags:
|
||||
state.tts_cancel_flags[k] = True
|
||||
|
||||
else:
|
||||
await ws.send(json.dumps({'type': 'error', 'message': f'unknown type {mtype}', 'id': req_id}))
|
||||
|
||||
except Exception as e:
|
||||
log.exception('request failed')
|
||||
try:
|
||||
await ws.send(json.dumps({'type': 'error', 'message': str(e), 'id': msg.get('id') if 'msg' in dir() else None}))
|
||||
except Exception:
|
||||
pass
|
||||
if is_unrecoverable_cuda_error(e):
|
||||
crash_and_restart('handle_connection', e)
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--port', type=int, default=8765)
|
||||
parser.add_argument('--stt-model', default='medium')
|
||||
parser.add_argument('--speaker-wav', default='')
|
||||
parser.add_argument('--device', default='cuda')
|
||||
args = parser.parse_args()
|
||||
|
||||
engine = Engine(args)
|
||||
(engine.stt_model, engine.tts_model, engine.xtts,
|
||||
engine.gpt_cond_latent, engine.speaker_embedding) = load_models(args)
|
||||
|
||||
async with websockets.serve(lambda ws: handle_connection(ws, engine), '127.0.0.1', args.port, max_size=64 * 1024 * 1024):
|
||||
log.info('voice_engine listening on 127.0.0.1:%d', args.port)
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
+9
-2
@@ -356,18 +356,25 @@ export interface SmallClawConfig {
|
||||
voice?: {
|
||||
enabled: boolean;
|
||||
stt: {
|
||||
provider: 'whisper_cpp';
|
||||
provider: 'whisper_cpp' | 'whisper_gpu';
|
||||
model: string;
|
||||
modelPath?: string;
|
||||
whisperPath?: string;
|
||||
language?: string;
|
||||
};
|
||||
tts: {
|
||||
provider: 'piper';
|
||||
provider: 'piper' | 'edge_tts' | 'xtts_gpu';
|
||||
model: string;
|
||||
piperPath?: string;
|
||||
modelPath?: string;
|
||||
configPath?: string;
|
||||
voice?: string;
|
||||
};
|
||||
engine?: {
|
||||
port?: number;
|
||||
sttModel?: string;
|
||||
speakerWav?: string;
|
||||
device?: string;
|
||||
};
|
||||
ffmpegPath?: string;
|
||||
maxVoiceDurationSeconds?: number;
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
// AudioWorkletProcessors for the real-time voice call mode.
|
||||
// Kept deliberately dumb: capture does downsampling + chunking + RMS only,
|
||||
// playback does ring-buffer PCM output only. All turn-taking / VAD / WS
|
||||
// logic lives in voice-call.js on the main thread.
|
||||
|
||||
class MicCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.targetRate = 16000;
|
||||
this.srcRate = sampleRate; // AudioWorkletGlobalScope — native context rate
|
||||
this.ratio = this.srcRate / this.targetRate;
|
||||
this.acc = 0; // fractional resample accumulator
|
||||
this.chunk = []; // pending int16 samples for the current outgoing chunk
|
||||
this.chunkSamples = Math.round(this.targetRate * 0.1); // ~100ms per chunk
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || !input[0]) return true;
|
||||
const ch = input[0];
|
||||
|
||||
for (let i = 0; i < ch.length; i++) {
|
||||
this.acc += 1;
|
||||
if (this.acc >= this.ratio) {
|
||||
this.acc -= this.ratio;
|
||||
const s = Math.max(-1, Math.min(1, ch[i]));
|
||||
this.chunk.push(s < 0 ? s * 0x8000 : s * 0x7fff);
|
||||
if (this.chunk.length >= this.chunkSamples) {
|
||||
const int16 = new Int16Array(this.chunk);
|
||||
let sumSq = 0;
|
||||
for (let j = 0; j < int16.length; j++) { const v = int16[j] / 32768; sumSq += v * v; }
|
||||
const rms = Math.sqrt(sumSq / int16.length);
|
||||
this.port.postMessage({ type: 'audio', pcm: int16, rms }, [int16.buffer]);
|
||||
this.chunk = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('mic-capture-processor', MicCaptureProcessor);
|
||||
|
||||
class PcmPlayerProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.queue = []; // array of Float32Array
|
||||
this.qi = 0; // read offset within queue[0]
|
||||
this.port.onmessage = (e) => {
|
||||
const msg = e.data;
|
||||
if (msg.type === 'push') {
|
||||
const int16 = msg.pcm;
|
||||
const f32 = new Float32Array(int16.length);
|
||||
for (let i = 0; i < int16.length; i++) f32[i] = int16[i] / 32768;
|
||||
this.queue.push(f32);
|
||||
} else if (msg.type === 'clear') {
|
||||
this.queue = [];
|
||||
this.qi = 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(_inputs, outputs) {
|
||||
const out = outputs[0][0];
|
||||
if (!out) return true;
|
||||
let oi = 0;
|
||||
while (oi < out.length) {
|
||||
if (this.queue.length === 0) {
|
||||
out[oi++] = 0;
|
||||
continue;
|
||||
}
|
||||
const cur = this.queue[0];
|
||||
out[oi++] = cur[this.qi++];
|
||||
if (this.qi >= cur.length) {
|
||||
this.queue.shift();
|
||||
this.qi = 0;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('pcm-player-processor', PcmPlayerProcessor);
|
||||
@@ -0,0 +1,300 @@
|
||||
// ── Voice: 실시간 연속 대화 모드 ───────────────────────────────────────────
|
||||
// AudioWorklet 캡처(16kHz PCM16) → /ws/voice-realtime → GPU STT/TTS 스트리밍.
|
||||
// VAD(에너지 기반)로 발화 시작/끝을 로컬에서 판단해 stt_start/stop을 보내고,
|
||||
// LLM 응답 SSE 토큰에서 완성되는 문장 단위로 잘라 TTS를 파이프라이닝한다.
|
||||
// barge-in: 재생 중 사용자가 다시 말하면 즉시 재생을 끊고 새 인식을 시작한다.
|
||||
|
||||
(function () {
|
||||
const RMS_SPEECH_ON = 0.020;
|
||||
const RMS_SPEECH_OFF = 0.012;
|
||||
const SPEECH_CONFIRM_CHUNKS = 2; // ~200ms of sustained energy to confirm speech start
|
||||
const SILENCE_HANGOVER_MS = 700; // silence before we consider the utterance finished
|
||||
const CHUNK_MS = 100;
|
||||
const PREROLL_CHUNKS = 3; // ~300ms of lead-in kept before speech is confirmed
|
||||
// XTTS's streaming inference_stream() can crash (CUDA device-side assert,
|
||||
// poisons the whole GPU context) on very short standalone text — merge short
|
||||
// sentence fragments together before sending them off as one TTS request.
|
||||
const MIN_TTS_CHARS = 15;
|
||||
|
||||
let ws = null;
|
||||
let captureCtx = null, playCtx = null;
|
||||
let micStream = null;
|
||||
let captureNode = null, playerNode = null;
|
||||
let state = 'idle'; // idle | listening | user_speaking | processing | assistant_speaking
|
||||
let speechChunks = 0;
|
||||
let silenceMs = 0;
|
||||
let preRoll = [];
|
||||
let ttsQueue = [];
|
||||
let ttsBusy = false;
|
||||
let currentTtsId = null;
|
||||
let spokenUpTo = 0;
|
||||
let pendingTts = '';
|
||||
let active = false;
|
||||
|
||||
function setStatus(text) {
|
||||
const el = document.getElementById('voice-call-status');
|
||||
if (el) el.textContent = text;
|
||||
}
|
||||
function setCaption(text) {
|
||||
const el = document.getElementById('voice-call-caption');
|
||||
if (el) el.textContent = text || '';
|
||||
}
|
||||
|
||||
window.toggleVoiceCall = function () {
|
||||
if (active) voiceCallHangup();
|
||||
else startVoiceCall();
|
||||
};
|
||||
|
||||
window.voiceCallHangup = function () {
|
||||
active = false;
|
||||
state = 'idle';
|
||||
try { ws && ws.close(); } catch {}
|
||||
ws = null;
|
||||
try { micStream && micStream.getTracks().forEach(t => t.stop()); } catch {}
|
||||
try { captureCtx && captureCtx.close(); } catch {}
|
||||
try { playCtx && playCtx.close(); } catch {}
|
||||
captureCtx = playCtx = micStream = captureNode = playerNode = null;
|
||||
ttsQueue = []; ttsBusy = false; currentTtsId = null;
|
||||
const bar = document.getElementById('voice-call-bar');
|
||||
if (bar) bar.style.display = 'none';
|
||||
const btn = document.getElementById('voice-call-btn');
|
||||
if (btn) { btn.textContent = '📞'; btn.classList.remove('recording'); }
|
||||
};
|
||||
|
||||
async function startVoiceCall() {
|
||||
const btn = document.getElementById('voice-call-btn');
|
||||
const bar = document.getElementById('voice-call-bar');
|
||||
try {
|
||||
micStream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
|
||||
} catch (e) {
|
||||
alert('마이크 권한이 필요합니다: ' + e.message);
|
||||
return;
|
||||
}
|
||||
|
||||
active = true;
|
||||
if (btn) { btn.textContent = '📵'; btn.classList.add('recording'); }
|
||||
if (bar) bar.style.display = 'flex';
|
||||
setStatus('통화 연결 중…');
|
||||
setCaption('');
|
||||
state = 'listening';
|
||||
speechChunks = 0; silenceMs = 0; preRoll = [];
|
||||
spokenUpTo = 0; pendingTts = ''; ttsQueue = []; ttsBusy = false; currentTtsId = null;
|
||||
|
||||
captureCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
await captureCtx.audioWorklet.addModule('voice-call-worklets.js');
|
||||
const source = captureCtx.createMediaStreamSource(micStream);
|
||||
captureNode = new AudioWorkletNode(captureCtx, 'mic-capture-processor');
|
||||
captureNode.port.onmessage = handleMicChunk;
|
||||
source.connect(captureNode);
|
||||
|
||||
playCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24000 });
|
||||
await playCtx.audioWorklet.addModule('voice-call-worklets.js');
|
||||
playerNode = new AudioWorkletNode(playCtx, 'pcm-player-processor');
|
||||
playerNode.connect(playCtx.destination);
|
||||
|
||||
const proto = location.protocol === 'https:' ? 'wss' : 'ws';
|
||||
const token = getAuthToken();
|
||||
const qs = token ? `?token=${encodeURIComponent(token)}` : '';
|
||||
ws = new WebSocket(`${proto}://${API.replace(/^https?:\/\//, '')}/ws/voice-realtime${qs}`);
|
||||
ws.binaryType = 'arraybuffer';
|
||||
ws.onopen = () => setStatus('듣는 중…');
|
||||
ws.onmessage = handleWsMessage;
|
||||
ws.onerror = () => setStatus('연결 오류');
|
||||
ws.onclose = () => { if (active) voiceCallHangup(); };
|
||||
}
|
||||
|
||||
function handleMicChunk(e) {
|
||||
if (!active) return;
|
||||
const { pcm, rms } = e.data;
|
||||
|
||||
if (state === 'listening' || state === 'assistant_speaking') {
|
||||
preRoll.push(pcm);
|
||||
if (preRoll.length > PREROLL_CHUNKS) preRoll.shift();
|
||||
if (rms > RMS_SPEECH_ON) {
|
||||
speechChunks++;
|
||||
if (speechChunks >= SPEECH_CONFIRM_CHUNKS) onSpeechStart();
|
||||
} else {
|
||||
speechChunks = 0;
|
||||
}
|
||||
} else if (state === 'user_speaking') {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(pcm.buffer);
|
||||
if (rms < RMS_SPEECH_OFF) {
|
||||
silenceMs += CHUNK_MS;
|
||||
if (silenceMs >= SILENCE_HANGOVER_MS) onSpeechEnd();
|
||||
} else {
|
||||
silenceMs = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function onSpeechStart() {
|
||||
if (state === 'assistant_speaking') {
|
||||
// barge-in: cut playback immediately and cancel whatever is still generating
|
||||
ttsQueue = [];
|
||||
if (playerNode) playerNode.port.postMessage({ type: 'clear' });
|
||||
if (ws && ws.readyState === WebSocket.OPEN && currentTtsId) {
|
||||
ws.send(JSON.stringify({ type: 'tts_cancel', id: currentTtsId }));
|
||||
}
|
||||
ttsBusy = false;
|
||||
currentTtsId = null;
|
||||
}
|
||||
state = 'user_speaking';
|
||||
silenceMs = 0;
|
||||
setStatus('듣는 중…');
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ type: 'stt_start', language: 'ko' }));
|
||||
for (const chunk of preRoll) ws.send(chunk.buffer);
|
||||
}
|
||||
preRoll = [];
|
||||
}
|
||||
|
||||
function onSpeechEnd() {
|
||||
state = 'processing';
|
||||
speechChunks = 0;
|
||||
setStatus('생각 중…');
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ type: 'stt_stop', id: 'turn-' + Date.now() }));
|
||||
}
|
||||
}
|
||||
|
||||
function handleWsMessage(e) {
|
||||
if (e.data instanceof ArrayBuffer) {
|
||||
if (playerNode) playerNode.port.postMessage({ type: 'push', pcm: new Int16Array(e.data) }, [e.data]);
|
||||
return;
|
||||
}
|
||||
let msg;
|
||||
try { msg = JSON.parse(e.data); } catch { return; }
|
||||
|
||||
switch (msg.type) {
|
||||
case 'stt_partial':
|
||||
setCaption(msg.text || '');
|
||||
break;
|
||||
|
||||
case 'stt_final': {
|
||||
setCaption('');
|
||||
const text = (msg.text || '').trim();
|
||||
if (!text) { state = 'listening'; setStatus('듣는 중…'); break; }
|
||||
spokenUpTo = 0; pendingTts = '';
|
||||
const input = document.getElementById('chat-input');
|
||||
if (input) {
|
||||
input.value = text;
|
||||
input.dispatchEvent(new Event('input'));
|
||||
}
|
||||
setStatus('생각 중…');
|
||||
setTimeout(() => (window._appSendFn || handleSendStop)(), 30);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'tts_stream_start':
|
||||
currentTtsId = msg.id;
|
||||
state = 'assistant_speaking';
|
||||
setStatus('말하는 중…');
|
||||
break;
|
||||
|
||||
case 'tts_end':
|
||||
ttsBusy = false;
|
||||
currentTtsId = null;
|
||||
pumpTtsQueue();
|
||||
if (ttsQueue.length === 0) {
|
||||
state = 'listening';
|
||||
setStatus('듣는 중…');
|
||||
}
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
console.warn('[voice-call] engine error:', msg.message);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function pumpTtsQueue() {
|
||||
if (ttsBusy || ttsQueue.length === 0) return;
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
||||
const text = ttsQueue.shift();
|
||||
ttsBusy = true;
|
||||
const id = 'tts-' + Date.now() + '-' + Math.random().toString(36).slice(2, 7);
|
||||
currentTtsId = id;
|
||||
ws.send(JSON.stringify({ type: 'tts_start', text, id }));
|
||||
}
|
||||
|
||||
// Strips markdown syntax, tables, URLs and emoji so raw LLM output
|
||||
// (**bold**, bullet markers, code blocks, | table | pipes |, links, 🦞
|
||||
// etc.) doesn't get force-pronounced by XTTS under language='ko' — those
|
||||
// symbols were producing garbled non-Korean-sounding artifacts mid-sentence
|
||||
// (tables were the worst offender: pipes + dash separator rows + raw URLs).
|
||||
function sanitizeForSpeech(text) {
|
||||
return text
|
||||
.replace(/```[\s\S]*?```/g, ' ')
|
||||
.replace(/`([^`]+)`/g, '$1')
|
||||
.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
||||
// markdown table separator rows, e.g. "|---|:--:|---|"
|
||||
.replace(/^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$/gm, ' ')
|
||||
// remaining table rows: "| a | b |" -> "a, b" so cells read as a list
|
||||
.replace(/^\s*\|(.+)\|\s*$/gm, (_m, row) => row.split('|').map((c) => c.trim()).filter(Boolean).join(', '))
|
||||
.replace(/https?:\/\/\S+/g, ' ')
|
||||
.replace(/^\s{0,3}#{1,6}\s+/gm, '')
|
||||
.replace(/^\s*[-*+]\s+/gm, '')
|
||||
.replace(/^\s*>\s?/gm, '')
|
||||
.replace(/\*\*([^*]+)\*\*/g, '$1')
|
||||
.replace(/\*([^*]+)\*/g, '$1')
|
||||
.replace(/__([^_]+)__/g, '$1')
|
||||
.replace(/_([^_]+)_/g, '$1')
|
||||
.replace(/[\u{1F300}-\u{1FAFF}\u{2600}-\u{27BF}\u{1F1E6}-\u{1F1FF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}]/gu, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function enqueueSentence(text) {
|
||||
text = sanitizeForSpeech((text || '').trim());
|
||||
if (!text) return;
|
||||
ttsQueue.push(text);
|
||||
pumpTtsQueue();
|
||||
}
|
||||
|
||||
// Accumulates sentence fragments until there's enough text to be safe to
|
||||
// stream to XTTS, then enqueues the merged chunk.
|
||||
function bufferSentence(sentence) {
|
||||
pendingTts = pendingTts ? pendingTts + ' ' + sentence : sentence;
|
||||
if (pendingTts.replace(/[^\p{L}\p{N}]/gu, '').length >= MIN_TTS_CHARS) {
|
||||
enqueueSentence(pendingTts);
|
||||
pendingTts = '';
|
||||
}
|
||||
}
|
||||
|
||||
function splitCompleteSentences(text) {
|
||||
const sentences = [];
|
||||
let start = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (/[.!?\n]/.test(text[i])) {
|
||||
let j = i + 1;
|
||||
while (j < text.length && /[\s.!?]/.test(text[j])) j++;
|
||||
const piece = text.slice(start, j).trim();
|
||||
if (piece) sentences.push(piece);
|
||||
start = j;
|
||||
i = j - 1;
|
||||
}
|
||||
}
|
||||
return { sentences, consumedLength: start };
|
||||
}
|
||||
|
||||
// Hooked from app.js's SSE token handler — only acts while a call is active.
|
||||
window._voiceCallOnToken = function (fullPartialContent) {
|
||||
if (!active) return;
|
||||
const tail = fullPartialContent.slice(spokenUpTo);
|
||||
const { sentences, consumedLength } = splitCompleteSentences(tail);
|
||||
if (consumedLength > 0) {
|
||||
spokenUpTo += consumedLength;
|
||||
for (const s of sentences) bufferSentence(s);
|
||||
}
|
||||
};
|
||||
|
||||
// Hooked from app.js when the SSE stream for a turn finishes.
|
||||
window._voiceCallOnTurnDone = function (finalText) {
|
||||
if (!active) return;
|
||||
const remaining = (finalText || '').slice(spokenUpTo).trim();
|
||||
if (remaining) bufferSentence(remaining);
|
||||
if (pendingTts) { enqueueSentence(pendingTts); pendingTts = ''; }
|
||||
spokenUpTo = 0;
|
||||
};
|
||||
})();
|
||||
Reference in New Issue
Block a user