diff --git a/package.json b/package.json index 7a23076..e5186d3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "smallclaw", - "version": "3.4.0", + "version": "3.5.0", "description": "Local AI agent framework powered by Ollama - OpenClaw alternative", "main": "dist/index.js", "bin": { diff --git a/src/tools/voice_engine.py b/src/tools/voice_engine.py index c0816d1..9bd1b12 100644 --- a/src/tools/voice_engine.py +++ b/src/tools/voice_engine.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Persistent GPU voice engine: loads faster-whisper (STT) and XTTS-v2 (TTS) +Persistent GPU voice engine: loads faster-whisper (STT) and OmniVoice (TTS) once and serves them over a localhost WebSocket so Node never pays model load latency per request. @@ -56,10 +56,10 @@ MIN_PARTIAL_BYTES = int(16000 * 2 * 0.6) # ~0.6s of 16kHz mono PCM16 before we def is_unrecoverable_cuda_error(e: Exception) -> bool: - # A CUDA "device-side assert" (seen from XTTS inference_stream on very short - # text) poisons the whole CUDA context for the rest of the process — every - # GPU call after it fails identically, whisper included. There is no - # in-process recovery; the only fix is a fresh CUDA context, i.e. a restart. + # A CUDA "device-side assert" poisons the whole CUDA context for the rest of + # the process — every GPU call after it fails identically, whisper included. + # There is no in-process recovery; the only fix is a fresh CUDA context, i.e. + # a restart. msg = str(e) return 'CUDA' in msg and ('assert' in msg.lower() or 'device-side' in msg.lower() or 'failed' in msg.lower()) @@ -75,15 +75,21 @@ def load_models(args): stt_model = WhisperModel(args.stt_model, device=args.device, compute_type='float16' if args.device == 'cuda' else 'int8') log.info('faster-whisper ready.') - log.info('Loading XTTS-v2 ...') - from TTS.api import TTS - tts_wrapper = TTS('tts_models/multilingual/multi-dataset/xtts_v2').to(args.device) - xtts = tts_wrapper.synthesizer.tts_model # underlying TTS.tts.models.xtts.Xtts — has inference_stream() - log.info('Computing speaker conditioning latents from %s ...', args.speaker_wav) - gpt_cond_latent, speaker_embedding = xtts.get_conditioning_latents(audio_path=[args.speaker_wav]) - log.info('XTTS-v2 ready.') + log.info('Transcribing speaker reference %s for the voice-clone prompt text ...', args.speaker_wav) + segments, _info = stt_model.transcribe(args.speaker_wav, language='ko', beam_size=5, vad_filter=True) + ref_text = ''.join(seg.text for seg in segments).strip() + log.info('Speaker reference text: %r', ref_text) - return stt_model, tts_wrapper, xtts, gpt_cond_latent, speaker_embedding + log.info('Loading OmniVoice (%s) ...', args.tts_model) + import torch + from omnivoice import OmniVoice + tts_model = OmniVoice.from_pretrained(args.tts_model, device_map=args.device, dtype=torch.float16, load_asr=False) + voice_clone_prompt = tts_model.create_voice_clone_prompt(ref_audio=args.speaker_wav, ref_text=ref_text) + log.info('Warming up OmniVoice ...') + tts_model.generate(text='안녕하세요.', language='Korean', voice_clone_prompt=voice_clone_prompt) + log.info('OmniVoice ready (sample_rate=%d).', tts_model.sampling_rate) + + return stt_model, tts_model, voice_clone_prompt, tts_model.sampling_rate def pcm16_from_float(audio: np.ndarray) -> bytes: @@ -99,13 +105,12 @@ class Engine: def __init__(self, args): self.args = args self.stt_model = None - self.tts_model = None # TTS.api.TTS wrapper (used for the batch .tts() call) - self.xtts = None # underlying Xtts model (used for inference_stream) - self.gpt_cond_latent = None - self.speaker_embedding = None + self.tts_model = None # omnivoice.OmniVoice + self.voice_clone_prompt = None # precomputed — avoids re-encoding the speaker ref every call + self._tts_sample_rate = 24000 def ready(self) -> bool: - return self.stt_model is not None and self.xtts is not None + return self.stt_model is not None and self.tts_model is not None # ── Batch STT/TTS (Telegram, single-shot web button) ────────────────── def transcribe_full(self, audio_bytes: bytes, fmt: str, language: str): @@ -124,15 +129,13 @@ class Engine: pass def synthesize_full(self, text: str): - wav = self.tts_model.tts( + audio = self.tts_model.generate( text=text[:4000], - speaker_wav=self.args.speaker_wav, - language='ko', + language='Korean', + voice_clone_prompt=self.voice_clone_prompt, ) - audio = np.asarray(wav, dtype=np.float32) - pcm = pcm16_from_float(audio) - sample_rate = int(self.tts_model.synthesizer.output_sample_rate) - return pcm, sample_rate + pcm = pcm16_from_float(np.asarray(audio[0], dtype=np.float32)) + return pcm, self._tts_sample_rate # ── Streaming STT (partial decode of an accumulating PCM buffer) ────── def transcribe_pcm(self, pcm: bytes, language: str): @@ -141,23 +144,23 @@ class Engine: text = ''.join(seg.text for seg in segments).strip() return text, (info.language if info else language) - # ── Streaming TTS (XTTS inference_stream, chunk-by-chunk) ───────────── + # ── "Streaming" TTS ───────────────────────────────────────────────── + # OmniVoice's public API is a batch call (no token-level generator), so a + # "stream" here is one full synthesis emitted as a single chunk. At ~2x + # real-time it's fast enough that per-sentence batching (the client already + # buffers LLM output into sentence-sized pieces before calling tts_start) + # doesn't feel laggy the way XTTS/Qwen3-TTS did. def synthesize_stream(self, text: str, is_cancelled): - for chunk in self.xtts.inference_stream( - text=text[:4000], - language='ko', - gpt_cond_latent=self.gpt_cond_latent, - speaker_embedding=self.speaker_embedding, - stream_chunk_size=20, - ): - if is_cancelled(): - return - audio = chunk.squeeze().detach().cpu().numpy().astype(np.float32) - yield pcm16_from_float(audio) + if is_cancelled() or not text.strip(): + return + pcm, _sample_rate = self.synthesize_full(text) + if is_cancelled(): + return + yield pcm @property def tts_sample_rate(self) -> int: - return int(self.tts_model.synthesizer.output_sample_rate) + return self._tts_sample_rate class ConnectionState: @@ -300,13 +303,14 @@ async def main(): parser = argparse.ArgumentParser() parser.add_argument('--port', type=int, default=8765) parser.add_argument('--stt-model', default='medium') + parser.add_argument('--tts-model', default='k2-fsa/OmniVoice') parser.add_argument('--speaker-wav', default='') parser.add_argument('--device', default='cuda') args = parser.parse_args() engine = Engine(args) - (engine.stt_model, engine.tts_model, engine.xtts, - engine.gpt_cond_latent, engine.speaker_embedding) = load_models(args) + (engine.stt_model, engine.tts_model, engine.voice_clone_prompt, + engine._tts_sample_rate) = load_models(args) async with websockets.serve(lambda ws: handle_connection(ws, engine), '127.0.0.1', args.port, max_size=64 * 1024 * 1024): log.info('voice_engine listening on 127.0.0.1:%d', args.port) diff --git a/web-ui/voice-call.js b/web-ui/voice-call.js index 8f87990..6be83aa 100644 --- a/web-ui/voice-call.js +++ b/web-ui/voice-call.js @@ -30,6 +30,23 @@ let spokenUpTo = 0; let pendingTts = ''; let active = false; + let wakeLock = null; + + // The Wake Lock is auto-released whenever the tab loses visibility (screen off, + // app-switch) — without it, mobile browsers throttle/suspend the background tab + // and the call silently drops. Re-acquire it once the tab is visible again. + async function acquireWakeLock() { + if (!('wakeLock' in navigator)) return; + try { + wakeLock = await navigator.wakeLock.request('screen'); + wakeLock.addEventListener('release', () => { wakeLock = null; }); + } catch (e) { + console.warn('[voice-call] wakeLock request failed:', e.message); + } + } + document.addEventListener('visibilitychange', () => { + if (active && wakeLock === null && document.visibilityState === 'visible') acquireWakeLock(); + }); function setStatus(text) { const el = document.getElementById('voice-call-status'); @@ -54,6 +71,8 @@ try { captureCtx && captureCtx.close(); } catch {} try { playCtx && playCtx.close(); } catch {} captureCtx = playCtx = micStream = captureNode = playerNode = null; + try { wakeLock && wakeLock.release(); } catch {} + wakeLock = null; ttsQueue = []; ttsBusy = false; currentTtsId = null; const bar = document.getElementById('voice-call-bar'); if (bar) bar.style.display = 'none'; @@ -72,6 +91,7 @@ } active = true; + acquireWakeLock(); if (btn) { btn.textContent = '📵'; btn.classList.add('recording'); } if (bar) bar.style.display = 'flex'; setStatus('통화 연결 중…');