v3.5.0: 실시간 음성통화 TTS 엔진을 OmniVoice로 교체 + Wake Lock 추가
XTTS는 한영 혼용 발음이 깨지고(Zoom→쭘 등) 짧은 문장에서 rambling이 발생했음. Qwen3-TTS로 교체해봤으나 realtime factor 0.3x로 통화에 쓰기엔 너무 느려서 폐기. 최종적으로 OmniVoice(k2-fsa)로 교체 — realtime factor 1.8~2.8x로 XTTS보다도 빠르고 한영 혼용 발음도 자연스러움. voice-call.js에는 navigator.wakeLock 추가 — 폰 화면이 꺼지면 브라우저가 백그라운드 탭을 스로틀링해서 통화가 끊기는 문제를 방지. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
+44
-40
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Persistent GPU voice engine: loads faster-whisper (STT) and XTTS-v2 (TTS)
|
||||
Persistent GPU voice engine: loads faster-whisper (STT) and OmniVoice (TTS)
|
||||
once and serves them over a localhost WebSocket so Node never pays model
|
||||
load latency per request.
|
||||
|
||||
@@ -56,10 +56,10 @@ MIN_PARTIAL_BYTES = int(16000 * 2 * 0.6) # ~0.6s of 16kHz mono PCM16 before we
|
||||
|
||||
|
||||
def is_unrecoverable_cuda_error(e: Exception) -> bool:
|
||||
# A CUDA "device-side assert" (seen from XTTS inference_stream on very short
|
||||
# text) poisons the whole CUDA context for the rest of the process — every
|
||||
# GPU call after it fails identically, whisper included. There is no
|
||||
# in-process recovery; the only fix is a fresh CUDA context, i.e. a restart.
|
||||
# A CUDA "device-side assert" poisons the whole CUDA context for the rest of
|
||||
# the process — every GPU call after it fails identically, whisper included.
|
||||
# There is no in-process recovery; the only fix is a fresh CUDA context, i.e.
|
||||
# a restart.
|
||||
msg = str(e)
|
||||
return 'CUDA' in msg and ('assert' in msg.lower() or 'device-side' in msg.lower() or 'failed' in msg.lower())
|
||||
|
||||
@@ -75,15 +75,21 @@ def load_models(args):
|
||||
stt_model = WhisperModel(args.stt_model, device=args.device, compute_type='float16' if args.device == 'cuda' else 'int8')
|
||||
log.info('faster-whisper ready.')
|
||||
|
||||
log.info('Loading XTTS-v2 ...')
|
||||
from TTS.api import TTS
|
||||
tts_wrapper = TTS('tts_models/multilingual/multi-dataset/xtts_v2').to(args.device)
|
||||
xtts = tts_wrapper.synthesizer.tts_model # underlying TTS.tts.models.xtts.Xtts — has inference_stream()
|
||||
log.info('Computing speaker conditioning latents from %s ...', args.speaker_wav)
|
||||
gpt_cond_latent, speaker_embedding = xtts.get_conditioning_latents(audio_path=[args.speaker_wav])
|
||||
log.info('XTTS-v2 ready.')
|
||||
log.info('Transcribing speaker reference %s for the voice-clone prompt text ...', args.speaker_wav)
|
||||
segments, _info = stt_model.transcribe(args.speaker_wav, language='ko', beam_size=5, vad_filter=True)
|
||||
ref_text = ''.join(seg.text for seg in segments).strip()
|
||||
log.info('Speaker reference text: %r', ref_text)
|
||||
|
||||
return stt_model, tts_wrapper, xtts, gpt_cond_latent, speaker_embedding
|
||||
log.info('Loading OmniVoice (%s) ...', args.tts_model)
|
||||
import torch
|
||||
from omnivoice import OmniVoice
|
||||
tts_model = OmniVoice.from_pretrained(args.tts_model, device_map=args.device, dtype=torch.float16, load_asr=False)
|
||||
voice_clone_prompt = tts_model.create_voice_clone_prompt(ref_audio=args.speaker_wav, ref_text=ref_text)
|
||||
log.info('Warming up OmniVoice ...')
|
||||
tts_model.generate(text='안녕하세요.', language='Korean', voice_clone_prompt=voice_clone_prompt)
|
||||
log.info('OmniVoice ready (sample_rate=%d).', tts_model.sampling_rate)
|
||||
|
||||
return stt_model, tts_model, voice_clone_prompt, tts_model.sampling_rate
|
||||
|
||||
|
||||
def pcm16_from_float(audio: np.ndarray) -> bytes:
|
||||
@@ -99,13 +105,12 @@ class Engine:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.stt_model = None
|
||||
self.tts_model = None # TTS.api.TTS wrapper (used for the batch .tts() call)
|
||||
self.xtts = None # underlying Xtts model (used for inference_stream)
|
||||
self.gpt_cond_latent = None
|
||||
self.speaker_embedding = None
|
||||
self.tts_model = None # omnivoice.OmniVoice
|
||||
self.voice_clone_prompt = None # precomputed — avoids re-encoding the speaker ref every call
|
||||
self._tts_sample_rate = 24000
|
||||
|
||||
def ready(self) -> bool:
|
||||
return self.stt_model is not None and self.xtts is not None
|
||||
return self.stt_model is not None and self.tts_model is not None
|
||||
|
||||
# ── Batch STT/TTS (Telegram, single-shot web button) ──────────────────
|
||||
def transcribe_full(self, audio_bytes: bytes, fmt: str, language: str):
|
||||
@@ -124,15 +129,13 @@ class Engine:
|
||||
pass
|
||||
|
||||
def synthesize_full(self, text: str):
|
||||
wav = self.tts_model.tts(
|
||||
audio = self.tts_model.generate(
|
||||
text=text[:4000],
|
||||
speaker_wav=self.args.speaker_wav,
|
||||
language='ko',
|
||||
language='Korean',
|
||||
voice_clone_prompt=self.voice_clone_prompt,
|
||||
)
|
||||
audio = np.asarray(wav, dtype=np.float32)
|
||||
pcm = pcm16_from_float(audio)
|
||||
sample_rate = int(self.tts_model.synthesizer.output_sample_rate)
|
||||
return pcm, sample_rate
|
||||
pcm = pcm16_from_float(np.asarray(audio[0], dtype=np.float32))
|
||||
return pcm, self._tts_sample_rate
|
||||
|
||||
# ── Streaming STT (partial decode of an accumulating PCM buffer) ──────
|
||||
def transcribe_pcm(self, pcm: bytes, language: str):
|
||||
@@ -141,23 +144,23 @@ class Engine:
|
||||
text = ''.join(seg.text for seg in segments).strip()
|
||||
return text, (info.language if info else language)
|
||||
|
||||
# ── Streaming TTS (XTTS inference_stream, chunk-by-chunk) ─────────────
|
||||
# ── "Streaming" TTS ─────────────────────────────────────────────────
|
||||
# OmniVoice's public API is a batch call (no token-level generator), so a
|
||||
# "stream" here is one full synthesis emitted as a single chunk. At ~2x
|
||||
# real-time it's fast enough that per-sentence batching (the client already
|
||||
# buffers LLM output into sentence-sized pieces before calling tts_start)
|
||||
# doesn't feel laggy the way XTTS/Qwen3-TTS did.
|
||||
def synthesize_stream(self, text: str, is_cancelled):
|
||||
for chunk in self.xtts.inference_stream(
|
||||
text=text[:4000],
|
||||
language='ko',
|
||||
gpt_cond_latent=self.gpt_cond_latent,
|
||||
speaker_embedding=self.speaker_embedding,
|
||||
stream_chunk_size=20,
|
||||
):
|
||||
if is_cancelled():
|
||||
return
|
||||
audio = chunk.squeeze().detach().cpu().numpy().astype(np.float32)
|
||||
yield pcm16_from_float(audio)
|
||||
if is_cancelled() or not text.strip():
|
||||
return
|
||||
pcm, _sample_rate = self.synthesize_full(text)
|
||||
if is_cancelled():
|
||||
return
|
||||
yield pcm
|
||||
|
||||
@property
|
||||
def tts_sample_rate(self) -> int:
|
||||
return int(self.tts_model.synthesizer.output_sample_rate)
|
||||
return self._tts_sample_rate
|
||||
|
||||
|
||||
class ConnectionState:
|
||||
@@ -300,13 +303,14 @@ async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--port', type=int, default=8765)
|
||||
parser.add_argument('--stt-model', default='medium')
|
||||
parser.add_argument('--tts-model', default='k2-fsa/OmniVoice')
|
||||
parser.add_argument('--speaker-wav', default='')
|
||||
parser.add_argument('--device', default='cuda')
|
||||
args = parser.parse_args()
|
||||
|
||||
engine = Engine(args)
|
||||
(engine.stt_model, engine.tts_model, engine.xtts,
|
||||
engine.gpt_cond_latent, engine.speaker_embedding) = load_models(args)
|
||||
(engine.stt_model, engine.tts_model, engine.voice_clone_prompt,
|
||||
engine._tts_sample_rate) = load_models(args)
|
||||
|
||||
async with websockets.serve(lambda ws: handle_connection(ws, engine), '127.0.0.1', args.port, max_size=64 * 1024 * 1024):
|
||||
log.info('voice_engine listening on 127.0.0.1:%d', args.port)
|
||||
|
||||
Reference in New Issue
Block a user