Add Level 1 push-to-talk voice chat (STT + TTS)
- POST /api/voice/stt: Whisper-cpp transcription of webm/ogg audio - POST /api/voice/tts: Piper Korean TTS returning ogg audio - Web UI: 🎤 mic button in chat input; click to start/stop recording MediaRecorder → /api/voice/stt → auto-fills chat input - Web UI: 🔊 TTS button on each AI message; click to play/stop - config.json: add absolute model/binary paths for whisper-cli and piper Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -214,15 +214,19 @@
|
||||
"path": "/hooks"
|
||||
},
|
||||
"voice": {
|
||||
"enabled": false,
|
||||
"enabled": true,
|
||||
"ffmpegPath": "/usr/bin/ffmpeg",
|
||||
"stt": {
|
||||
"provider": "whisper_cpp",
|
||||
"model": "ggml-base.bin",
|
||||
"whisperPath": "/usr/local/bin/whisper-cli",
|
||||
"modelPath": "/usr/local/share/whisper-models/ggml-base.bin",
|
||||
"language": "ko"
|
||||
},
|
||||
"tts": {
|
||||
"provider": "piper",
|
||||
"model": "ko_KR-kss-medium"
|
||||
"piperPath": "/usr/local/bin/piper",
|
||||
"modelPath": "/usr/local/share/piper-voices/ko_KR-kss-medium.onnx",
|
||||
"configPath": "/usr/local/share/piper-voices/ko_KR-kss-medium.onnx.json"
|
||||
}
|
||||
},
|
||||
"search": {
|
||||
|
||||
@@ -144,6 +144,8 @@ import {
|
||||
clearFileOpCheckpoint,
|
||||
} from '../orchestration/file-op-v2';
|
||||
import { OllamaProcessManager } from './ollama-process-manager';
|
||||
import { transcribeAudio } from '../tools/stt.js';
|
||||
import { synthesizeSpeech, isTTSAvailable } from '../tools/tts.js';
|
||||
import { raceWithWatchdog, PreemptState } from './preempt-watchdog';
|
||||
import { detectGpu, logGpuStatus } from './gpu-detector';
|
||||
import { internalAgentTaskRouter } from './internal-agent-task';
|
||||
@@ -8718,6 +8720,42 @@ app.put('/api/files/{*filePath}', (req: express.Request, res: express.Response)
|
||||
}
|
||||
});
|
||||
|
||||
// ── Voice: STT (Whisper) ──────────────────────────────────────────────────
|
||||
app.post('/api/voice/stt', async (req: express.Request, res: express.Response) => {
|
||||
const contentType = String(req.headers['content-type'] || 'audio/webm');
|
||||
const fmt = contentType.includes('ogg') ? 'ogg'
|
||||
: contentType.includes('wav') ? 'wav'
|
||||
: contentType.includes('mp4') || contentType.includes('m4a') ? 'm4a'
|
||||
: 'webm';
|
||||
const chunks: Buffer[] = [];
|
||||
req.on('data', (c: Buffer) => chunks.push(c));
|
||||
req.on('end', async () => {
|
||||
const buf = Buffer.concat(chunks);
|
||||
if (!buf.length) { res.status(400).json({ success: false, error: 'No audio data' }); return; }
|
||||
try {
|
||||
const result = await transcribeAudio(buf, fmt);
|
||||
res.json({ success: true, text: result.text.trim(), language: result.language });
|
||||
} catch (e: any) {
|
||||
res.status(500).json({ success: false, error: String(e?.message || e) });
|
||||
}
|
||||
});
|
||||
req.on('error', (e: any) => res.status(500).json({ success: false, error: String(e?.message || e) }));
|
||||
});
|
||||
|
||||
// ── Voice: TTS (Piper) ────────────────────────────────────────────────────
|
||||
app.post('/api/voice/tts', async (req: express.Request, res: express.Response) => {
|
||||
const { text } = req.body || {};
|
||||
if (!text || typeof text !== 'string') { res.status(400).json({ success: false, error: 'text required' }); return; }
|
||||
if (!isTTSAvailable()) { res.status(503).json({ success: false, error: 'TTS not configured' }); return; }
|
||||
try {
|
||||
const audio = await synthesizeSpeech(text.slice(0, 2000));
|
||||
res.set('Content-Type', 'audio/ogg');
|
||||
res.send(audio);
|
||||
} catch (e: any) {
|
||||
res.status(500).json({ success: false, error: String(e?.message || e) });
|
||||
}
|
||||
});
|
||||
|
||||
// ── Canvas code execution ──────────────────────────────────────────────────
|
||||
const CANVAS_EXEC_EXT: Record<string, string> = {
|
||||
'.py': 'python3', '.js': 'node', '.ts': 'npx tsx', '.mjs': 'node',
|
||||
|
||||
+128
-1
@@ -1021,6 +1021,10 @@
|
||||
|
||||
.btn { padding: 10px 16px; font-size: 12px; }
|
||||
.btn-sm { padding: 6px 10px; font-size: 11px; border-radius: 10px; }
|
||||
#mic-btn.recording { background: #ef4444 !important; animation: mic-pulse 1s ease-in-out infinite; }
|
||||
@keyframes mic-pulse { 0%,100%{opacity:1} 50%{opacity:0.55} }
|
||||
.tts-btn { background:none;border:none;cursor:pointer;font-size:13px;padding:1px 4px;border-radius:4px;opacity:0.45;vertical-align:middle;line-height:1; }
|
||||
.tts-btn:hover,.tts-btn.playing{opacity:1;background:var(--panel-2);}
|
||||
.export-dropdown { position:relative; display:inline-block; }
|
||||
.export-menu { display:none; position:absolute; right:0; top:calc(100% + 4px); background:var(--panel-2); border:1px solid var(--border); border-radius:8px; box-shadow:0 4px 16px rgba(0,0,0,0.18); z-index:2000; min-width:160px; overflow:hidden; }
|
||||
.export-menu button { display:block; width:100%; padding:8px 14px; text-align:left; background:none; border:none; cursor:pointer; font-size:12px; color:var(--text); white-space:nowrap; }
|
||||
@@ -2420,6 +2424,7 @@
|
||||
<div class="chat-input-row">
|
||||
<input type="file" id="file-upload" accept="image/*,.pdf,.txt,.md,.csv,.xlsx,.xls,.docx,.pptx" style="display:none" onchange="handleFileUpload(this)" />
|
||||
<button class="send-btn" onclick="document.getElementById('file-upload').click()" title="파일 첨부 (이미지·PDF·Excel·txt 등)" style="font-size:18px;width:44px;height:44px" aria-label="파일 첨부">📎</button>
|
||||
<button class="send-btn" id="mic-btn" onclick="toggleMicRecording()" title="음성 입력 (클릭: 시작/중지)" style="font-size:18px;width:44px;height:44px" aria-label="음성 입력">🎤</button>
|
||||
<textarea
|
||||
id="chat-input"
|
||||
placeholder="메시지를 입력하세요... (Enter로 전송, Shift+Enter로 줄바꿈)"
|
||||
@@ -4745,7 +4750,7 @@ function renderChatMessages() {
|
||||
<div class="msg ${msg.role}${pinClass}${selectedClass}${confirmedClass}"${clickHandler}>
|
||||
<div class="msg-avatar">${msg.role === 'user' ? 'U' : '<img src="cherry_logo.png" style="width:48px;height:48px;border-radius:6px">'}</div>
|
||||
<div class="msg-body">
|
||||
<div class="msg-role">${msg.role === 'user' ? 'You' : (msg.mode === 'agentic' ? 'CherryClaw (Agent)' : 'CherryClaw')}</div>
|
||||
<div class="msg-role">${msg.role === 'user' ? 'You' : (msg.mode === 'agentic' ? 'CherryClaw (Agent)' : 'CherryClaw')}${msg.role === 'ai' ? ` <button class="tts-btn" onclick="playTTSMsg(this,${idx})" title="음성으로 듣기">🔊</button>` : ''}</div>
|
||||
${msg.role === 'ai' ? renderAssistantContent(msg.content) : `<div class="msg-content">${renderMarkdown(msg.display !== undefined ? msg.display : msg.content, { hideDownload: true })}</div>`}
|
||||
${msg.role === 'ai' ? renderArtifacts(msg.artifacts) : ''}
|
||||
${msg.role === 'ai' ? renderFileDownloads(msg.steps, msg.processEntries, msg.content) : ''}
|
||||
@@ -8827,6 +8832,128 @@ function initLuckyModalInteraction() {
|
||||
});
|
||||
}
|
||||
|
||||
// ── Voice: STT (마이크 입력) ─────────────────────────────────────────────
|
||||
var _mediaRecorder = null;
|
||||
var _audioChunks = [];
|
||||
var _isRecording = false;
|
||||
var _micMimeType = '';
|
||||
|
||||
function toggleMicRecording() {
|
||||
if (_isRecording) stopMicRecording();
|
||||
else startMicRecording();
|
||||
}
|
||||
|
||||
function startMicRecording() {
|
||||
navigator.mediaDevices.getUserMedia({ audio: true, video: false })
|
||||
.then(function(stream) {
|
||||
_audioChunks = [];
|
||||
_micMimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') ? 'audio/webm;codecs=opus'
|
||||
: MediaRecorder.isTypeSupported('audio/webm') ? 'audio/webm'
|
||||
: MediaRecorder.isTypeSupported('audio/ogg;codecs=opus') ? 'audio/ogg;codecs=opus'
|
||||
: '';
|
||||
var opts = _micMimeType ? { mimeType: _micMimeType } : {};
|
||||
_mediaRecorder = new MediaRecorder(stream, opts);
|
||||
_mediaRecorder.ondataavailable = function(e) { if (e.data.size > 0) _audioChunks.push(e.data); };
|
||||
_mediaRecorder.onstop = function() {
|
||||
stream.getTracks().forEach(function(t) { t.stop(); });
|
||||
var blob = new Blob(_audioChunks, { type: _micMimeType || 'audio/webm' });
|
||||
sendToSTT(blob);
|
||||
};
|
||||
_mediaRecorder.start();
|
||||
_isRecording = true;
|
||||
var btn = document.getElementById('mic-btn');
|
||||
if (btn) { btn.textContent = '⏹'; btn.title = '녹음 중지 (클릭)'; btn.classList.add('recording'); }
|
||||
})
|
||||
.catch(function(e) { alert('마이크 접근 실패: ' + e.message); });
|
||||
}
|
||||
|
||||
function stopMicRecording() {
|
||||
if (_mediaRecorder && _isRecording) {
|
||||
_mediaRecorder.stop();
|
||||
_isRecording = false;
|
||||
var btn = document.getElementById('mic-btn');
|
||||
if (btn) { btn.textContent = '⌛'; btn.title = '인식 중…'; btn.classList.remove('recording'); }
|
||||
}
|
||||
}
|
||||
|
||||
function sendToSTT(blob) {
|
||||
var ct = _micMimeType || 'audio/webm';
|
||||
fetch('/api/voice/stt', {
|
||||
method: 'POST',
|
||||
headers: Object.assign({ 'Content-Type': ct }, authHeaders()),
|
||||
credentials: 'include',
|
||||
body: blob,
|
||||
})
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(data) {
|
||||
var btn = document.getElementById('mic-btn');
|
||||
if (btn) { btn.textContent = '🎤'; btn.title = '음성 입력 (클릭: 시작/중지)'; }
|
||||
if (data.success && data.text) {
|
||||
var input = document.getElementById('chat-input');
|
||||
if (input) {
|
||||
input.value = (input.value ? input.value + ' ' : '') + data.text;
|
||||
input.focus();
|
||||
input.dispatchEvent(new Event('input'));
|
||||
}
|
||||
} else if (!data.success) {
|
||||
alert('음성 인식 실패: ' + (data.error || '알 수 없는 오류'));
|
||||
}
|
||||
})
|
||||
.catch(function(e) {
|
||||
var btn = document.getElementById('mic-btn');
|
||||
if (btn) { btn.textContent = '🎤'; btn.title = '음성 입력 (클릭: 시작/중지)'; }
|
||||
alert('STT 오류: ' + e.message);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Voice: TTS (음성 출력) ───────────────────────────────────────────────
|
||||
var _ttsAudio = null;
|
||||
var _ttsBtnActive = null;
|
||||
|
||||
function playTTSMsg(btn, idx) {
|
||||
var msg = chatHistory[idx];
|
||||
if (!msg) return;
|
||||
var text = (msg.content || '').replace(/!\[.*?\]\(.*?\)/g, '').replace(/\[([^\]]+)\]\([^)]+\)/g, '$1').replace(/```[\s\S]*?```/g, '').replace(/[#*`_~>]/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
if (!text) return;
|
||||
// Toggle off if same button is playing
|
||||
if (_ttsBtnActive === btn && _ttsAudio) {
|
||||
_ttsAudio.pause(); _ttsAudio = null;
|
||||
btn.textContent = '🔊'; btn.classList.remove('playing');
|
||||
_ttsBtnActive = null;
|
||||
return;
|
||||
}
|
||||
// Stop previous
|
||||
if (_ttsAudio) { _ttsAudio.pause(); _ttsAudio = null; }
|
||||
if (_ttsBtnActive) { _ttsBtnActive.textContent = '🔊'; _ttsBtnActive.classList.remove('playing'); }
|
||||
btn.textContent = '⌛'; btn.classList.add('playing');
|
||||
_ttsBtnActive = btn;
|
||||
fetch('/api/voice/tts', {
|
||||
method: 'POST',
|
||||
headers: Object.assign({ 'Content-Type': 'application/json' }, authHeaders()),
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({ text: text.slice(0, 2000) }),
|
||||
})
|
||||
.then(function(r) { if (!r.ok) throw new Error('TTS 오류 ' + r.status); return r.blob(); })
|
||||
.then(function(blob) {
|
||||
var url = URL.createObjectURL(blob);
|
||||
_ttsAudio = new Audio(url);
|
||||
_ttsAudio.onended = function() {
|
||||
URL.revokeObjectURL(url); _ttsAudio = null;
|
||||
btn.textContent = '🔊'; btn.classList.remove('playing'); _ttsBtnActive = null;
|
||||
};
|
||||
_ttsAudio.onerror = function() {
|
||||
URL.revokeObjectURL(url); _ttsAudio = null;
|
||||
btn.textContent = '🔊'; btn.classList.remove('playing'); _ttsBtnActive = null;
|
||||
};
|
||||
btn.textContent = '⏹';
|
||||
_ttsAudio.play();
|
||||
})
|
||||
.catch(function(e) {
|
||||
btn.textContent = '🔊'; btn.classList.remove('playing'); _ttsBtnActive = null;
|
||||
alert('TTS 오류: ' + e.message);
|
||||
});
|
||||
}
|
||||
|
||||
function saveLuckyModal() {
|
||||
var url = _luckyCurrentSaveUrl;
|
||||
if (!url) { alert('저장할 파일 경로가 없습니다.'); return; }
|
||||
|
||||
Reference in New Issue
Block a user