v.2.5.0
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { getConfig } from '../config/config.js';
|
||||
import { ToolResult } from '../types.js';
|
||||
import { transcribeAudio } from './stt.js';
|
||||
|
||||
const AUDIO_EXTS = new Set(['.ogg', '.mp3', '.wav', '.flac', '.m4a', '.wma', '.opus']);
|
||||
|
||||
export const audioTranscribeTool = {
|
||||
name: 'audio_transcribe',
|
||||
description: 'Transcribe an audio file to text using Whisper. Supports OGG, MP3, WAV, FLAC, M4A, WMA, and OPUS formats. Returns the transcribed text.',
|
||||
schema: {
|
||||
path: 'Path to the audio file — absolute or relative to workspace',
|
||||
language: 'Language hint: "ko" (Korean), "en" (English), "auto" (default)',
|
||||
},
|
||||
jsonSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: { type: 'string', description: 'Path to the audio file' },
|
||||
language: { type: 'string', description: 'Language code: "ko" | "en" | "auto" (default: "auto")' },
|
||||
},
|
||||
required: ['path'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
execute: async (args: any): Promise<ToolResult> => {
|
||||
const filePath = String(args?.path || '').trim();
|
||||
if (!filePath) return { success: false, error: 'path is required' };
|
||||
|
||||
const workspacePath = getConfig().getConfig()?.workspace?.path || process.cwd();
|
||||
const resolved = path.isAbsolute(filePath) ? filePath : path.resolve(workspacePath, filePath);
|
||||
|
||||
if (!fs.existsSync(resolved)) {
|
||||
return { success: false, error: `File not found: ${resolved}` };
|
||||
}
|
||||
const ext = path.extname(resolved).toLowerCase();
|
||||
if (!AUDIO_EXTS.has(ext)) {
|
||||
return { success: false, error: `Unsupported audio format "${ext}". Supported: ${[...AUDIO_EXTS].join(', ')}` };
|
||||
}
|
||||
|
||||
const voiceConfig = (getConfig().getConfig() as any)?.voice;
|
||||
if (!voiceConfig?.enabled) {
|
||||
return { success: false, error: 'Voice/STT is not enabled in config (voice.enabled)' };
|
||||
}
|
||||
|
||||
const stat = fs.statSync(resolved);
|
||||
const sizeMB = (stat.size / 1024 / 1024).toFixed(2);
|
||||
const language = String(args?.language || 'auto').replace(/[^a-zA-Z]/g, '');
|
||||
|
||||
try {
|
||||
const audioBuffer = fs.readFileSync(resolved);
|
||||
const format = ext.slice(1);
|
||||
const { text, language: detectedLang } = await transcribeAudio(audioBuffer, format, language === 'auto' ? undefined : language);
|
||||
|
||||
const lines: string[] = [
|
||||
`File: ${path.basename(resolved)}`,
|
||||
`Size: ${sizeMB} MB | Format: ${format.toUpperCase()} | Lang: ${detectedLang || language}`,
|
||||
'',
|
||||
'[Transcription]',
|
||||
text || '(no speech detected)',
|
||||
];
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stdout: lines.join('\n'),
|
||||
data: {
|
||||
path: resolved,
|
||||
size_bytes: stat.size,
|
||||
transcription: text || '',
|
||||
language: detectedLang,
|
||||
},
|
||||
};
|
||||
} catch (err: any) {
|
||||
return { success: false, error: `Transcription failed: ${err.message}` };
|
||||
}
|
||||
},
|
||||
};
|
||||
+176
-26
@@ -263,7 +263,17 @@ async function downloadImageByUrl(url: string, dest: string): Promise<boolean> {
|
||||
}
|
||||
}
|
||||
|
||||
async function searchUnsplash(query: string, accessKey: string): Promise<string | null> {
|
||||
// UTM tag identifying homeclaw on Unsplash/Pexels referral links (per their guidelines).
|
||||
const ATTRIBUTION_UTM = 'utm_source=homeclaw&utm_medium=referral';
|
||||
|
||||
type ImageHit = {
|
||||
url: string;
|
||||
source: 'Unsplash' | 'Pexels' | 'Pixabay' | 'Browser';
|
||||
credit: string; // human-readable line for slide notes / chat
|
||||
photoId?: string; // present for Unsplash (needed for download trigger)
|
||||
};
|
||||
|
||||
async function searchUnsplash(query: string, accessKey: string): Promise<ImageHit | null> {
|
||||
const url = `https://api.unsplash.com/search/photos?query=${encodeURIComponent(query)}&per_page=3&orientation=landscape`;
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
@@ -281,14 +291,36 @@ async function searchUnsplash(query: string, accessKey: string): Promise<string
|
||||
const photo = data?.results?.[0];
|
||||
const imageUrl = photo?.urls?.regular || photo?.urls?.full || null;
|
||||
console.log(`[pptx] Unsplash: ${data?.results?.length ?? 0} results for "${query}", url=${imageUrl?.slice(0, 80) ?? 'none'}`);
|
||||
return imageUrl;
|
||||
if (!imageUrl) return null;
|
||||
const name = photo?.user?.name || photo?.user?.username || 'Unsplash contributor';
|
||||
const username = photo?.user?.username;
|
||||
const credit = username
|
||||
? `Photo by ${name} (https://unsplash.com/@${username}?${ATTRIBUTION_UTM}) on Unsplash (https://unsplash.com/?${ATTRIBUTION_UTM})`
|
||||
: `Photo by ${name} on Unsplash (https://unsplash.com/?${ATTRIBUTION_UTM})`;
|
||||
return { url: imageUrl, source: 'Unsplash', credit, photoId: photo?.id };
|
||||
} catch (e: any) {
|
||||
console.log(`[pptx] Unsplash error for "${query}": ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function searchPexels(query: string, apiKey: string): Promise<string | null> {
|
||||
/**
|
||||
* Per Unsplash API guidelines: GET /photos/:id/download must be hit when the
|
||||
* photo is actually used (embedded in a deliverable, displayed to the user).
|
||||
* Fire-and-forget — failure must not block the user-facing flow.
|
||||
*/
|
||||
function triggerUnsplashDownload(photoId: string, accessKey: string): void {
|
||||
const url = `https://api.unsplash.com/photos/${encodeURIComponent(photoId)}/download`;
|
||||
fetch(url, {
|
||||
headers: { 'Authorization': `Client-ID ${accessKey}` },
|
||||
signal: AbortSignal.timeout(5000),
|
||||
}).then(
|
||||
r => console.log(`[pptx] Unsplash download trigger: ${photoId} -> ${r.status}`),
|
||||
e => console.log(`[pptx] Unsplash download trigger error for ${photoId}: ${e.message}`),
|
||||
);
|
||||
}
|
||||
|
||||
async function searchPexels(query: string, apiKey: string): Promise<ImageHit | null> {
|
||||
const url = `https://api.pexels.com/v1/search?query=${encodeURIComponent(query)}&per_page=3&orientation=landscape`;
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
@@ -306,13 +338,101 @@ async function searchPexels(query: string, apiKey: string): Promise<string | nul
|
||||
const photo = data?.photos?.[0];
|
||||
const imageUrl = photo?.src?.large2x || photo?.src?.large || photo?.src?.original || null;
|
||||
console.log(`[pptx] Pexels: ${data?.photos?.length ?? 0} results for "${query}", url=${imageUrl?.slice(0, 80) ?? 'none'}`);
|
||||
return imageUrl;
|
||||
if (!imageUrl) return null;
|
||||
const name = photo?.photographer || 'Pexels contributor';
|
||||
const profileUrl = photo?.photographer_url;
|
||||
const credit = profileUrl
|
||||
? `Photo by ${name} (${profileUrl}) on Pexels (https://www.pexels.com/?${ATTRIBUTION_UTM})`
|
||||
: `Photo by ${name} on Pexels (https://www.pexels.com/?${ATTRIBUTION_UTM})`;
|
||||
return { url: imageUrl, source: 'Pexels', credit };
|
||||
} catch (e: any) {
|
||||
console.log(`[pptx] Pexels error for "${query}": ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function searchPixabay(query: string, apiKey: string): Promise<ImageHit | null> {
|
||||
const url = `https://pixabay.com/api/?key=${encodeURIComponent(apiKey)}&q=${encodeURIComponent(query)}&per_page=3&image_type=photo&orientation=horizontal&safesearch=true`;
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 10000);
|
||||
const resp = await fetch(url, { signal: controller.signal });
|
||||
clearTimeout(timeout);
|
||||
if (!resp.ok) {
|
||||
console.log(`[pptx] Pixabay: HTTP ${resp.status} for "${query}"`);
|
||||
return null;
|
||||
}
|
||||
const data = await resp.json() as any;
|
||||
const photo = data?.hits?.[0];
|
||||
const imageUrl = photo?.largeImageURL || photo?.webformatURL || null;
|
||||
console.log(`[pptx] Pixabay: ${data?.hits?.length ?? 0} results for "${query}", url=${imageUrl?.slice(0, 80) ?? 'none'}`);
|
||||
if (!imageUrl) return null;
|
||||
const name = photo?.user || 'Pixabay contributor';
|
||||
const pageUrl = photo?.pageURL;
|
||||
const credit = pageUrl
|
||||
? `Image by ${name} on Pixabay (${pageUrl})`
|
||||
: `Image by ${name} on Pixabay`;
|
||||
return { url: imageUrl, source: 'Pixabay', credit };
|
||||
} catch (e: any) {
|
||||
console.log(`[pptx] Pixabay error for "${query}": ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Browser-driven fallback used when all API providers (Pexels/Pixabay/Unsplash)
|
||||
* fail or are unconfigured. Scrapes Google Images via the existing Playwright
|
||||
* browser session helpers — same pattern already used by `search_images` for
|
||||
* Korean queries. Lazy-imported so pptx.ts doesn't pull Playwright at boot.
|
||||
*/
|
||||
async function searchBrowser(query: string): Promise<ImageHit | null> {
|
||||
let bt: typeof import('../gateway/browser-tools.js');
|
||||
try {
|
||||
bt = await import('../gateway/browser-tools.js');
|
||||
} catch (e: any) {
|
||||
console.log(`[pptx] Browser fallback: import failed: ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const sid = `pptx-img-${Date.now()}`;
|
||||
const searchUrl = `https://www.google.com/search?tbm=isch&q=${encodeURIComponent(query)}`;
|
||||
try {
|
||||
const opened = await bt.browserOpen(sid, searchUrl);
|
||||
if (opened.startsWith('ERROR:')) {
|
||||
console.log(`[pptx] Browser fallback: open failed: ${opened.slice(0, 120)}`);
|
||||
return null;
|
||||
}
|
||||
await bt.browserWait(sid, 2500);
|
||||
const result = await bt.browserGetImages(sid, { max_images: 12, download: false });
|
||||
if (result.startsWith('ERROR:') || result.includes('Found 0 images')) {
|
||||
console.log(`[pptx] Browser fallback: no images for "${query}"`);
|
||||
return null;
|
||||
}
|
||||
// Parse " - [type] https://..." lines from browserGetImages text output.
|
||||
const regex = /^\s*-\s*\[[^\]]+\]\s+(https?:\/\/\S+)/gm;
|
||||
const urls: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = regex.exec(result)) !== null) {
|
||||
// Skip Google's UI sprites/favicons; prefer real content URLs.
|
||||
if (m[1].includes('gstatic.com/images') || m[1].includes('/favicon')) continue;
|
||||
if (!urls.includes(m[1])) urls.push(m[1]);
|
||||
}
|
||||
const url = urls[0];
|
||||
if (!url) return null;
|
||||
console.log(`[pptx] Browser fallback: picked ${url.slice(0, 80)}`);
|
||||
return {
|
||||
url,
|
||||
source: 'Browser',
|
||||
credit: `Image via web search for "${query}"`,
|
||||
};
|
||||
} catch (e: any) {
|
||||
console.log(`[pptx] Browser fallback error for "${query}": ${e.message}`);
|
||||
return null;
|
||||
} finally {
|
||||
try { await bt.browserClose(sid); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
async function searchAndDownloadImage(
|
||||
query: string,
|
||||
projectDir: string,
|
||||
@@ -320,7 +440,8 @@ async function searchAndDownloadImage(
|
||||
sendSSE?: SSESender,
|
||||
unsplashKey?: string,
|
||||
pexelsKey?: string,
|
||||
): Promise<string | null> {
|
||||
pixabayKey?: string,
|
||||
): Promise<{ filename: string; credit: string } | null> {
|
||||
console.log(`[pptx] searchAndDownloadImage: slide ${slideIndex + 1}, query="${query}"`);
|
||||
sendSSE?.('info', { message: `🖼 Slide ${slideIndex + 1}: searching for "${query}"...` });
|
||||
|
||||
@@ -330,41 +451,66 @@ async function searchAndDownloadImage(
|
||||
// Strategy 1: Pexels (200 req/hour — primary)
|
||||
if (pexelsKey) {
|
||||
sendSSE?.('info', { message: `🔍 Slide ${slideIndex + 1}: searching Pexels...` });
|
||||
const imageUrl = await searchPexels(query, pexelsKey);
|
||||
if (imageUrl) {
|
||||
const hit = await searchPexels(query, pexelsKey);
|
||||
if (hit) {
|
||||
sendSSE?.('info', { message: `📥 Slide ${slideIndex + 1}: downloading from Pexels...` });
|
||||
if (await downloadImageByUrl(imageUrl, dest)) {
|
||||
if (await downloadImageByUrl(hit.url, dest)) {
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Pexels OK`);
|
||||
sendSSE?.('info', { message: `✅ Slide ${slideIndex + 1}: image ready (Pexels)` });
|
||||
return fname;
|
||||
return { filename: fname, credit: hit.credit };
|
||||
}
|
||||
}
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Pexels failed for "${query}"`);
|
||||
sendSSE?.('info', { message: `⚠️ Slide ${slideIndex + 1}: Pexels failed` });
|
||||
}
|
||||
|
||||
// Strategy 2: Unsplash (50 req/hour — fallback)
|
||||
// Strategy 2: Pixabay (200 req/min — secondary)
|
||||
if (pixabayKey) {
|
||||
sendSSE?.('info', { message: `🔍 Slide ${slideIndex + 1}: searching Pixabay...` });
|
||||
const hit = await searchPixabay(query, pixabayKey);
|
||||
if (hit) {
|
||||
sendSSE?.('info', { message: `📥 Slide ${slideIndex + 1}: downloading from Pixabay...` });
|
||||
if (await downloadImageByUrl(hit.url, dest)) {
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Pixabay OK`);
|
||||
sendSSE?.('info', { message: `✅ Slide ${slideIndex + 1}: image ready (Pixabay)` });
|
||||
return { filename: fname, credit: hit.credit };
|
||||
}
|
||||
}
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Pixabay failed for "${query}"`);
|
||||
sendSSE?.('info', { message: `⚠️ Slide ${slideIndex + 1}: Pixabay failed` });
|
||||
}
|
||||
|
||||
// Strategy 3: Unsplash (50 req/hour — fallback)
|
||||
if (unsplashKey) {
|
||||
sendSSE?.('info', { message: `🔍 Slide ${slideIndex + 1}: searching Unsplash...` });
|
||||
const imageUrl = await searchUnsplash(query, unsplashKey);
|
||||
if (imageUrl) {
|
||||
const hit = await searchUnsplash(query, unsplashKey);
|
||||
if (hit) {
|
||||
sendSSE?.('info', { message: `📥 Slide ${slideIndex + 1}: downloading from Unsplash...` });
|
||||
if (await downloadImageByUrl(imageUrl, dest)) {
|
||||
if (await downloadImageByUrl(hit.url, dest)) {
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Unsplash OK`);
|
||||
sendSSE?.('info', { message: `✅ Slide ${slideIndex + 1}: image ready (Unsplash)` });
|
||||
return fname;
|
||||
if (hit.photoId) triggerUnsplashDownload(hit.photoId, unsplashKey);
|
||||
return { filename: fname, credit: hit.credit };
|
||||
}
|
||||
}
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Unsplash failed for "${query}"`);
|
||||
sendSSE?.('info', { message: `⚠️ Slide ${slideIndex + 1}: Unsplash failed` });
|
||||
}
|
||||
|
||||
if (!unsplashKey && !pexelsKey) {
|
||||
sendSSE?.('info', { message: `⚠️ Slide ${slideIndex + 1}: no image API keys configured (set UNSPLASH_ACCESS_KEY or PEXELS_API_KEY)` });
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: no API keys for image search`);
|
||||
} else {
|
||||
sendSSE?.('info', { message: `❌ Slide ${slideIndex + 1}: all image sources failed for "${query}"` });
|
||||
// Strategy 4: Browser fallback (Google Images via Playwright) — last resort,
|
||||
// works even with no API keys configured.
|
||||
sendSSE?.('info', { message: `🌐 Slide ${slideIndex + 1}: trying browser fallback...` });
|
||||
const browserHit = await searchBrowser(query);
|
||||
if (browserHit) {
|
||||
sendSSE?.('info', { message: `📥 Slide ${slideIndex + 1}: downloading from web search...` });
|
||||
if (await downloadImageByUrl(browserHit.url, dest)) {
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Browser fallback OK`);
|
||||
sendSSE?.('info', { message: `✅ Slide ${slideIndex + 1}: image ready (web search)` });
|
||||
return { filename: fname, credit: browserHit.credit };
|
||||
}
|
||||
}
|
||||
console.log(`[pptx] slide ${slideIndex + 1}: Browser fallback failed for "${query}"`);
|
||||
sendSSE?.('info', { message: `❌ Slide ${slideIndex + 1}: all image sources failed for "${query}"` });
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -461,6 +607,7 @@ export const pptxTool: import('./registry.js').Tool = {
|
||||
// Resolve image API keys: env var > config (vault-resolved)
|
||||
const unsplashKey = process.env.UNSPLASH_ACCESS_KEY || configMgr.resolveSecret(config.ppt?.unsplash_key) || '';
|
||||
const pexelsKey = process.env.PEXELS_API_KEY || configMgr.resolveSecret(config.ppt?.pexels_key) || '';
|
||||
const pixabayKey = process.env.PIXABAY_API_KEY || configMgr.resolveSecret(config.ppt?.pixabay_key) || '';
|
||||
|
||||
// Fix deprecated/defunct image URLs before downloading
|
||||
for (const slide of spec.slides) {
|
||||
@@ -485,10 +632,11 @@ export const pptxTool: import('./registry.js').Tool = {
|
||||
|
||||
// image_search takes priority
|
||||
if (slide.image_search && !slide.image_path) {
|
||||
const localPath = await searchAndDownloadImage(slide.image_search, projectDir, i, sendSSE, unsplashKey, pexelsKey);
|
||||
console.log(`[pptx] slide ${i + 1} image_search="${slide.image_search}" -> "${localPath}"`);
|
||||
if (localPath) {
|
||||
slide.image_path = localPath;
|
||||
const match = await searchAndDownloadImage(slide.image_search, projectDir, i, sendSSE, unsplashKey, pexelsKey, pixabayKey);
|
||||
console.log(`[pptx] slide ${i + 1} image_search="${slide.image_search}" -> "${match?.filename}"`);
|
||||
if (match) {
|
||||
slide.image_path = match.filename;
|
||||
slide.notes = slide.notes ? `${slide.notes}\n\n${match.credit}` : match.credit;
|
||||
}
|
||||
delete slide.image_search;
|
||||
// image_search overrides image_url
|
||||
@@ -605,6 +753,7 @@ export const editPptxTool: import('./registry.js').Tool = {
|
||||
// Resolve image API keys
|
||||
const unsplashKey = process.env.UNSPLASH_ACCESS_KEY || configMgr.resolveSecret(config.ppt?.unsplash_key) || '';
|
||||
const pexelsKey = process.env.PEXELS_API_KEY || configMgr.resolveSecret(config.ppt?.pexels_key) || '';
|
||||
const pixabayKey = process.env.PIXABAY_API_KEY || configMgr.resolveSecret(config.ppt?.pixabay_key) || '';
|
||||
|
||||
// Fix deprecated/defunct image URLs before downloading
|
||||
for (const slide of spec.slides) {
|
||||
@@ -622,9 +771,10 @@ export const editPptxTool: import('./registry.js').Tool = {
|
||||
|
||||
// image_search takes priority
|
||||
if (slide.image_search && !slide.image_path) {
|
||||
const localPath = await searchAndDownloadImage(slide.image_search, editProjectDir, i, sendSSE, unsplashKey, pexelsKey);
|
||||
if (localPath) {
|
||||
slide.image_path = localPath;
|
||||
const match = await searchAndDownloadImage(slide.image_search, editProjectDir, i, sendSSE, unsplashKey, pexelsKey, pixabayKey);
|
||||
if (match) {
|
||||
slide.image_path = match.filename;
|
||||
slide.notes = slide.notes ? `${slide.notes}\n\n${match.credit}` : match.credit;
|
||||
}
|
||||
delete slide.image_search;
|
||||
if (slide.image_url) delete slide.image_url;
|
||||
|
||||
@@ -16,6 +16,7 @@ import { pubmedSearchTool, pubmedFetchTool, pubmedFulltextTool } from './pubmed.
|
||||
import { openalexSearchTool, semanticSearchTool } from './scholar.js';
|
||||
import { pdfReadTool } from './pdf.js';
|
||||
import { imageReadTool } from './image.js';
|
||||
import { audioTranscribeTool } from './audio-transcribe.js';
|
||||
import { pythonEvalTool } from './python.js';
|
||||
import { sqliteTool } from './sqlite.js';
|
||||
import { weatherSearchTool } from './weather.js';
|
||||
@@ -56,6 +57,7 @@ const TOOL_PROFILE_TOOL_NAMES: Record<Exclude<ToolProfile, 'full'>, ReadonlySet<
|
||||
'memory_write',
|
||||
'pdf_read',
|
||||
'image_read',
|
||||
'audio_transcribe',
|
||||
'python_eval',
|
||||
'sqlite_query',
|
||||
]),
|
||||
@@ -184,6 +186,7 @@ class ToolRegistry {
|
||||
// Document & data tools
|
||||
this.registerSafe(pdfReadTool);
|
||||
this.registerSafe(imageReadTool);
|
||||
this.registerSafe(audioTranscribeTool);
|
||||
this.registerSafe(pythonEvalTool);
|
||||
this.registerSafe(sqliteTool);
|
||||
// Weather tool
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import { getConfig } from '../config/config.js';
|
||||
|
||||
interface STTConfig {
|
||||
whisperPath: string;
|
||||
modelPath: string;
|
||||
language: string;
|
||||
ffmpegPath: string;
|
||||
tempDir: string;
|
||||
}
|
||||
|
||||
function getSTTConfig(): STTConfig {
|
||||
const cfg = getConfig().getConfig() as any;
|
||||
const voice = cfg?.voice;
|
||||
return {
|
||||
whisperPath: voice?.stt?.whisperPath || process.env.WHISPER_CLI_PATH || 'whisper-cli',
|
||||
modelPath: voice?.stt?.modelPath || process.env.WHISPER_MODEL_PATH || '',
|
||||
language: voice?.stt?.language || 'ko',
|
||||
ffmpegPath: voice?.ffmpegPath || 'ffmpeg',
|
||||
tempDir: voice?.tempDir || os.tmpdir(),
|
||||
};
|
||||
}
|
||||
|
||||
async function convertAudioToWav(
|
||||
inputBuffer: Buffer,
|
||||
sourceFormat: string,
|
||||
config: STTConfig
|
||||
): Promise<string> {
|
||||
const tempDir = config.tempDir;
|
||||
const inputPath = path.join(tempDir, `stt_input_${Date.now()}.${sourceFormat}`);
|
||||
const outputPath = path.join(tempDir, `stt_output_${Date.now()}.wav`);
|
||||
|
||||
try {
|
||||
fs.writeFileSync(inputPath, inputBuffer);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn(config.ffmpegPath, [
|
||||
'-y', '-i', inputPath,
|
||||
'-ar', '16000', '-ac', '1', '-f', 'wav',
|
||||
outputPath,
|
||||
]);
|
||||
let stderr = '';
|
||||
ffmpeg.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`ffmpeg exited ${code}: ${stderr}`));
|
||||
});
|
||||
ffmpeg.on('error', reject);
|
||||
});
|
||||
|
||||
return outputPath;
|
||||
} finally {
|
||||
try { fs.unlinkSync(inputPath); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
export async function transcribeAudio(
|
||||
audioBuffer: Buffer,
|
||||
sourceFormat: string = 'ogg',
|
||||
languageOverride?: string
|
||||
): Promise<{ text: string; language?: string }> {
|
||||
const config = getSTTConfig();
|
||||
|
||||
if (!config.modelPath) {
|
||||
throw new Error('STT model path not configured. Set voice.stt.modelPath or WHISPER_MODEL_PATH env var.');
|
||||
}
|
||||
|
||||
const wavPath = await convertAudioToWav(audioBuffer, sourceFormat, config);
|
||||
const language = languageOverride || config.language;
|
||||
|
||||
try {
|
||||
// whisper-cli: capture transcription from stdout
|
||||
const args = [
|
||||
'-m', config.modelPath,
|
||||
'-f', wavPath,
|
||||
'-l', language,
|
||||
'--no-timestamps',
|
||||
'-nt', // no print timestamps
|
||||
];
|
||||
|
||||
const result = await new Promise<{ text: string; language?: string }>((resolve, reject) => {
|
||||
const whisper = spawn(config.whisperPath, args);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
whisper.stdout.on('data', (d: Buffer) => { stdout += d.toString(); });
|
||||
whisper.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
|
||||
whisper.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
reject(new Error(`whisper-cli exited ${code}: ${stderr.slice(0, 500)}`));
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse stdout: whisper-cli outputs transcription lines like:
|
||||
// [00:00:00.000 --> 00:00:02.000] transcribed text
|
||||
// With --no-timestamps, lines are just the text
|
||||
const lines = stdout.split('\n')
|
||||
.map(line => line.replace(/^\[\d{2}:\d{2}:\d{2}\.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}\.\d{3}\]\s*/, '').trim())
|
||||
.filter(line => line.length > 0 && !line.startsWith('[') && !line.startsWith('whisper_'));
|
||||
|
||||
const text = lines.join(' ').trim();
|
||||
resolve({ text, language });
|
||||
});
|
||||
whisper.on('error', reject);
|
||||
});
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
try { fs.unlinkSync(wavPath); } catch {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import { getConfig } from '../config/config.js';
|
||||
|
||||
interface TTSConfig {
|
||||
piperPath: string;
|
||||
modelPath: string;
|
||||
configPath?: string;
|
||||
ffmpegPath: string;
|
||||
tempDir: string;
|
||||
}
|
||||
|
||||
function getTTSConfig(): TTSConfig | null {
|
||||
const cfg = getConfig().getConfig() as any;
|
||||
const voice = cfg?.voice;
|
||||
if (!voice?.tts) return null;
|
||||
|
||||
return {
|
||||
piperPath: voice.tts.piperPath || 'piper',
|
||||
modelPath: voice.tts.modelPath || process.env.PIPER_MODEL_PATH || '',
|
||||
configPath: voice.tts.configPath || process.env.PIPER_CONFIG_PATH,
|
||||
ffmpegPath: voice.ffmpegPath || 'ffmpeg',
|
||||
tempDir: voice.tempDir || os.tmpdir(),
|
||||
};
|
||||
}
|
||||
|
||||
export function isTTSAvailable(): boolean {
|
||||
return getTTSConfig() !== null;
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(text: string): Promise<Buffer> {
|
||||
const config = getTTSConfig();
|
||||
if (!config) throw new Error('TTS not configured');
|
||||
|
||||
const truncatedText = text.slice(0, 4000);
|
||||
const tempDir = config.tempDir;
|
||||
const wavPath = path.join(tempDir, `tts_output_${Date.now()}.wav`);
|
||||
const oggPath = path.join(tempDir, `tts_output_${Date.now()}.ogg`);
|
||||
|
||||
try {
|
||||
// Generate WAV via Piper
|
||||
const piperArgs = ['--model', config.modelPath, '--output_file', wavPath];
|
||||
if (config.configPath) {
|
||||
piperArgs.push('--config', config.configPath);
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const piper = spawn(config.piperPath, piperArgs);
|
||||
let stderr = '';
|
||||
piper.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
|
||||
piper.stdin.write(truncatedText);
|
||||
piper.stdin.end();
|
||||
piper.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`Piper exited ${code}: ${stderr}`));
|
||||
});
|
||||
piper.on('error', reject);
|
||||
});
|
||||
|
||||
// Convert WAV to OGG/Opus for Telegram sendVoice
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn(config.ffmpegPath, [
|
||||
'-y', '-i', wavPath,
|
||||
'-c:a', 'libopus', '-b:a', '48k', '-vbr', 'on',
|
||||
'-compression_level', '10',
|
||||
oggPath,
|
||||
]);
|
||||
let stderr = '';
|
||||
ffmpeg.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`ffmpeg exited ${code}: ${stderr}`));
|
||||
});
|
||||
ffmpeg.on('error', reject);
|
||||
});
|
||||
|
||||
return fs.readFileSync(oggPath);
|
||||
} finally {
|
||||
try { fs.unlinkSync(wavPath); } catch {}
|
||||
try { fs.unlinkSync(oggPath); } catch {}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user