diff --git a/src/gateway/server-v2.ts b/src/gateway/server-v2.ts index e6e3082..7e67b0e 100644 --- a/src/gateway/server-v2.ts +++ b/src/gateway/server-v2.ts @@ -6510,7 +6510,7 @@ RULES: // Model priority: explicit request > vision fallback > skill override > config default const effectiveModel = String(modelOverride || '').trim() || _visionFallbackModel - || skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null) + || skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null, message) || undefined; // Always use streaming so text tokens appear in real-time. The streaming // handler collects tool_calls from chunks and returns them in the final @@ -9942,6 +9942,85 @@ app.post('/api/pptx/extract-images', requireGatewayAuth, async (req: express.Req } }); +// POST /api/pptx/analyze-image — vision-model caption for a single extracted figure +// Caches results in workspace/pptx//.image-captions.json +app.post('/api/pptx/analyze-image', requireGatewayAuth, async (req: express.Request, res: express.Response) => { + try { + const session = getSessionUser(req); + const username = (session as any)?.username; + const workspacePath = username ? getUserWorkspace(username) : getConfig().getWorkspacePath(); + + const imgUrl = String(req.body?.url || '').trim(); + const project = pptxProjectSlug(String(req.body?.project || '')); + if (!imgUrl) { res.status(400).json({ error: 'url required' }); return; } + + // Load cache + const cacheFile = project + ? path.join(workspacePath, PPTX_BASE, project, '.image-captions.json') + : null; + let cached: Record = {}; + if (cacheFile && fs.existsSync(cacheFile)) { + try { cached = JSON.parse(fs.readFileSync(cacheFile, 'utf-8')); } catch {} + } + if (cached[imgUrl]) { + res.json({ success: true, url: imgUrl, caption: cached[imgUrl], fromCache: true }); + return; + } + + // Resolve file path + const relParts = imgUrl.replace(/^\/api\/files\//, '').split('/').map(decodeURIComponent); + const filePath = path.resolve(workspacePath, ...relParts); + if (!fs.existsSync(filePath)) { + res.json({ success: false, url: imgUrl, caption: '' }); + return; + } + + // Read + thumbnail if large + let buf: Buffer | null = null; + const stat = fs.statSync(filePath); + if (stat.size <= 300_000) { + buf = fs.readFileSync(filePath); + } else { + try { + const { execSync } = await import('child_process'); + const script = `from PIL import Image; import sys; img=Image.open(sys.argv[1]); img.thumbnail((600,600)); img.save(sys.stdout.buffer,'JPEG',quality=80)`; + buf = execSync(`python3 -c "${script}" "${filePath}"`, { maxBuffer: 5 * 1024 * 1024 }); + } catch { buf = null; } + } + if (!buf) { res.json({ success: false, url: imgUrl, caption: '' }); return; } + + const ext = path.extname(filePath).toLowerCase(); + const mime = ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : 'image/jpeg'; + const b64 = `data:${mime};base64,${buf.toString('base64')}`; + + // Call vision model + const ollama2 = getOllamaClient(); + const rawCfg2 = getConfig().getConfig() as any; + const sec2 = rawCfg2.orchestration?.secondary; + const visionModel = (sec2?.model && sec2?.vision === true) ? sec2.model : 'mistral-large-3:675b-cloud'; + + const result = await ollama2.chatWithThinking([{ + role: 'user' as const, + content: [ + { type: 'text' as const, text: '이 그림을 한 줄(30자 이내)로 설명하세요. 그래프/차트/다이어그램 종류와 핵심 내용 위주로. 슬라이드에 쓰기 부적합한 그림(흐릿함·텍스트만 있는 표·빈 이미지)이면 "제외"라고만 답하세요.' }, + { type: 'image_url' as const, image_url: { url: b64 } }, + ], + }], 'executor', { model: visionModel, num_predict: 80 }); + + const caption = String(result.message?.content || '').trim().replace(/\n/g, ' ').slice(0, 80); + cached[imgUrl] = caption; + if (cacheFile) { + try { + fs.mkdirSync(path.dirname(cacheFile), { recursive: true }); + fs.writeFileSync(cacheFile, JSON.stringify(cached, null, 2), 'utf-8'); + } catch {} + } + res.json({ success: true, url: imgUrl, caption }); + } catch (err) { + res.status(500).json({ error: String(err) }); + } +}); + // GET /api/pptx/project-pdfs?project= — list PDF files in project folder // Also returns companion .txt / .ko.txt paths if they exist. app.get('/api/pptx/project-pdfs', requireGatewayAuth, async (req: express.Request, res: express.Response) => { diff --git a/src/gateway/skills-manager.ts b/src/gateway/skills-manager.ts index 5f01380..657600a 100644 --- a/src/gateway/skills-manager.ts +++ b/src/gateway/skills-manager.ts @@ -286,8 +286,9 @@ export class SkillsManager { return parts.join('\n'); } - /** Return the model override from the first enabled skill that specifies one, or undefined. */ - getModelOverrideForUser(userDir: string | null): string | undefined { + /** Return the model override from the first triggered+enabled skill that specifies one, or undefined. */ + getModelOverrideForUser(userDir: string | null, userMessage = ''): string | undefined { + const msgLower = userMessage.toLowerCase(); const userState = userDir ? this.getUserState(userDir) : null; const enabled = this.getAll().filter(s => { if (this.GLOBAL_ONLY_SKILLS.has(s.id)) return s.enabled; @@ -295,7 +296,12 @@ export class SkillsManager { return s.enabled; }); for (const skill of enabled) { - if (skill.model) return skill.model; + if (!skill.model) continue; + // Only apply model override when the skill is triggered (or has no triggers) + if (skill.triggers && skill.triggers.length > 0) { + if (!skill.triggers.some(t => msgLower.includes(t.toLowerCase()))) continue; + } + return skill.model; } return undefined; } diff --git a/web-ui/pptx-wizard.html b/web-ui/pptx-wizard.html index 5a2e566..ac8fe1f 100644 --- a/web-ui/pptx-wizard.html +++ b/web-ui/pptx-wizard.html @@ -343,6 +343,7 @@ body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,san
캡처 이미지 +