From edf2c3947205a3cb3c1524220d45d677b9ce0819 Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 2 Jun 2026 14:58:22 +0900 Subject: [PATCH] Release 2.9.4: PPTX wizard figure analysis + skill model trigger fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pptx-wizard: add "๐Ÿ” ์ด๋ฏธ์ง€ ๋ถ„์„" button in Step 2 image panel * calls /api/pptx/analyze-image per figure via mistral vision * shows per-image progress (1/10, 2/10...) * "์ œ์™ธ" caption โ†’ dimmed card + filtered from outline prompt * captions shown below each thumbnail in the grid - server: POST /api/pptx/analyze-image endpoint * resolves /api/files/ path to workspace file, thumbnails if >300KB * calls orchestration.secondary vision model (mistral-large-3:675b-cloud) * caches results in project/.image-captions.json (skips on re-analysis) - pptx-wizard: generateOutline() includes captions in imgList * "์ œ์™ธ" images filtered out before sending to LLM * format: "/api/files/... (figure_p2_1) โ€” ์นดํ”Œ๋ž€-๋งˆ์ด์–ด ์ƒ์กด๊ณก์„  3๊ทธ๋ฃน" - skills-manager: fix getModelOverrideForUser() trigger filtering * was returning skill model for ALL messages regardless of trigger keywords * now only overrides when message matches skill triggers (same logic as buildPromptContext) * presenter model (mistral) now only applies for ๋ฐœํ‘œ/pptx/์Šฌ๋ผ์ด๋“œ keywords Co-Authored-By: Claude Sonnet 4.6 --- src/gateway/server-v2.ts | 81 ++++++++++++++++++++++++++++++++++- src/gateway/skills-manager.ts | 12 ++++-- web-ui/pptx-wizard.html | 49 ++++++++++++++++++--- 3 files changed, 131 insertions(+), 11 deletions(-) diff --git a/src/gateway/server-v2.ts b/src/gateway/server-v2.ts index e6e3082..7e67b0e 100644 --- a/src/gateway/server-v2.ts +++ b/src/gateway/server-v2.ts @@ -6510,7 +6510,7 @@ RULES: // Model priority: explicit request > vision fallback > skill override > config default const effectiveModel = String(modelOverride || '').trim() || _visionFallbackModel - || skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null) + || skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null, message) || undefined; // Always use streaming so text tokens appear in real-time. The streaming // handler collects tool_calls from chunks and returns them in the final @@ -9942,6 +9942,85 @@ app.post('/api/pptx/extract-images', requireGatewayAuth, async (req: express.Req } }); +// POST /api/pptx/analyze-image โ€” vision-model caption for a single extracted figure +// Caches results in workspace/pptx//.image-captions.json +app.post('/api/pptx/analyze-image', requireGatewayAuth, async (req: express.Request, res: express.Response) => { + try { + const session = getSessionUser(req); + const username = (session as any)?.username; + const workspacePath = username ? getUserWorkspace(username) : getConfig().getWorkspacePath(); + + const imgUrl = String(req.body?.url || '').trim(); + const project = pptxProjectSlug(String(req.body?.project || '')); + if (!imgUrl) { res.status(400).json({ error: 'url required' }); return; } + + // Load cache + const cacheFile = project + ? path.join(workspacePath, PPTX_BASE, project, '.image-captions.json') + : null; + let cached: Record = {}; + if (cacheFile && fs.existsSync(cacheFile)) { + try { cached = JSON.parse(fs.readFileSync(cacheFile, 'utf-8')); } catch {} + } + if (cached[imgUrl]) { + res.json({ success: true, url: imgUrl, caption: cached[imgUrl], fromCache: true }); + return; + } + + // Resolve file path + const relParts = imgUrl.replace(/^\/api\/files\//, '').split('/').map(decodeURIComponent); + const filePath = path.resolve(workspacePath, ...relParts); + if (!fs.existsSync(filePath)) { + res.json({ success: false, url: imgUrl, caption: '' }); + return; + } + + // Read + thumbnail if large + let buf: Buffer | null = null; + const stat = fs.statSync(filePath); + if (stat.size <= 300_000) { + buf = fs.readFileSync(filePath); + } else { + try { + const { execSync } = await import('child_process'); + const script = `from PIL import Image; import sys; img=Image.open(sys.argv[1]); img.thumbnail((600,600)); img.save(sys.stdout.buffer,'JPEG',quality=80)`; + buf = execSync(`python3 -c "${script}" "${filePath}"`, { maxBuffer: 5 * 1024 * 1024 }); + } catch { buf = null; } + } + if (!buf) { res.json({ success: false, url: imgUrl, caption: '' }); return; } + + const ext = path.extname(filePath).toLowerCase(); + const mime = ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : 'image/jpeg'; + const b64 = `data:${mime};base64,${buf.toString('base64')}`; + + // Call vision model + const ollama2 = getOllamaClient(); + const rawCfg2 = getConfig().getConfig() as any; + const sec2 = rawCfg2.orchestration?.secondary; + const visionModel = (sec2?.model && sec2?.vision === true) ? sec2.model : 'mistral-large-3:675b-cloud'; + + const result = await ollama2.chatWithThinking([{ + role: 'user' as const, + content: [ + { type: 'text' as const, text: '์ด ๊ทธ๋ฆผ์„ ํ•œ ์ค„(30์ž ์ด๋‚ด)๋กœ ์„ค๋ช…ํ•˜์„ธ์š”. ๊ทธ๋ž˜ํ”„/์ฐจํŠธ/๋‹ค์ด์–ด๊ทธ๋žจ ์ข…๋ฅ˜์™€ ํ•ต์‹ฌ ๋‚ด์šฉ ์œ„์ฃผ๋กœ. ์Šฌ๋ผ์ด๋“œ์— ์“ฐ๊ธฐ ๋ถ€์ ํ•ฉํ•œ ๊ทธ๋ฆผ(ํ๋ฆฟํ•จยทํ…์ŠคํŠธ๋งŒ ์žˆ๋Š” ํ‘œยท๋นˆ ์ด๋ฏธ์ง€)์ด๋ฉด "์ œ์™ธ"๋ผ๊ณ ๋งŒ ๋‹ตํ•˜์„ธ์š”.' }, + { type: 'image_url' as const, image_url: { url: b64 } }, + ], + }], 'executor', { model: visionModel, num_predict: 80 }); + + const caption = String(result.message?.content || '').trim().replace(/\n/g, ' ').slice(0, 80); + cached[imgUrl] = caption; + if (cacheFile) { + try { + fs.mkdirSync(path.dirname(cacheFile), { recursive: true }); + fs.writeFileSync(cacheFile, JSON.stringify(cached, null, 2), 'utf-8'); + } catch {} + } + res.json({ success: true, url: imgUrl, caption }); + } catch (err) { + res.status(500).json({ error: String(err) }); + } +}); + // GET /api/pptx/project-pdfs?project= โ€” list PDF files in project folder // Also returns companion .txt / .ko.txt paths if they exist. app.get('/api/pptx/project-pdfs', requireGatewayAuth, async (req: express.Request, res: express.Response) => { diff --git a/src/gateway/skills-manager.ts b/src/gateway/skills-manager.ts index 5f01380..657600a 100644 --- a/src/gateway/skills-manager.ts +++ b/src/gateway/skills-manager.ts @@ -286,8 +286,9 @@ export class SkillsManager { return parts.join('\n'); } - /** Return the model override from the first enabled skill that specifies one, or undefined. */ - getModelOverrideForUser(userDir: string | null): string | undefined { + /** Return the model override from the first triggered+enabled skill that specifies one, or undefined. */ + getModelOverrideForUser(userDir: string | null, userMessage = ''): string | undefined { + const msgLower = userMessage.toLowerCase(); const userState = userDir ? this.getUserState(userDir) : null; const enabled = this.getAll().filter(s => { if (this.GLOBAL_ONLY_SKILLS.has(s.id)) return s.enabled; @@ -295,7 +296,12 @@ export class SkillsManager { return s.enabled; }); for (const skill of enabled) { - if (skill.model) return skill.model; + if (!skill.model) continue; + // Only apply model override when the skill is triggered (or has no triggers) + if (skill.triggers && skill.triggers.length > 0) { + if (!skill.triggers.some(t => msgLower.includes(t.toLowerCase()))) continue; + } + return skill.model; } return undefined; } diff --git a/web-ui/pptx-wizard.html b/web-ui/pptx-wizard.html index 5a2e566..ac8fe1f 100644 --- a/web-ui/pptx-wizard.html +++ b/web-ui/pptx-wizard.html @@ -343,6 +343,7 @@ body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,san
์บก์ฒ˜ ์ด๋ฏธ์ง€ +