Release 2.9.4: PPTX wizard figure analysis + skill model trigger fix

- pptx-wizard: add "🔍 이미지 분석" button in Step 2 image panel
  * calls /api/pptx/analyze-image per figure via mistral vision
  * shows per-image progress (1/10, 2/10...)
  * "제외" caption → dimmed card + filtered from outline prompt
  * captions shown below each thumbnail in the grid
- server: POST /api/pptx/analyze-image endpoint
  * resolves /api/files/ path to workspace file, thumbnails if >300KB
  * calls orchestration.secondary vision model (mistral-large-3:675b-cloud)
  * caches results in project/.image-captions.json (skips on re-analysis)
- pptx-wizard: generateOutline() includes captions in imgList
  * "제외" images filtered out before sending to LLM
  * format: "/api/files/... (figure_p2_1) — 카플란-마이어 생존곡선 3그룹"
- skills-manager: fix getModelOverrideForUser() trigger filtering
  * was returning skill model for ALL messages regardless of trigger keywords
  * now only overrides when message matches skill triggers (same logic as buildPromptContext)
  * presenter model (mistral) now only applies for 발표/pptx/슬라이드 keywords

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
kim
2026-06-02 14:58:22 +09:00
co-authored by Claude Sonnet 4.6
parent 69734e2e58
commit edf2c39472
3 changed files with 131 additions and 11 deletions
+80 -1
View File
@@ -6510,7 +6510,7 @@ RULES:
// Model priority: explicit request > vision fallback > skill override > config default // Model priority: explicit request > vision fallback > skill override > config default
const effectiveModel = String(modelOverride || '').trim() const effectiveModel = String(modelOverride || '').trim()
|| _visionFallbackModel || _visionFallbackModel
|| skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null) || skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null, message)
|| undefined; || undefined;
// Always use streaming so text tokens appear in real-time. The streaming // Always use streaming so text tokens appear in real-time. The streaming
// handler collects tool_calls from chunks and returns them in the final // handler collects tool_calls from chunks and returns them in the final
@@ -9942,6 +9942,85 @@ app.post('/api/pptx/extract-images', requireGatewayAuth, async (req: express.Req
} }
}); });
// POST /api/pptx/analyze-image — vision-model caption for a single extracted figure
// Caches results in workspace/pptx/<project>/.image-captions.json
app.post('/api/pptx/analyze-image', requireGatewayAuth, async (req: express.Request, res: express.Response) => {
try {
const session = getSessionUser(req);
const username = (session as any)?.username;
const workspacePath = username ? getUserWorkspace(username) : getConfig().getWorkspacePath();
const imgUrl = String(req.body?.url || '').trim();
const project = pptxProjectSlug(String(req.body?.project || ''));
if (!imgUrl) { res.status(400).json({ error: 'url required' }); return; }
// Load cache
const cacheFile = project
? path.join(workspacePath, PPTX_BASE, project, '.image-captions.json')
: null;
let cached: Record<string, string> = {};
if (cacheFile && fs.existsSync(cacheFile)) {
try { cached = JSON.parse(fs.readFileSync(cacheFile, 'utf-8')); } catch {}
}
if (cached[imgUrl]) {
res.json({ success: true, url: imgUrl, caption: cached[imgUrl], fromCache: true });
return;
}
// Resolve file path
const relParts = imgUrl.replace(/^\/api\/files\//, '').split('/').map(decodeURIComponent);
const filePath = path.resolve(workspacePath, ...relParts);
if (!fs.existsSync(filePath)) {
res.json({ success: false, url: imgUrl, caption: '' });
return;
}
// Read + thumbnail if large
let buf: Buffer | null = null;
const stat = fs.statSync(filePath);
if (stat.size <= 300_000) {
buf = fs.readFileSync(filePath);
} else {
try {
const { execSync } = await import('child_process');
const script = `from PIL import Image; import sys; img=Image.open(sys.argv[1]); img.thumbnail((600,600)); img.save(sys.stdout.buffer,'JPEG',quality=80)`;
buf = execSync(`python3 -c "${script}" "${filePath}"`, { maxBuffer: 5 * 1024 * 1024 });
} catch { buf = null; }
}
if (!buf) { res.json({ success: false, url: imgUrl, caption: '' }); return; }
const ext = path.extname(filePath).toLowerCase();
const mime = ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : 'image/jpeg';
const b64 = `data:${mime};base64,${buf.toString('base64')}`;
// Call vision model
const ollama2 = getOllamaClient();
const rawCfg2 = getConfig().getConfig() as any;
const sec2 = rawCfg2.orchestration?.secondary;
const visionModel = (sec2?.model && sec2?.vision === true) ? sec2.model : 'mistral-large-3:675b-cloud';
const result = await ollama2.chatWithThinking([{
role: 'user' as const,
content: [
{ type: 'text' as const, text: '이 그림을 한 줄(30자 이내)로 설명하세요. 그래프/차트/다이어그램 종류와 핵심 내용 위주로. 슬라이드에 쓰기 부적합한 그림(흐릿함·텍스트만 있는 표·빈 이미지)이면 "제외"라고만 답하세요.' },
{ type: 'image_url' as const, image_url: { url: b64 } },
],
}], 'executor', { model: visionModel, num_predict: 80 });
const caption = String(result.message?.content || '').trim().replace(/\n/g, ' ').slice(0, 80);
cached[imgUrl] = caption;
if (cacheFile) {
try {
fs.mkdirSync(path.dirname(cacheFile), { recursive: true });
fs.writeFileSync(cacheFile, JSON.stringify(cached, null, 2), 'utf-8');
} catch {}
}
res.json({ success: true, url: imgUrl, caption });
} catch (err) {
res.status(500).json({ error: String(err) });
}
});
// GET /api/pptx/project-pdfs?project=<slug> — list PDF files in project folder // GET /api/pptx/project-pdfs?project=<slug> — list PDF files in project folder
// Also returns companion .txt / .ko.txt paths if they exist. // Also returns companion .txt / .ko.txt paths if they exist.
app.get('/api/pptx/project-pdfs', requireGatewayAuth, async (req: express.Request, res: express.Response) => { app.get('/api/pptx/project-pdfs', requireGatewayAuth, async (req: express.Request, res: express.Response) => {
+9 -3
View File
@@ -286,8 +286,9 @@ export class SkillsManager {
return parts.join('\n'); return parts.join('\n');
} }
/** Return the model override from the first enabled skill that specifies one, or undefined. */ /** Return the model override from the first triggered+enabled skill that specifies one, or undefined. */
getModelOverrideForUser(userDir: string | null): string | undefined { getModelOverrideForUser(userDir: string | null, userMessage = ''): string | undefined {
const msgLower = userMessage.toLowerCase();
const userState = userDir ? this.getUserState(userDir) : null; const userState = userDir ? this.getUserState(userDir) : null;
const enabled = this.getAll().filter(s => { const enabled = this.getAll().filter(s => {
if (this.GLOBAL_ONLY_SKILLS.has(s.id)) return s.enabled; if (this.GLOBAL_ONLY_SKILLS.has(s.id)) return s.enabled;
@@ -295,7 +296,12 @@ export class SkillsManager {
return s.enabled; return s.enabled;
}); });
for (const skill of enabled) { for (const skill of enabled) {
if (skill.model) return skill.model; if (!skill.model) continue;
// Only apply model override when the skill is triggered (or has no triggers)
if (skill.triggers && skill.triggers.length > 0) {
if (!skill.triggers.some(t => msgLower.includes(t.toLowerCase()))) continue;
}
return skill.model;
} }
return undefined; return undefined;
} }
+42 -7
View File
@@ -343,6 +343,7 @@ body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,san
<div style="flex:2;min-width:0"> <div style="flex:2;min-width:0">
<div style="display:flex;align-items:center;gap:8px;margin-bottom:6px"> <div style="display:flex;align-items:center;gap:8px;margin-bottom:6px">
<span style="font-size:10px;color:var(--muted);font-weight:600;text-transform:uppercase;letter-spacing:.04em">캡처 이미지 <span id="imgcnt"></span></span> <span style="font-size:10px;color:var(--muted);font-weight:600;text-transform:uppercase;letter-spacing:.04em">캡처 이미지 <span id="imgcnt"></span></span>
<button id="btn-analyze-imgs" onclick="analyzeImages()" style="font-size:11px;color:#10b981;background:rgba(16,185,129,.1);border:1px solid #10b981;border-radius:5px;padding:2px 8px;cursor:pointer;flex-shrink:0" title="AI(mistral vision)가 각 이미지 내용 분석 — 아웃라인 생성 시 정확한 슬라이드 배치에 활용">🔍 이미지 분석</button>
<label style="display:flex;align-items:center;gap:4px;font-size:11px;color:var(--brand2);background:rgba(99,102,241,.1);border:1px solid var(--brand);border-radius:5px;padding:2px 8px;cursor:pointer;flex-shrink:0" title="이미지 파일 업로드"> <label style="display:flex;align-items:center;gap:4px;font-size:11px;color:var(--brand2);background:rgba(99,102,241,.1);border:1px solid var(--brand);border-radius:5px;padding:2px 8px;cursor:pointer;flex-shrink:0" title="이미지 파일 업로드">
<input type="file" id="img-upload-input" accept="image/*" multiple style="display:none" onchange="handleImgUpload(this.files)"> <input type="file" id="img-upload-input" accept="image/*" multiple style="display:none" onchange="handleImgUpload(this.files)">
+ 이미지 업로드 + 이미지 업로드
@@ -762,7 +763,7 @@ function updSrcState(){
const checks=[...document.querySelectorAll('.src-chk input')]; const checks=[...document.querySelectorAll('.src-chk input')];
if(checks.every(c=>!c.checked)) checks[0].checked=true; if(checks.every(c=>!c.checked)) checks[0].checked=true;
} }
let extractedImages = [], paperTexts = {}; let extractedImages = [], paperTexts = {}, imageCaptions = {};
let prepLog = {}, outline = null, outlineEdits = {}, currentProcessingKey = null; let prepLog = {}, outline = null, outlineEdits = {}, currentProcessingKey = null;
let pptxRelPath = null, previewImages = []; let pptxRelPath = null, previewImages = [];
let currentStep = 1, abTabMode = 'en', modalSlide = -1; let currentStep = 1, abTabMode = 'en', modalSlide = -1;
@@ -854,7 +855,7 @@ async function savePapersToProject() {
function newProject(){ function newProject(){
papers=[]; selected=new Set(); paperMeta={}; papers=[]; selected=new Set(); paperMeta={};
paperTexts={}; paperTranslations={}; txlStatus={}; paperTexts={}; paperTranslations={}; txlStatus={};
prepLog={}; extractedImages={}; abPaper=null; prepLog={}; extractedImages={}; imageCaptions={}; abPaper=null;
outline=null; outlineEdits={}; pptxRelPath=null; previewImages=[]; outline=null; outlineEdits={}; pptxRelPath=null; previewImages=[];
document.querySelectorAll('.preview-card[data-edited]').forEach(c=>delete c.dataset.edited); document.querySelectorAll('.preview-card[data-edited]').forEach(c=>delete c.dataset.edited);
const sel=document.getElementById('cfg-open-proj'); if(sel) sel.value=''; const sel=document.getElementById('cfg-open-proj'); if(sel) sel.value='';
@@ -907,7 +908,7 @@ async function onOpenProjChange(relPath){
document.querySelectorAll('.preview-card[data-edited]').forEach(c=>delete c.dataset.edited); document.querySelectorAll('.preview-card[data-edited]').forEach(c=>delete c.dataset.edited);
papers=[]; selected=new Set(); paperMeta={}; papers=[]; selected=new Set(); paperMeta={};
paperTexts={}; paperTranslations={}; txlStatus={}; paperTexts={}; paperTranslations={}; txlStatus={};
prepLog={}; extractedImages=[]; abPaper=null; prepLog={}; extractedImages=[]; imageCaptions={}; abPaper=null;
// PPTX 없는 프로젝트 — step 1로 이동해서 논문 목록 확인 // PPTX 없는 프로젝트 — step 1로 이동해서 논문 목록 확인
if(!hasPptx){ if(!hasPptx){
@@ -2071,12 +2072,42 @@ function renderImgGrid(){
const g=document.getElementById('imgrid'); if(!g)return; const g=document.getElementById('imgrid'); if(!g)return;
document.getElementById('imgcnt').textContent='('+extractedImages.length+')'; document.getElementById('imgcnt').textContent='('+extractedImages.length+')';
if(!extractedImages.length){g.innerHTML='<div style="color:var(--muted);font-size:11px">없음</div>';return;} if(!extractedImages.length){g.innerHTML='<div style="color:var(--muted);font-size:11px">없음</div>';return;}
g.innerHTML=extractedImages.map((img,i)=> g.innerHTML=extractedImages.map((img,i)=>{
`<div class="icard" title="${esc(img.label)}" style="position:relative" onmouseenter="this.querySelector('.icard-del').style.opacity=1" onmouseleave="this.querySelector('.icard-del').style.opacity=0"> const cap=imageCaptions[img.url]||'';
const excluded=cap==='제외';
const capHtml=cap?`<div style="font-size:8px;padding:1px 4px;line-height:1.3;color:${excluded?'#f87171':'#34d399'};overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="${esc(cap)}">${excluded?'⛔ 제외':cap}</div>`:'';
return `<div class="icard" title="${esc(img.label)}" style="position:relative;opacity:${excluded?.5:1}" onmouseenter="this.querySelector('.icard-del').style.opacity=1" onmouseleave="this.querySelector('.icard-del').style.opacity=0">
<img src="${esc(img.url)}" loading="lazy" onerror="this.style.display='none'" onclick="showImgPreview('${esc(img.url)}','${esc(img.label)}')" style="cursor:zoom-in;width:100%;height:100%;object-fit:cover"> <img src="${esc(img.url)}" loading="lazy" onerror="this.style.display='none'" onclick="showImgPreview('${esc(img.url)}','${esc(img.label)}')" style="cursor:zoom-in;width:100%;height:100%;object-fit:cover">
<button class="icard-del" onclick="event.stopPropagation();deleteImg(${i})" style="position:absolute;top:4px;right:4px;opacity:0;transition:.15s;background:rgba(220,38,38,.85);border:none;color:#fff;border-radius:4px;width:22px;height:22px;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;line-height:1" title="삭제">✕</button> <button class="icard-del" onclick="event.stopPropagation();deleteImg(${i})" style="position:absolute;top:4px;right:4px;opacity:0;transition:.15s;background:rgba(220,38,38,.85);border:none;color:#fff;border-radius:4px;width:22px;height:22px;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;line-height:1" title="삭제">✕</button>
<div class="ilabel" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:9px;padding:2px 5px">${esc(img.label)}</div> <div class="ilabel" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:9px;padding:2px 5px">${esc(img.label)}</div>
</div>`).join(''); ${capHtml}
</div>`;
}).join('');
}
async function analyzeImages(){
if(!extractedImages.length){setMsg(2,'분석할 이미지가 없습니다.','var(--yellow)');return;}
const btn=document.getElementById('btn-analyze-imgs');
btn.disabled=true;
const project=resolveProjectSlug();
const total=extractedImages.length;
let done=0;
for(const img of extractedImages){
btn.textContent=`🔍 ${done}/${total}`;
if(!imageCaptions[img.url]){
try{
const r=await fetch('/api/pptx/analyze-image',{...jh(),method:'POST',body:JSON.stringify({url:img.url,label:img.label,project})});
const d=await r.json();
if(d.caption) imageCaptions[img.url]=d.caption;
}catch(e){console.warn('[analyzeImages]',e);}
}
done++;
renderImgGrid();
}
btn.disabled=false;
btn.textContent='🔍 이미지 분석';
const excluded=Object.values(imageCaptions).filter(c=>c==='제외').length;
setMsg(2,`이미지 분석 완료 (${total}장${excluded?', 제외 '+excluded+'장':''})`, 'var(--green)');
} }
async function deleteImg(i){ async function deleteImg(i){
@@ -2194,7 +2225,11 @@ async function generateOutline() {
contentBlocks.push(block); contentBlocks.push(block);
} }
const imgList=extractedImages.slice(0,30).map((img,i)=>`${i+1}. ${img.url} (${img.label})`).join('\n'); const imgList=extractedImages
.filter(img=>(imageCaptions[img.url]||'')!=='제외')
.slice(0,30)
.map((img,i)=>{const cap=imageCaptions[img.url]?` — ${imageCaptions[img.url]}`:'';return `${i+1}. ${img.url} (${img.label})${cap}`;})
.join('\n');
const prompt=`[PPTX 워크플로 시작]\n요청: ${topic}\nPMCIDs: ${pmcids}\n템플릿: ${tmpl}${skin?'\n배경 스킨: '+skin:''}\n슬라이드 수: ${slides}장\n\n=== 논문 내용 ===\n${contentBlocks.join('\n\n---\n')}\n\n=== 추출된 이미지 ===\n${imgList||'없음'}\n\n위 논문 내용과 이미지를 활용해 슬라이드 아웃라인을 \`\`\`pptx-outline 블록으로 출력해주세요. create_presentation은 아직 호출하지 마세요.`; const prompt=`[PPTX 워크플로 시작]\n요청: ${topic}\nPMCIDs: ${pmcids}\n템플릿: ${tmpl}${skin?'\n배경 스킨: '+skin:''}\n슬라이드 수: ${slides}장\n\n=== 논문 내용 ===\n${contentBlocks.join('\n\n---\n')}\n\n=== 추출된 이미지 ===\n${imgList||'없음'}\n\n위 논문 내용과 이미지를 활용해 슬라이드 아웃라인을 \`\`\`pptx-outline 블록으로 출력해주세요. create_presentation은 아직 호출하지 마세요.`;