Release 2.9.4: PPTX wizard figure analysis + skill model trigger fix
- pptx-wizard: add "🔍 이미지 분석" button in Step 2 image panel * calls /api/pptx/analyze-image per figure via mistral vision * shows per-image progress (1/10, 2/10...) * "제외" caption → dimmed card + filtered from outline prompt * captions shown below each thumbnail in the grid - server: POST /api/pptx/analyze-image endpoint * resolves /api/files/ path to workspace file, thumbnails if >300KB * calls orchestration.secondary vision model (mistral-large-3:675b-cloud) * caches results in project/.image-captions.json (skips on re-analysis) - pptx-wizard: generateOutline() includes captions in imgList * "제외" images filtered out before sending to LLM * format: "/api/files/... (figure_p2_1) — 카플란-마이어 생존곡선 3그룹" - skills-manager: fix getModelOverrideForUser() trigger filtering * was returning skill model for ALL messages regardless of trigger keywords * now only overrides when message matches skill triggers (same logic as buildPromptContext) * presenter model (mistral) now only applies for 발표/pptx/슬라이드 keywords Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -6510,7 +6510,7 @@ RULES:
|
||||
// Model priority: explicit request > vision fallback > skill override > config default
|
||||
const effectiveModel = String(modelOverride || '').trim()
|
||||
|| _visionFallbackModel
|
||||
|| skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null)
|
||||
|| skillsManager.getModelOverrideForUser(username ? getUserWorkspace(username) : null, message)
|
||||
|| undefined;
|
||||
// Always use streaming so text tokens appear in real-time. The streaming
|
||||
// handler collects tool_calls from chunks and returns them in the final
|
||||
@@ -9942,6 +9942,85 @@ app.post('/api/pptx/extract-images', requireGatewayAuth, async (req: express.Req
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/pptx/analyze-image — vision-model caption for a single extracted figure
|
||||
// Caches results in workspace/pptx/<project>/.image-captions.json
|
||||
app.post('/api/pptx/analyze-image', requireGatewayAuth, async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const session = getSessionUser(req);
|
||||
const username = (session as any)?.username;
|
||||
const workspacePath = username ? getUserWorkspace(username) : getConfig().getWorkspacePath();
|
||||
|
||||
const imgUrl = String(req.body?.url || '').trim();
|
||||
const project = pptxProjectSlug(String(req.body?.project || ''));
|
||||
if (!imgUrl) { res.status(400).json({ error: 'url required' }); return; }
|
||||
|
||||
// Load cache
|
||||
const cacheFile = project
|
||||
? path.join(workspacePath, PPTX_BASE, project, '.image-captions.json')
|
||||
: null;
|
||||
let cached: Record<string, string> = {};
|
||||
if (cacheFile && fs.existsSync(cacheFile)) {
|
||||
try { cached = JSON.parse(fs.readFileSync(cacheFile, 'utf-8')); } catch {}
|
||||
}
|
||||
if (cached[imgUrl]) {
|
||||
res.json({ success: true, url: imgUrl, caption: cached[imgUrl], fromCache: true });
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve file path
|
||||
const relParts = imgUrl.replace(/^\/api\/files\//, '').split('/').map(decodeURIComponent);
|
||||
const filePath = path.resolve(workspacePath, ...relParts);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
res.json({ success: false, url: imgUrl, caption: '' });
|
||||
return;
|
||||
}
|
||||
|
||||
// Read + thumbnail if large
|
||||
let buf: Buffer | null = null;
|
||||
const stat = fs.statSync(filePath);
|
||||
if (stat.size <= 300_000) {
|
||||
buf = fs.readFileSync(filePath);
|
||||
} else {
|
||||
try {
|
||||
const { execSync } = await import('child_process');
|
||||
const script = `from PIL import Image; import sys; img=Image.open(sys.argv[1]); img.thumbnail((600,600)); img.save(sys.stdout.buffer,'JPEG',quality=80)`;
|
||||
buf = execSync(`python3 -c "${script}" "${filePath}"`, { maxBuffer: 5 * 1024 * 1024 });
|
||||
} catch { buf = null; }
|
||||
}
|
||||
if (!buf) { res.json({ success: false, url: imgUrl, caption: '' }); return; }
|
||||
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const mime = ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : 'image/jpeg';
|
||||
const b64 = `data:${mime};base64,${buf.toString('base64')}`;
|
||||
|
||||
// Call vision model
|
||||
const ollama2 = getOllamaClient();
|
||||
const rawCfg2 = getConfig().getConfig() as any;
|
||||
const sec2 = rawCfg2.orchestration?.secondary;
|
||||
const visionModel = (sec2?.model && sec2?.vision === true) ? sec2.model : 'mistral-large-3:675b-cloud';
|
||||
|
||||
const result = await ollama2.chatWithThinking([{
|
||||
role: 'user' as const,
|
||||
content: [
|
||||
{ type: 'text' as const, text: '이 그림을 한 줄(30자 이내)로 설명하세요. 그래프/차트/다이어그램 종류와 핵심 내용 위주로. 슬라이드에 쓰기 부적합한 그림(흐릿함·텍스트만 있는 표·빈 이미지)이면 "제외"라고만 답하세요.' },
|
||||
{ type: 'image_url' as const, image_url: { url: b64 } },
|
||||
],
|
||||
}], 'executor', { model: visionModel, num_predict: 80 });
|
||||
|
||||
const caption = String(result.message?.content || '').trim().replace(/\n/g, ' ').slice(0, 80);
|
||||
cached[imgUrl] = caption;
|
||||
if (cacheFile) {
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(cacheFile), { recursive: true });
|
||||
fs.writeFileSync(cacheFile, JSON.stringify(cached, null, 2), 'utf-8');
|
||||
} catch {}
|
||||
}
|
||||
res.json({ success: true, url: imgUrl, caption });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/pptx/project-pdfs?project=<slug> — list PDF files in project folder
|
||||
// Also returns companion .txt / .ko.txt paths if they exist.
|
||||
app.get('/api/pptx/project-pdfs', requireGatewayAuth, async (req: express.Request, res: express.Response) => {
|
||||
|
||||
@@ -286,8 +286,9 @@ export class SkillsManager {
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/** Return the model override from the first enabled skill that specifies one, or undefined. */
|
||||
getModelOverrideForUser(userDir: string | null): string | undefined {
|
||||
/** Return the model override from the first triggered+enabled skill that specifies one, or undefined. */
|
||||
getModelOverrideForUser(userDir: string | null, userMessage = ''): string | undefined {
|
||||
const msgLower = userMessage.toLowerCase();
|
||||
const userState = userDir ? this.getUserState(userDir) : null;
|
||||
const enabled = this.getAll().filter(s => {
|
||||
if (this.GLOBAL_ONLY_SKILLS.has(s.id)) return s.enabled;
|
||||
@@ -295,7 +296,12 @@ export class SkillsManager {
|
||||
return s.enabled;
|
||||
});
|
||||
for (const skill of enabled) {
|
||||
if (skill.model) return skill.model;
|
||||
if (!skill.model) continue;
|
||||
// Only apply model override when the skill is triggered (or has no triggers)
|
||||
if (skill.triggers && skill.triggers.length > 0) {
|
||||
if (!skill.triggers.some(t => msgLower.includes(t.toLowerCase()))) continue;
|
||||
}
|
||||
return skill.model;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
+42
-7
@@ -343,6 +343,7 @@ body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,san
|
||||
<div style="flex:2;min-width:0">
|
||||
<div style="display:flex;align-items:center;gap:8px;margin-bottom:6px">
|
||||
<span style="font-size:10px;color:var(--muted);font-weight:600;text-transform:uppercase;letter-spacing:.04em">캡처 이미지 <span id="imgcnt"></span></span>
|
||||
<button id="btn-analyze-imgs" onclick="analyzeImages()" style="font-size:11px;color:#10b981;background:rgba(16,185,129,.1);border:1px solid #10b981;border-radius:5px;padding:2px 8px;cursor:pointer;flex-shrink:0" title="AI(mistral vision)가 각 이미지 내용 분석 — 아웃라인 생성 시 정확한 슬라이드 배치에 활용">🔍 이미지 분석</button>
|
||||
<label style="display:flex;align-items:center;gap:4px;font-size:11px;color:var(--brand2);background:rgba(99,102,241,.1);border:1px solid var(--brand);border-radius:5px;padding:2px 8px;cursor:pointer;flex-shrink:0" title="이미지 파일 업로드">
|
||||
<input type="file" id="img-upload-input" accept="image/*" multiple style="display:none" onchange="handleImgUpload(this.files)">
|
||||
＋ 이미지 업로드
|
||||
@@ -762,7 +763,7 @@ function updSrcState(){
|
||||
const checks=[...document.querySelectorAll('.src-chk input')];
|
||||
if(checks.every(c=>!c.checked)) checks[0].checked=true;
|
||||
}
|
||||
let extractedImages = [], paperTexts = {};
|
||||
let extractedImages = [], paperTexts = {}, imageCaptions = {};
|
||||
let prepLog = {}, outline = null, outlineEdits = {}, currentProcessingKey = null;
|
||||
let pptxRelPath = null, previewImages = [];
|
||||
let currentStep = 1, abTabMode = 'en', modalSlide = -1;
|
||||
@@ -854,7 +855,7 @@ async function savePapersToProject() {
|
||||
function newProject(){
|
||||
papers=[]; selected=new Set(); paperMeta={};
|
||||
paperTexts={}; paperTranslations={}; txlStatus={};
|
||||
prepLog={}; extractedImages={}; abPaper=null;
|
||||
prepLog={}; extractedImages={}; imageCaptions={}; abPaper=null;
|
||||
outline=null; outlineEdits={}; pptxRelPath=null; previewImages=[];
|
||||
document.querySelectorAll('.preview-card[data-edited]').forEach(c=>delete c.dataset.edited);
|
||||
const sel=document.getElementById('cfg-open-proj'); if(sel) sel.value='';
|
||||
@@ -907,7 +908,7 @@ async function onOpenProjChange(relPath){
|
||||
document.querySelectorAll('.preview-card[data-edited]').forEach(c=>delete c.dataset.edited);
|
||||
papers=[]; selected=new Set(); paperMeta={};
|
||||
paperTexts={}; paperTranslations={}; txlStatus={};
|
||||
prepLog={}; extractedImages=[]; abPaper=null;
|
||||
prepLog={}; extractedImages=[]; imageCaptions={}; abPaper=null;
|
||||
|
||||
// PPTX 없는 프로젝트 — step 1로 이동해서 논문 목록 확인
|
||||
if(!hasPptx){
|
||||
@@ -2071,12 +2072,42 @@ function renderImgGrid(){
|
||||
const g=document.getElementById('imgrid'); if(!g)return;
|
||||
document.getElementById('imgcnt').textContent='('+extractedImages.length+')';
|
||||
if(!extractedImages.length){g.innerHTML='<div style="color:var(--muted);font-size:11px">없음</div>';return;}
|
||||
g.innerHTML=extractedImages.map((img,i)=>
|
||||
`<div class="icard" title="${esc(img.label)}" style="position:relative" onmouseenter="this.querySelector('.icard-del').style.opacity=1" onmouseleave="this.querySelector('.icard-del').style.opacity=0">
|
||||
g.innerHTML=extractedImages.map((img,i)=>{
|
||||
const cap=imageCaptions[img.url]||'';
|
||||
const excluded=cap==='제외';
|
||||
const capHtml=cap?`<div style="font-size:8px;padding:1px 4px;line-height:1.3;color:${excluded?'#f87171':'#34d399'};overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="${esc(cap)}">${excluded?'⛔ 제외':cap}</div>`:'';
|
||||
return `<div class="icard" title="${esc(img.label)}" style="position:relative;opacity:${excluded?.5:1}" onmouseenter="this.querySelector('.icard-del').style.opacity=1" onmouseleave="this.querySelector('.icard-del').style.opacity=0">
|
||||
<img src="${esc(img.url)}" loading="lazy" onerror="this.style.display='none'" onclick="showImgPreview('${esc(img.url)}','${esc(img.label)}')" style="cursor:zoom-in;width:100%;height:100%;object-fit:cover">
|
||||
<button class="icard-del" onclick="event.stopPropagation();deleteImg(${i})" style="position:absolute;top:4px;right:4px;opacity:0;transition:.15s;background:rgba(220,38,38,.85);border:none;color:#fff;border-radius:4px;width:22px;height:22px;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;line-height:1" title="삭제">✕</button>
|
||||
<div class="ilabel" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:9px;padding:2px 5px">${esc(img.label)}</div>
|
||||
</div>`).join('');
|
||||
${capHtml}
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
async function analyzeImages(){
|
||||
if(!extractedImages.length){setMsg(2,'분석할 이미지가 없습니다.','var(--yellow)');return;}
|
||||
const btn=document.getElementById('btn-analyze-imgs');
|
||||
btn.disabled=true;
|
||||
const project=resolveProjectSlug();
|
||||
const total=extractedImages.length;
|
||||
let done=0;
|
||||
for(const img of extractedImages){
|
||||
btn.textContent=`🔍 ${done}/${total}`;
|
||||
if(!imageCaptions[img.url]){
|
||||
try{
|
||||
const r=await fetch('/api/pptx/analyze-image',{...jh(),method:'POST',body:JSON.stringify({url:img.url,label:img.label,project})});
|
||||
const d=await r.json();
|
||||
if(d.caption) imageCaptions[img.url]=d.caption;
|
||||
}catch(e){console.warn('[analyzeImages]',e);}
|
||||
}
|
||||
done++;
|
||||
renderImgGrid();
|
||||
}
|
||||
btn.disabled=false;
|
||||
btn.textContent='🔍 이미지 분석';
|
||||
const excluded=Object.values(imageCaptions).filter(c=>c==='제외').length;
|
||||
setMsg(2,`이미지 분석 완료 (${total}장${excluded?', 제외 '+excluded+'장':''})`, 'var(--green)');
|
||||
}
|
||||
|
||||
async function deleteImg(i){
|
||||
@@ -2194,7 +2225,11 @@ async function generateOutline() {
|
||||
contentBlocks.push(block);
|
||||
}
|
||||
|
||||
const imgList=extractedImages.slice(0,30).map((img,i)=>`${i+1}. ${img.url} (${img.label})`).join('\n');
|
||||
const imgList=extractedImages
|
||||
.filter(img=>(imageCaptions[img.url]||'')!=='제외')
|
||||
.slice(0,30)
|
||||
.map((img,i)=>{const cap=imageCaptions[img.url]?` — ${imageCaptions[img.url]}`:'';return `${i+1}. ${img.url} (${img.label})${cap}`;})
|
||||
.join('\n');
|
||||
|
||||
const prompt=`[PPTX 워크플로 시작]\n요청: ${topic}\nPMCIDs: ${pmcids}\n템플릿: ${tmpl}${skin?'\n배경 스킨: '+skin:''}\n슬라이드 수: ${slides}장\n\n=== 논문 내용 ===\n${contentBlocks.join('\n\n---\n')}\n\n=== 추출된 이미지 ===\n${imgList||'없음'}\n\n위 논문 내용과 이미지를 활용해 슬라이드 아웃라인을 \`\`\`pptx-outline 블록으로 출력해주세요. create_presentation은 아직 호출하지 마세요.`;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user