kimandClaude Sonnet 5
7bc1b1bb65
- USER.md/SOUL.md 문자캡 잘림 보완용 벡터 검색(Chroma, EmbeddingGemma) 신설 - 일별 대화로그 LLM 추출 파이프라인(user_facts/daily_extracts 컬렉션 분리) - 검증-강제 게이트(isFactualInfoRequest 등)를 prompt-gates.ts로 분리, 개인 인프라 별명(지서버/클로서버) 관련 불필요한 web_search 강제 호출 버그 수정 - Qwen3-Embedding-8B는 실측 결과 이미지생성 GPU와 자원 충돌 확인되어 폐기, EmbeddingGemma(경량+다국어)로 최종 선택 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
50 lines
2.1 KiB
TypeScript
50 lines
2.1 KiB
TypeScript
// One-off: re-embed existing Chroma collections with a new embedding model. Reuses already-
|
|
// stored document text + metadata + ids — does NOT re-run migrate-memory-to-chroma.ts's USER.md
|
|
// parsing or extract-daily-memory.ts's LLM extraction, since only the embedding vectors need to
|
|
// change, not the underlying facts. Run whenever EMBED_MODEL changes in memory-vector.ts.
|
|
import { ChromaClient } from 'chromadb';
|
|
import { addVector, USER_FACTS_COLLECTION, DAILY_EXTRACTS_COLLECTION } from '../src/gateway/memory-vector';
|
|
|
|
const CHROMA_HOST = 'localhost';
|
|
const CHROMA_PORT = 8100;
|
|
|
|
async function reembedCollection(name: string) {
|
|
const client = new ChromaClient({ host: CHROMA_HOST, port: CHROMA_PORT });
|
|
let existing: { ids: string[]; documents: (string | null)[]; metadatas: any[] } | null = null;
|
|
try {
|
|
const col = await client.getCollection({ name });
|
|
const res = await col.get({ limit: 10000 });
|
|
existing = { ids: res.ids || [], documents: res.documents || [], metadatas: res.metadatas || [] };
|
|
} catch (err: any) {
|
|
console.log(`[${name}] no existing collection or empty (${err.message}) — nothing to reembed`);
|
|
return;
|
|
}
|
|
|
|
console.log(`[${name}] ${existing.ids.length}개 기존 항목 발견, 재임베딩 시작`);
|
|
await client.deleteCollection({ name });
|
|
|
|
let done = 0, failed = 0;
|
|
for (let i = 0; i < existing.ids.length; i++) {
|
|
const id = existing.ids[i];
|
|
const text = existing.documents[i];
|
|
const metadata = existing.metadatas[i] || undefined;
|
|
if (!text) continue;
|
|
try {
|
|
await addVector(name, { id, text, metadata });
|
|
done++;
|
|
if (done % 50 === 0) console.log(`[${name}] ${done}/${existing.ids.length}...`);
|
|
} catch (err: any) {
|
|
failed++;
|
|
console.warn(`[${name}] failed ${id}: ${err.message}`);
|
|
}
|
|
}
|
|
console.log(`[${name}] 완료 — ${done}개 재임베딩, ${failed}개 실패`);
|
|
}
|
|
|
|
async function main() {
|
|
await reembedCollection(USER_FACTS_COLLECTION);
|
|
await reembedCollection(DAILY_EXTRACTS_COLLECTION);
|
|
}
|
|
|
|
main().then(() => process.exit(0)).catch(err => { console.error(err); process.exit(1); });
|