kimandClaude Sonnet 5
70c1e1c23f
Chroma 벡터검색만으로는 IP·모델명 같은 정확한 용어를 놓치는 경우가 있어 SQLite FTS5(trigram) 키워드검색을 병합하고, 애매한 경우(예: 클로서버 vs 지서버 GPU 스펙 혼동)만 LLM 재랭킹으로 오답을 걸러내도록 함. 재랭킹은 후보가 없거나 확실한 단일매치일 때는 건너뛰어 대부분의 대화에서는 지연시간 증가가 거의 없음. 기존 570개 기록은 scripts/backfill-fts.ts로 백필. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
40 lines
1.6 KiB
TypeScript
40 lines
1.6 KiB
TypeScript
// One-time backfill: mirror existing Chroma vector-memory records into the new FTS5
|
|
// keyword-search sidecar (memory-fts.ts, added 2026-08-06 hybrid-search upgrade). Only
|
|
// needed once — addVector() keeps the two in sync for everything written after this ran.
|
|
import { ChromaClient } from 'chromadb';
|
|
import { USER_FACTS_COLLECTION, DAILY_EXTRACTS_COLLECTION } from '../src/gateway/memory/memory-vector';
|
|
import { upsertFtsRecord } from '../src/gateway/memory/memory-fts';
|
|
|
|
async function backfillCollection(client: ChromaClient, name: string) {
|
|
const col = await client.getOrCreateCollection({ name, embeddingFunction: null });
|
|
const count = await col.count();
|
|
console.log(`[${name}] ${count} records`);
|
|
const batchSize = 200;
|
|
let offset = 0;
|
|
let written = 0;
|
|
while (offset < count) {
|
|
const res = await col.get({ limit: batchSize, offset });
|
|
const ids = res.ids || [];
|
|
const docs = res.documents || [];
|
|
const metas = res.metadatas || [];
|
|
for (let i = 0; i < ids.length; i++) {
|
|
const text = docs[i] || '';
|
|
if (!text) continue;
|
|
const workspace = String((metas[i] as any)?.workspace || '');
|
|
upsertFtsRecord(name, ids[i], text, workspace);
|
|
written++;
|
|
}
|
|
offset += batchSize;
|
|
}
|
|
console.log(`[${name}] backfilled ${written} records into FTS`);
|
|
}
|
|
|
|
async function main() {
|
|
const client = new ChromaClient({ host: 'localhost', port: 8100 });
|
|
await backfillCollection(client, USER_FACTS_COLLECTION);
|
|
await backfillCollection(client, DAILY_EXTRACTS_COLLECTION);
|
|
console.log('done');
|
|
}
|
|
|
|
main().catch(e => { console.error(e); process.exit(1); });
|