homeclaw/.smallclaw/databases/translate_psychotherapy_abstracts.py

#!/usr/bin/env python3
"""Translate psychotherapy_cases abstract_en to abstract_ko using the Anthropic API."""

import sqlite3
import json
import time
import sys
import re
import html
import urllib.request

DB_PATH = "/home/kim/homeclaw/.smallclaw/databases/psychotherapy_cases.db"
API_URL = "http://localhost:11434/v1/chat/completions"
MODEL = "kimi-k2.6:cloud"


def clean_abstract(text):
    """Remove HTML entities and scraped noise from abstract text."""
    text = html.unescape(text)
    text = re.sub(r'\s*(Downloads|PDF|Published|[0-9]{2}/[0-9]{2}/[0-9]{4})\s*', ' ', text)
    text = re.sub(r'\t+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]{2,}', ' ', text)
    return text.strip()


def translate_abstract(id_, title_en, abstract_en):
    cleaned = clean_abstract(abstract_en)
    if len(cleaned) < 50:
        return None

    prompt = f"""Translate the following psychotherapy research article abstract from English to Korean.

Article title: {title_en}

Abstract:
{cleaned}

Rules:
- Translate the full abstract accurately and professionally
- Keep therapy acronyms (DBT, CBT, AEDP, EFT, ACT, OCD, BPD, PTSD, etc.) as-is
- Keep person names in quotes as-is (e.g., "Jane", "Tommy")
- Keep statistical values and measurements as-is
- Use professional Korean psychology/psychotherapy terminology
- Return ONLY the Korean translation, no explanations
"""

    data = json.dumps({
        "model": MODEL,
        "messages": [
            {"role": "system", "content": "You are a professional Korean-English translator specializing in psychotherapy and clinical psychology. Translate accurately and naturally using standard Korean psychological terminology."},
            {"role": "user", "content": prompt},
        ],
        "temperature": 0.3,
    }).encode("utf-8")

    req = urllib.request.Request(
        API_URL,
        data=data,
        headers={"Content-Type": "application/json"},
    )

    try:
        with urllib.request.urlopen(req, timeout=180) as resp:
            result = json.loads(resp.read().decode("utf-8"))
            return result["choices"][0]["message"]["content"].strip()
    except Exception as e:
        print(f"  API error for id={id_}: {e}", file=sys.stderr, flush=True)
        return None


def main():
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()

    c.execute("""SELECT id, title_en, abstract_en FROM cases
                 WHERE (abstract_ko = '' OR abstract_ko IS NULL)
                   AND abstract_en IS NOT NULL AND abstract_en != ''
                 ORDER BY id""")
    rows = c.fetchall()

    if not rows:
        print("All abstracts translated!")
        conn.close()
        return

    print(f"Translating {len(rows)} abstracts...", flush=True)

    total_translated = 0
    for i, (id_, title_en, abstract_en) in enumerate(rows):
        print(f"\n[{i+1}/{len(rows)}] id={id_}: {title_en[:60]}...", flush=True)

        abstract_ko = translate_abstract(id_, title_en, abstract_en)
        if not abstract_ko:
            print("  Failed, skipping", flush=True)
            continue

        c.execute("UPDATE cases SET abstract_ko = ? WHERE id = ?", (abstract_ko, id_))
        conn.commit()
        total_translated += 1
        print(f"  Done ({len(abstract_ko)} chars)", flush=True)
        time.sleep(3)

    c.execute("SELECT COUNT(*) FROM cases WHERE abstract_ko != '' AND abstract_ko IS NOT NULL")
    done = c.fetchone()[0]
    c.execute("SELECT COUNT(*) FROM cases WHERE (abstract_ko = '' OR abstract_ko IS NULL) AND abstract_en IS NOT NULL AND abstract_en != ''")
    remaining = c.fetchone()[0]

    print(f"\nDone! Translated: {done} total, {remaining} remaining with English abstracts", flush=True)

    print("\n--- Sample translations ---")
    c.execute("""SELECT title_en, abstract_ko FROM cases
                 WHERE abstract_ko != '' AND abstract_ko IS NOT NULL
                 ORDER BY RANDOM() LIMIT 3""")
    for row in c.fetchall():
        print(f"  Title: {row[0][:70]}")
        print(f"  KO:    {row[1][:200]}...")
        print()

    conn.close()


if __name__ == "__main__":
    main()