From 1bfadd6060be19fde5943863724b753529eecb65 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 21 May 2026 13:54:15 +0900 Subject: [PATCH] Add MIDI editor + fix external synth instrument mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - web-ui/midi-editor.html: full piano roll MIDI editor (new tab) - all tracks simultaneously, color-coded per channel - note add/delete/move/resize, multi-select, copy/paste - velocity lane, track panel (mute/solo/instrument), undo/redo - GM reset + bank select on playback, MIDI output device select - export to .mid, open via ?url= param or file picker - web-ui/index.html: add ✏ 편집 button to MIDI widget (opens editor) - GM SysEx reset + bank select before note playback - 200ms note delay for synth init time - getRenderFile: prefer original.midi over LilyPond output.midi - src/gateway/music-renderer.ts: getRenderFile returns original.midi first - LilyPond SVG render overwrites output.midi (no Program Changes) - original.midi preserved from upload, served via /api/music/file/:id/midi Co-Authored-By: Claude Sonnet 4.6 --- src/gateway/music-renderer.ts | 450 +++++++++++++++++ web-ui/index.html | 626 ++++++++++++++++++++++- web-ui/midi-editor.html | 901 ++++++++++++++++++++++++++++++++++ 3 files changed, 1966 insertions(+), 11 deletions(-) create mode 100644 src/gateway/music-renderer.ts create mode 100644 web-ui/midi-editor.html diff --git a/src/gateway/music-renderer.ts b/src/gateway/music-renderer.ts new file mode 100644 index 0000000..d2b3f3d --- /dev/null +++ b/src/gateway/music-renderer.ts @@ -0,0 +1,450 @@ +/** + * music-renderer.ts + * LilyPond-based sheet music renderer. + * Accepts ABC notation, LilyPond source, or MIDI file path. + * Returns SVG (cropped preview) + MIDI + PDF file paths. + */ + +import { spawnSync, spawn } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import * as crypto from 'crypto'; + +export interface RenderResult { + id: string; + svgPath: string; + svgPages: string[]; // all page SVG paths (length >= 1) + pdfPath: string | null; + midiPath: string | null; +} + +const LILYPOND_BIN = '/usr/bin/lilypond'; +const ABC2LY_BIN = '/usr/bin/abc2ly'; +const MIDI2LY_BIN = '/usr/bin/midi2ly'; + +const RENDER_DIR = path.join(os.tmpdir(), 'homeclaw-music'); + +function ensureRenderDir() { + if (!fs.existsSync(RENDER_DIR)) fs.mkdirSync(RENDER_DIR, { recursive: true }); +} + +function detectFormat(source: string): 'abc' | 'lilypond' { + const trimmed = source.trimStart(); + if (/^(X:|T:|M:|L:|K:|%abc)/m.test(trimmed)) return 'abc'; + if (trimmed.includes('\\version') || trimmed.includes('\\score') || trimmed.includes('\\relative')) return 'lilypond'; + if (/\\(clef|key|time|new Staff|Staff)/.test(trimmed)) return 'lilypond'; + return 'abc'; +} + +function wrapLilypond(source: string, tempo = 120): string { + if (source.includes('\\score')) { + if (!source.includes('\\midi')) { + return source.replace(/\\layout\s*\{[^}]*\}/, m => m + `\n \\midi { \\tempo 4 = ${tempo} }`); + } + return source; + } + return `\\version "2.24.0" +\\score { + \\new Staff { + ${source} + } + \\layout { } + \\midi { \\tempo 4 = ${tempo} } +} +`; +} + +function abcToLilypond(abcSource: string, workDir: string): string { + const abcFile = path.join(workDir, 'input.abc'); + const lyFile = path.join(workDir, 'input.ly'); + fs.writeFileSync(abcFile, abcSource, 'utf8'); + + const result = spawnSync(ABC2LY_BIN, ['-o', lyFile, abcFile], { + cwd: workDir, timeout: 15_000, encoding: 'utf8', + }); + if (result.status !== 0) throw new Error(`abc2ly failed: ${result.stderr || result.stdout}`); + if (!fs.existsSync(lyFile)) throw new Error('abc2ly produced no output file'); + + let ly = fs.readFileSync(lyFile, 'utf8'); + if (!ly.includes('\\midi')) { + ly = ly.replace(/\\layout\s*\{[^}]*\}/, m => m + '\n \\midi { \\tempo 4 = 120 }'); + if (!ly.includes('\\midi')) { + ly = ly.replace(/(\\\s*score\s*\{)([\s\S]*?)(\}\s*)$/, (_, open, body, close) => + open + body + ' \\midi { \\tempo 4 = 120 }\n' + close); + } + fs.writeFileSync(lyFile, ly, 'utf8'); + } + return lyFile; +} + +const LILYPOND_PYTHON_DIR = '/usr/share/lilypond/2.24.4/python'; + +/** + * Write a patched midi.py into workDir and a thin wrapper script. + * midi2ly does sys.path.insert(0, lilypond_python_dir) which overrides PYTHONPATH, + * so we pre-load our patched midi into sys.modules before exec'ing midi2ly. + * midi2ly's own "import midi" then finds sys.modules['midi'] and skips the import. + */ +function ensureMidi2lyPatch(workDir: string): void { + const patchedMidiPath = path.join(workDir, 'midi.py'); + const wrapperPath = path.join(workDir, 'midi2ly_wrapper.py'); + if (fs.existsSync(wrapperPath)) return; + + // Read original midi.py and patch _read_string to tolerate truncated SysEx. + const originalMidi = fs.readFileSync(path.join(LILYPOND_PYTHON_DIR, 'midi.py'), 'utf8'); + const patched = originalMidi.replace( + /def _read_string\(nextbyte, getbyte\):\n length = _get_variable_length_number\(nextbyte, getbyte\)\n return ''.join\(chr\(getbyte\(\)\) for i in range\(length\)\)/, + [ + 'def _read_string(nextbyte, getbyte):', + ' length = _get_variable_length_number(nextbyte, getbyte)', + ' out = []', + ' for _ in range(length):', + ' try:', + ' out.append(chr(getbyte()))', + ' except (StopIteration, RuntimeError):', + ' break', + ' return \'\'.join(out)', + ].join('\n'), + ); + fs.writeFileSync(patchedMidiPath, patched, 'utf8'); + + // Wrapper: pre-import patched midi, then exec the real midi2ly + const wrapper = [ + 'import sys, os', + `sys.path.insert(0, ${JSON.stringify(workDir)})`, + 'import midi # loads patched version into sys.modules', + '# exec real midi2ly — its own "import midi" finds sys.modules cache', + `with open(${JSON.stringify(MIDI2LY_BIN)}) as _f:`, + ` exec(compile(_f.read(), ${JSON.stringify(MIDI2LY_BIN)}, 'exec'),`, + ` {'__file__': ${JSON.stringify(MIDI2LY_BIN)}, '__name__': '__main__'})`, + ].join('\n'); + fs.writeFileSync(wrapperPath, wrapper, 'utf8'); +} + +/** Convert a MIDI file to LilyPond via midi2ly (with SysEx-tolerant patch). */ +function midiToLilypond(midiPath: string, workDir: string): string { + const lyFile = path.join(workDir, 'input.ly'); + ensureMidi2lyPatch(workDir); + const wrapperPath = path.join(workDir, 'midi2ly_wrapper.py'); + const result = spawnSync('python3', [wrapperPath, midiPath, '-o', lyFile], { + cwd: workDir, timeout: 15_000, encoding: 'utf8', + }); + if (result.status !== 0) throw new Error(`midi2ly failed: ${result.stderr || result.stdout}`); + if (!fs.existsSync(lyFile)) throw new Error('midi2ly produced no output file'); + + // Fix old version string that midi2ly emits (2.14.0 → 2.24.0) + let ly = fs.readFileSync(lyFile, 'utf8'); + ly = ly.replace(/\\version\s+"[^"]*"/, '\\version "2.24.0"'); + // Remove bar check assertions ("| % N") that midi2ly emits — these fail on + // irregular time signatures. Keep the comment but drop the assertion. + ly = ly.replace(/\|\s*(%\s*\d+)/g, '$1'); + + // Compact rendering: staff size 10, tight margins, and compressed note spacing. + // Remove any \paper block midi2ly may have emitted, then inject our own. + ly = ly.replace(/\\paper\s*\{[^}]*\}/s, ''); + + // Count staves to decide spacing strategy. + const trackCount = (ly.match(/^track[A-Z] = < 2; + + const compactBlock = [ + `#(set-global-staff-size ${isMultiStaff ? 10 : 12})`, + '\\paper {', + ' left-margin = 0.5\\cm', + ' right-margin = 0.5\\cm', + ' top-margin = 0.5\\cm', + ' bottom-margin = 0.5\\cm', + ' ragged-last-bottom = ##f', + ' system-system-spacing =', + ' #\'((basic-distance . 8) (minimum-distance . 4) (padding . 1) (stretchability . 60))', + '}', + '% compress horizontal note spacing', + '\\layout {', + ' \\context {', + ' \\Score', + ' \\override SpacingSpanner.base-shortest-duration =', + ' #(ly:make-moment 1/32)', + ' \\override SpacingSpanner.shortest-duration-space = #0.6', + ' }', + '}', + ].join('\n'); + ly = ly.replace(/(\\version\s+"[^"]*")/, `$1\n${compactBlock}`); + + // Limit voices per track to 4 — more than 4 causes overlapping noteheads. + ly = ly.replace(/^(track[A-Z] = <<\n)([\s\S]*?)^(>>)/gm, (match, open, inner, close) => { + const voiceLines = inner.split('\n').filter((l: string) => /^\s+\\context Voice/.test(l)); + if (voiceLines.length <= 4) return match; + return `${open}${voiceLines.slice(0, 4).join('\n')}\n${close}`; + }); + + // Add layout block if missing + if (!ly.includes('\\layout')) { + ly += '\n\\score {\n \\new Staff { \\unfoldRepeats \\midi {} }\n \\layout {}\n}\n'; + } + fs.writeFileSync(lyFile, ly, 'utf8'); + return lyFile; +} + +/** Collect all SVG page files produced by LilyPond in workDir. */ +function collectSvgPages(workDir: string): string[] { + const previewSvg = path.join(workDir, 'output.preview.svg'); + if (fs.existsSync(previewSvg)) return [previewSvg]; + + const fullSvg = path.join(workDir, 'output.svg'); + if (fs.existsSync(fullSvg)) return [fullSvg]; + + // Multi-page: output-1.svg, output-2.svg, ... + const pages: string[] = []; + for (let i = 1; ; i++) { + const p = path.join(workDir, `output-${i}.svg`); + if (!fs.existsSync(p)) break; + pages.push(p); + } + return pages; +} + +/** Run LilyPond: SVG + PDF + MIDI. + * preview=true → -dpreview --svg (compact single-page crop for short ABC/LilyPond) + * preview=false → --svg (all pages, for MIDI or long scores) + */ +function runLilypond(lyFile: string, workDir: string, preview = true): { svg: string; pages: string[]; pdf: string | null; midi: string | null } { + const svgArgs = preview + ? ['-dpreview', '--svg', '-o', 'output', lyFile] + : ['--svg', '-o', 'output', lyFile]; + const r1 = spawnSync(LILYPOND_BIN, svgArgs, { + cwd: workDir, timeout: 60_000, encoding: 'utf8', + }); + + // Collect SVG pages regardless of exit status — LilyPond often exits 1 on + // warnings (e.g. barcheck) but still writes complete output files. + const pages = collectSvgPages(workDir); + if (pages.length === 0) { + throw new Error(`LilyPond (svg) failed: ${(r1.stderr || r1.stdout || '').slice(0, 2000)}`); + } + + // Pass 2: PDF + spawnSync(LILYPOND_BIN, ['--pdf', '-o', 'output', lyFile], { + cwd: workDir, timeout: 60_000, encoding: 'utf8', + }); + // PDF failure is non-fatal + + const pdfPath = path.join(workDir, 'output.pdf'); + const midiPath = path.join(workDir, 'output.midi'); + + return { + svg: pages[0], + pages, + pdf: fs.existsSync(pdfPath) ? pdfPath : null, + midi: fs.existsSync(midiPath) ? midiPath : null, + }; +} + +/** Render ABC or LilyPond source → SVG + PDF + MIDI. */ +export function renderMusic(source: string): RenderResult { + ensureRenderDir(); + + const id = crypto.createHash('sha1').update(source).digest('hex').slice(0, 12); + const workDir = path.join(RENDER_DIR, id); + + const cached = collectSvgPages(workDir); + if (cached.length > 0) { + const pdfOut = path.join(workDir, 'output.pdf'); + const midiOut = path.join(workDir, 'output.midi'); + return { + id, + svgPath: cached[0], + svgPages: cached, + pdfPath: fs.existsSync(pdfOut) ? pdfOut : null, + midiPath: fs.existsSync(midiOut) ? midiOut : null, + }; + } + + fs.mkdirSync(workDir, { recursive: true }); + + const fmt = detectFormat(source); + const lyFile = fmt === 'abc' + ? abcToLilypond(source, workDir) + : (() => { const f = path.join(workDir, 'input.ly'); fs.writeFileSync(f, wrapLilypond(source), 'utf8'); return f; })(); + + const { svg, pages, pdf, midi } = runLilypond(lyFile, workDir); + return { id, svgPath: svg, svgPages: pages, pdfPath: pdf, midiPath: midi }; +} + +/** Render a MIDI file → SVG + PDF (no MIDI output). */ +export function renderMidi(midiFilePath: string): RenderResult { + ensureRenderDir(); + + const content = fs.readFileSync(midiFilePath); + const id = crypto.createHash('sha1').update(content).digest('hex').slice(0, 12); + const workDir = path.join(RENDER_DIR, 'midi-' + id); + + const cached = collectSvgPages(workDir); + if (cached.length > 0) { + const pdfOut = path.join(workDir, 'output.pdf'); + const originalOut = path.join(workDir, 'original.midi'); + const midiOut = path.join(workDir, 'output.midi'); + return { + id: 'midi-' + id, + svgPath: cached[0], + svgPages: cached, + pdfPath: fs.existsSync(pdfOut) ? pdfOut : null, + midiPath: fs.existsSync(originalOut) ? originalOut : + fs.existsSync(midiOut) ? midiOut : midiFilePath, + }; + } + + fs.mkdirSync(workDir, { recursive: true }); + // Save original MIDI as original.midi — LilyPond overwrites output.midi during SVG render + const originalMidi = path.join(workDir, 'original.midi'); + const midiCopy = path.join(workDir, 'output.midi'); + fs.copyFileSync(midiFilePath, originalMidi); + fs.copyFileSync(midiFilePath, midiCopy); + const lyFile = midiToLilypond(midiCopy, workDir); + const { svg, pages, pdf } = runLilypond(lyFile, workDir, false); + + return { id: 'midi-' + id, svgPath: svg, svgPages: pages, pdfPath: pdf, midiPath: originalMidi }; +} + +/** Serve a specific SVG page (1-indexed). */ +export function getRenderPageFile(id: string, page: number): string | null { + const workDir = path.join(RENDER_DIR, id); + // Multi-page output + const paged = path.join(workDir, `output-${page}.svg`); + if (fs.existsSync(paged)) return paged; + // Single-page: only valid for page 1 + if (page === 1) return getRenderFile(id, 'svg'); + return null; +} + +/** Serve a rendered file. ext: 'svg' | 'midi' | 'pdf' */ +export function getRenderFile(id: string, ext: 'svg' | 'midi' | 'pdf'): string | null { + const workDir = path.join(RENDER_DIR, id); + if (ext === 'svg') { + const preview = path.join(workDir, 'output.preview.svg'); + const full = path.join(workDir, 'output.svg'); + const page1 = path.join(workDir, 'output-1.svg'); + if (fs.existsSync(preview)) return preview; + if (fs.existsSync(full)) return full; + if (fs.existsSync(page1)) return page1; + return null; + } + if (ext === 'midi') { + const orig = path.join(workDir, 'original.midi'); + const out = path.join(workDir, 'output.midi'); + if (fs.existsSync(orig)) return orig; + if (fs.existsSync(out)) return out; + return null; + } + const filePath = path.join(workDir, `output.${ext}`); + return fs.existsSync(filePath) ? filePath : null; +} + +/** Async MIDI → sheet music render with progress callbacks. No timeout. */ +export function renderMidiAsync( + midiFilePath: string, + onProgress: (msg: string) => void, + onDone: (result: RenderResult) => void, + onError: (err: Error) => void, +): void { + ensureRenderDir(); + + let content: Buffer; + try { content = fs.readFileSync(midiFilePath); } + catch (e: any) { onError(e); return; } + + const id = crypto.createHash('sha1').update(content).digest('hex').slice(0, 12); + const workDir = path.join(RENDER_DIR, 'midi-' + id); + + // Cache hit + const cached = collectSvgPages(workDir); + if (cached.length > 0) { + const pdfOut = path.join(workDir, 'output.pdf'); + const midiOut = path.join(workDir, 'output.midi'); + onProgress('캐시에서 불러오는 중...'); + onDone({ + id: 'midi-' + id, + svgPath: cached[0], + svgPages: cached, + pdfPath: fs.existsSync(pdfOut) ? pdfOut : null, + midiPath: fs.existsSync(midiOut) ? midiOut : midiFilePath, + }); + return; + } + + fs.mkdirSync(workDir, { recursive: true }); + // Save original as original.midi — LilyPond overwrites output.midi during SVG render + const originalMidi = path.join(workDir, 'original.midi'); + const midiCopy = path.join(workDir, 'output.midi'); + try { + fs.copyFileSync(midiFilePath, originalMidi); + fs.copyFileSync(midiFilePath, midiCopy); + } catch (e: any) { onError(e); return; } + + // Step 1: midi2ly (sync, fast — typically <2s) + let lyFile: string; + try { + onProgress('MIDI 파싱 중...'); + lyFile = midiToLilypond(midiCopy, workDir); + } catch (e: any) { onError(e); return; } + + // Step 2: LilyPond SVG async — no timeout + onProgress('악보 렌더링 시작...'); + const proc = spawn(LILYPOND_BIN, ['--svg', '-o', 'output', lyFile], { cwd: workDir }); + + let totalPages = 0; + let stderrBuf = ''; + let lastSvgCount = 0; + + // Poll filesystem every 2s to report SVG page generation progress + const svgPoll = setInterval(() => { + const n = collectSvgPages(workDir).length; + if (n > 0 && n !== lastSvgCount) { + lastSvgCount = n; + const label = totalPages > 0 ? `${n} / ${totalPages}페이지` : `${n}페이지`; + onProgress(`악보 생성 중... ${label}`); + } + }, 2000); + + proc.stderr.on('data', (chunk: Buffer) => { + const text = chunk.toString('utf8'); + stderrBuf += text; + + const measures = text.match(/\[(\d+)\]/g); + if (measures) { + const last = measures[measures.length - 1].replace(/[\[\]]/g, ''); + onProgress(`${last}마디 처리 중...`); + } + const pageMatch = text.match(/Fitting music on (\d+) pages?/); + if (pageMatch) { + totalPages = parseInt(pageMatch[1], 10); + onProgress(`${totalPages}페이지 레이아웃 중...`); + } + }); + + proc.on('error', (err: Error) => { clearInterval(svgPoll); onError(err); }); + + proc.on('close', () => { + clearInterval(svgPoll); + const pages = collectSvgPages(workDir); + if (pages.length === 0) { + onError(new Error(`LilyPond SVG 렌더 실패:\n${stderrBuf.slice(0, 1000)}`)); + return; + } + + const pdfOut = path.join(workDir, 'output.pdf'); + onDone({ + id: 'midi-' + id, + svgPath: pages[0], + svgPages: pages, + pdfPath: fs.existsSync(pdfOut) ? pdfOut : null, + midiPath: originalMidi, + }); + + // PDF fire-and-forget in background + if (!fs.existsSync(pdfOut)) { + const pdfProc = spawn(LILYPOND_BIN, ['--pdf', '-o', 'output', lyFile], { cwd: workDir }); + pdfProc.on('error', () => { /* ignore */ }); + } + }); +} diff --git a/web-ui/index.html b/web-ui/index.html index 1669030..7b91b2c 100644 --- a/web-ui/index.html +++ b/web-ui/index.html @@ -2248,7 +2248,127 @@ .error-context-detail.visible { display: block; } + + /* Music widget */ + .music-widget { + border: 1px solid var(--line); + border-radius: 12px; + overflow: hidden; + margin: 10px 0; + background: var(--surface); + } + .music-widget-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 12px; + background: var(--hover); + border-bottom: 1px solid var(--line); + } + .music-widget-label { + font-size: 12px; + font-weight: 600; + color: var(--muted); + } + .music-render-btn { + font-size: 12px; + padding: 4px 12px; + border-radius: 8px; + border: 1px solid var(--brand); + background: var(--brand); + color: #fff; + cursor: pointer; + font-family: inherit; + } + .music-render-btn:disabled { + opacity: 0.5; + cursor: default; + } + .music-widget-body { + overflow: hidden; + transition: max-height 0.2s ease; + } + .music-widget-body.collapsed { + max-height: 0 !important; + } + .music-output { + padding: 12px; + } + .music-output img { + width: 100%; + height: auto; + display: block; + } + midi-player { + display: block; + width: 100%; + max-width: 420px; + margin-top: 6px; + --player-row-height: 16px; + --player-font-size: 11px; + } + .music-actions { + display: flex; + gap: 8px; + margin-top: 10px; + flex-wrap: wrap; + align-items: center; + } + .music-dl-btn { + font-size: 12px; + padding: 4px 12px; + border-radius: 8px; + border: 1px solid var(--line); + background: var(--surface); + color: var(--text); + cursor: pointer; + text-decoration: none; + font-family: inherit; + } + .music-dl-btn:hover { background: var(--hover); } + .music-page-nav { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 8px; + font-size: 12px; + color: var(--muted); + } + .music-page-btn { + padding: 2px 10px; + border-radius: 6px; + border: 1px solid var(--line); + background: var(--surface); + color: var(--text); + cursor: pointer; + font-size: 13px; + font-family: inherit; + line-height: 1.4; + } + .music-page-btn:disabled { opacity: 0.35; cursor: default; } + .music-page-btn:not(:disabled):hover { background: var(--hover); } + .midi-router-panel { + display: flex; + align-items: center; + gap: 6px; + flex-wrap: wrap; + margin-top: 4px; + width: 100%; + } + .midi-router-sel { + font-size: 11px; + padding: 2px 6px; + border: 1px solid var(--line); + border-radius: 6px; + background: var(--surface); + color: var(--text); + max-width: 200px; + flex: 1; + } + .midi-router-status { font-size: 11px; color: var(--muted); } + + @@ -2335,6 +2455,12 @@ Skills +
스킬 로딩 중...
@@ -2381,6 +2507,12 @@ + | + 바로보내기 +