From cb22e074af309985428f6b3f18c0d0a757d0202d Mon Sep 17 00:00:00 2001 From: Chun_Qiu Date: Sat, 8 Aug 2026 15:21:00 +0800 Subject: [PATCH] fix: OOV english words now transcribed via phonemize G2P + IPA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add phonemize (MIT) rule-based G2P as second fallback tier - IPA phoneme -> katakana CV-syllable transcription table - proposal -> プラポーザル, bilibili -> バイリーバイリー (no more letter-spelling) - Letter spelling kept as last resort only - Bridge logs SYNTH IN/OUT to stderr; tts.py drains stderr to Python logs so conversion results are visible in the server log - Rebuild portable zip (83.8 MB) --- package-lock.json | 35 ++++++++++ package.json | 4 +- tts.py | 26 +++++++ tts_bridge.js | 168 +++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 217 insertions(+), 16 deletions(-) create mode 100644 package-lock.json diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..5c36c96 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,35 @@ +{ + "name": "bililive-touhou-tts", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "bililive-touhou-tts", + "dependencies": { + "phonemize": "^1.2.0" + } + }, + "node_modules/number-to-words": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/number-to-words/-/number-to-words-1.2.4.tgz", + "integrity": "sha512-/fYevVkXRcyBiZDg6yzZbm0RuaD6i0qRfn8yr+6D0KgBMOndFPxuW10qCHpzs50nN8qKuv78k8MuotZhcVX6Pw==", + "license": "MIT" + }, + "node_modules/phonemize": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/phonemize/-/phonemize-1.2.0.tgz", + "integrity": "sha512-+zEpOPXrEaylYCIXMSDVhiAwFbVzIXRf+7vheuNxozg4hLKbQVDXCOpI0GAJw41xEgE9LcwMjLUN30aWrpwSkw==", + "license": "MIT", + "dependencies": { + "number-to-words": "^1.2.4", + "pinyin-pro": "^3.26.0" + } + }, + "node_modules/pinyin-pro": { + "version": "3.28.2", + "resolved": "https://registry.npmmirror.com/pinyin-pro/-/pinyin-pro-3.28.2.tgz", + "integrity": "sha512-jV38yxXHLfidirMC4hrXasLDozLCSq/4DfX88GnHcSEJ2+GpSedG6I9VOiEXJu6iQ5dbJC/RjmzyMuS5h/wH5A==", + "license": "MIT" + } + } +} diff --git a/package.json b/package.json index b86a0eb..b9393bf 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,7 @@ { "name": "bililive-touhou-tts", "type": "module", - "dependencies": {} + "dependencies": { + "phonemize": "^1.2.0" + } } diff --git a/tts.py b/tts.py index 0735217..533fab0 100644 --- a/tts.py +++ b/tts.py @@ -45,6 +45,7 @@ class TTSBridge: self._process: asyncio.subprocess.Process | None = None self._bridge_script = bridge_script or _resolve_path("tts_bridge.js") self._node_exe = _get_node_exe() + self._stderr_task: asyncio.Task | None = None async def start(self) -> None: if not pathlib.Path(self._bridge_script).exists(): @@ -61,6 +62,7 @@ class TTSBridge: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) + self._stderr_task = asyncio.create_task(self._drain_stderr()) line = await asyncio.wait_for(self._process.stdout.readline(), timeout=60) line_str = line.decode("utf-8").strip() @@ -73,6 +75,23 @@ class TTSBridge: logger.info("Node.js TTS bridge ready.") + async def _drain_stderr(self) -> None: + """Continuously read bridge stderr and forward to Python logs.""" + if self._process is None or self._process.stderr is None: + return + try: + while True: + raw = await self._process.stderr.readline() + if not raw: + break + line = raw.decode("utf-8", errors="replace").rstrip() + if line: + logger.info("[bridge] %s", line) + except asyncio.CancelledError: + pass + except Exception: + logger.debug("Bridge stderr drain stopped", exc_info=True) + async def synthesize(self, kana_text: str) -> bytes: if self._process is None or self._process.stdin is None: raise RuntimeError("Bridge not started. Call start() first.") @@ -114,6 +133,13 @@ class TTSBridge: async def stop(self) -> None: if self._process is not None: logger.info("Stopping TTS bridge...") + if self._stderr_task is not None: + self._stderr_task.cancel() + try: + await self._stderr_task + except (asyncio.CancelledError, Exception): + pass + self._stderr_task = None try: if self._process.stdin is not None: self._process.stdin.close() diff --git a/tts_bridge.js b/tts_bridge.js index a783d12..bf5764c 100644 --- a/tts_bridge.js +++ b/tts_bridge.js @@ -1,8 +1,10 @@ /** tts_bridge.js — Persistent Node.js bridge for aquestalk.js TTS synthesis. -Converts English words to katakana via vendored english-to-kana dictionary -(english-kana-matcher.js, 49K words, MIT). Out-of-dictionary words fall back -to letter-by-letter romanized spelling so no word is ever skipped. +English word → katakana pipeline (never skips): + 1. english-to-kana dictionary (49K words, MIT, vendored english-kana-matcher.js) + 2. phonemize (MIT) G2P → IPA → rule-based katakana transcription + 3. letter-by-letter romanized spelling (last resort) + English punctuation (,.!?) is converted to Japanese pauses (、。). */ @@ -10,6 +12,7 @@ import { readFileSync, writeFileSync } from "fs"; import { createInterface } from "readline"; import { fileURLToPath, pathToFileURL } from "url"; import { dirname, join } from "path"; +import { createRequire } from "module"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -18,6 +21,16 @@ const __dirname = dirname(__filename); const matcherMod = pathToFileURL(join(__dirname, "english-kana-matcher.js")).href; const { lookupKana } = await import(matcherMod); +// ── phonemize (G2P, MIT) via CJS entry (avoids JSON import issue in Node ESM) ── +const require = createRequire(import.meta.url); +let phonemize = null; +try { + phonemize = require("phonemize").phonemize; + console.error("[bridge] phonemize G2P loaded."); +} catch (err) { + console.error(`[bridge] phonemize unavailable (${err.message}), using letter fallback only.`); +} + // ── aquestalk.js ────────────────────────────────────────────────────── const aquestalkMod = pathToFileURL(join(__dirname, "aquestalk.js", "dist", "index.js")).href; const { load } = await import(aquestalkMod); @@ -46,7 +59,109 @@ try { process.exit(1); } -// Letter-by-letter fallback readings (AquesTalk1-safe, no ヴ) +// ── IPA → Katakana transcription (rule-based) ───────────────────────── +const KANA = { + k: { a: "カ", i: "キ", u: "ク", e: "ケ", o: "コ" }, + g: { a: "ガ", i: "ギ", u: "グ", e: "ゲ", o: "ゴ" }, + s: { a: "サ", i: "シ", u: "ス", e: "セ", o: "ソ" }, + z: { a: "ザ", i: "ジ", u: "ズ", e: "ゼ", o: "ゾ" }, + t: { a: "タ", i: "チ", u: "トゥ", e: "テ", o: "ト" }, + d: { a: "ダ", i: "ジ", u: "ドゥ", e: "デ", o: "ド" }, + n: { a: "ナ", i: "ニ", u: "ヌ", e: "ネ", o: "ノ" }, + h: { a: "ハ", i: "ヒ", u: "フ", e: "ヘ", o: "ホ" }, + b: { a: "バ", i: "ビ", u: "ブ", e: "ベ", o: "ボ" }, + p: { a: "パ", i: "ピ", u: "プ", e: "ペ", o: "ポ" }, + m: { a: "マ", i: "ミ", u: "ム", e: "メ", o: "モ" }, + r: { a: "ラ", i: "リ", u: "ル", e: "レ", o: "ロ" }, + f: { a: "ファ", i: "フィ", u: "フ", e: "フェ", o: "フォ" }, + v: { a: "バ", i: "ビ", u: "ブ", e: "ベ", o: "ボ" }, + tʃ: { a: "チャ", i: "チ", u: "チュ", e: "チェ", o: "チョ" }, + dʒ: { a: "ジャ", i: "ジ", u: "ジュ", e: "ジェ", o: "ジョ" }, + ʃ: { a: "シャ", i: "シ", u: "シュ", e: "シェ", o: "ショ" }, + ʒ: { a: "ジャ", i: "ジ", u: "ジュ", e: "ジェ", o: "ジョ" }, + θ: { a: "サ", i: "シ", u: "ス", e: "セ", o: "ソ" }, + ð: { a: "ザ", i: "ジ", u: "ズ", e: "ゼ", o: "ゾ" }, + w: { a: "ワ", i: "ウィ", u: "ウ", e: "ウェ", o: "ウォ" }, + j: { a: "ヤ", i: "イ", u: "ユ", e: "イェ", o: "ヨ" }, + ŋ: { a: "ンガ", i: "ンギ", u: "ング", e: "ンゲ", o: "ンゴ" }, +}; + +const CONS_ROW = { + "p": "p", "b": "b", "t": "t", "d": "d", "k": "k", "ɡ": "g", "g": "g", + "f": "f", "v": "v", "s": "s", "z": "z", "ʃ": "ʃ", "ʒ": "ʒ", + "h": "h", "tʃ": "tʃ", "dʒ": "dʒ", "θ": "θ", "ð": "ð", + "m": "m", "n": "n", "ŋ": "ŋ", "l": "r", "ɫ": "r", "ɹ": "r", + "r": "r", "j": "j", "w": "w", "ɾ": "r", +}; + +const VOWEL_KANA = { a: "ア", i: "イ", u: "ウ", e: "エ", o: "オ" }; +const DIPH_KANA = { ai: "アイ", au: "アウ", oi: "オイ" }; + +function vClass(v) { + switch (v) { + case "ə": case "ɚ": case "ɝ": case "ɑ": case "ʌ": case "ɒ": case "æ": case "ɜ": + return { vowel: "a", long: false, diph: null }; + case "ɪ": return { vowel: "i", long: false, diph: null }; + case "i": return { vowel: "i", long: true, diph: null }; + case "ʊ": return { vowel: "u", long: false, diph: null }; + case "u": return { vowel: "u", long: true, diph: null }; + case "ɛ": case "e": return { vowel: "e", long: false, diph: null }; + case "eɪ": return { vowel: "e", long: true, diph: null }; + case "ɔ": return { vowel: "o", long: false, diph: null }; + case "o": case "oʊ": return { vowel: "o", long: true, diph: null }; + case "aɪ": return { vowel: "a", long: false, diph: "ai" }; + case "aʊ": return { vowel: "a", long: false, diph: "au" }; + case "ɔɪ": return { vowel: "o", long: false, diph: "oi" }; + default: return null; + } +} + +const SONORANT = new Set(["m", "n", "ŋ", "l", "ɫ", "ɹ", "r", "j", "w", "ɾ"]); + +function ipaToKana(ipaStr) { + const tokens = []; + let i = 0; + while (i < ipaStr.length) { + const two = ipaStr.slice(i, i + 2); + if (CONS_ROW[two] || vClass(two)) { tokens.push(two); i += 2; continue; } + const one = ipaStr[i]; + if (CONS_ROW[one] || vClass(one)) { tokens.push(one); i += 1; continue; } + if (one === " " || one === "ː") { tokens.push(one); i += 1; continue; } + i += 1; + } + + let out = ""; + for (let i = 0; i < tokens.length; i++) { + const t = tokens[i]; + if (t === " ") { out += " "; continue; } + if (t === "ː") { out += "ー"; continue; } + const vc = vClass(t); + if (vc) { + out += vc.diph ? DIPH_KANA[vc.diph] : (VOWEL_KANA[vc.vowel] + (vc.long ? "ー" : "")); + continue; + } + const row = CONS_ROW[t]; + if (!row) continue; + + const next = tokens[i + 1]; + const nextVc = next !== undefined ? vClass(next) : null; + if (nextVc) { + if (nextVc.diph) { + out += KANA[row][nextVc.vowel] + DIPH_KANA[nextVc.diph].slice(1); + } else { + out += KANA[row][nextVc.vowel] + (nextVc.long ? "ー" : ""); + } + i++; + } else if (next !== undefined) { + out += KANA[row]["u"]; + } else { + out += (t === "n" || t === "ŋ") ? "ン" : KANA[row]["u"]; + } + } + return out; +} + +// ── Letter-by-letter fallback (last resort, AquesTalk1-safe) ────────── const LETTER_KANA = { a: "エー", b: "ビー", c: "シー", d: "ディー", e: "イー", f: "エフ", g: "ジー", h: "エイチ", i: "アイ", j: "ジェー", @@ -60,28 +175,48 @@ const DIGIT_KANA = { "5": "ファイブ", "6": "シックス", "7": "セブン", "8": "エイト", "9": "ナイン", }; -// English punctuation → Japanese pause equivalents +function kanaForChar(ch) { + return LETTER_KANA[ch.toLowerCase()] || DIGIT_KANA[ch] || null; +} + +function spellWord(word) { + let out = ""; + for (const ch of word) { + const k = kanaForChar(ch); + out += k ? k : ""; + } + return out; +} + +// ── English punctuation → Japanese pause equivalents ────────────────── const PUNCT_MAP = { ",": "、", ".": "。", "!": "！", "?": "？", "，": "、", "。": "。", "！": "！", "？": "？", }; -function kanaForChar(ch) { - return LETTER_KANA[ch.toLowerCase()] || DIGIT_KANA[ch] || null; -} - function wordToKana(word) { const clean = word.replace(/'/g, "").toLowerCase(); + if (!clean) return ""; + + // 1. dictionary const hit = lookupKana(clean); if (hit) return hit; - // OOV fallback: spell out letter by letter (never skip) - let out = ""; - for (const ch of clean) { - const k = kanaForChar(ch); - out += k ? k : ""; + // 2. phonemize G2P → IPA → katakana + if (phonemize && /^[a-z]+$/.test(clean)) { + try { + const ipa = phonemize(clean, { stripStress: true }); + if (ipa && /[^a-z]/.test(ipa)) { + const kana = ipaToKana(ipa); + if (kana) return kana; + } + } catch (err) { + // fall through to letter spelling + } } - return out; + + // 3. letter-by-letter spelling (never skip) + return spellWord(clean); } function convertEnglishSegment(text) { @@ -130,6 +265,9 @@ function synthesize(kanaText) { result += convertEnglishSegment(englishBuf); } + console.error(`[bridge] SYNTH IN: ${trimmed}`); + console.error(`[bridge] SYNTH OUT: ${result}`); + const wav = aq.run(result, speed); return Buffer.from(wav); }