/** * Builds content/kanji/ids.json: the top-level IDS decomposition of the 2,136 jōyō kanji. * * npx tsx scripts/build-kanji-data.ts * * Sources (see content/kanji/README.md for licences): * - IDS: cjkvi-ids ids.txt (CHISE-derived), pinned to a commit. * - Jōyō list: KANJIDIC2 1–6 and 8 (= jōyō). * * Only the top level is kept. A nested operand (e.g. the ⿰亻木 inside ⿱⿰亻木…) is replaced by the * character whose own IDS is exactly that sequence; if none exists that decomposition is dropped. * Only IDS alternatives tagged for Japan (J) or untagged are kept. */ import { gunzipSync } from "node:zlib"; import { writeFileSync } from "node:fs"; import { join } from "node:path"; const IDS_COMMIT = "86b4d16159f0079437870408f0ca186e529015db"; const IDS_URL = `https://raw.githubusercontent.com/cjkvi/cjkvi-ids/${IDS_COMMIT}/ids.txt`; const KANJIDIC_URL = "http://www.edrdg.org/kanjidic/kanjidic2.xml.gz"; const IDC_ARITY: Record = { "⿰": 2, "⿱": 2, "⿲": 3, "⿳": 3, "⿴": 2, "⿵": 2, "⿶": 2, "⿷": 2, "⿸": 2, "⿹": 2, "⿺": 2, "⿻": 2, }; interface Entry { layout: string; parts: string[] } async function get(url: string): Promise { const res = await fetch(url); if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); return Buffer.from(await res.arrayBuffer()); } /** Splits an IDS into its IDC and top-level operand sequences. */ function topLevel(ids: string): { layout: string; operands: string[] } | null { const chars = Array.from(ids); const layout = chars[0]; if (!(layout in IDC_ARITY)) return null; let pos = 1; // Count-based reader: an IDC opens `arity` slots, any other character fills one. const readOperand = (): string | null => { if (pos >= chars.length) return null; const start = pos; let open = 1; while (open > 0) { if (pos >= chars.length) return null; const c = chars[pos++]; open += c in IDC_ARITY ? IDC_ARITY[c] - 1 : -1; } return chars.slice(start, pos).join(""); }; const operands: string[] = []; for (let i = 0; i < IDC_ARITY[layout]; i++) { const op = readOperand(); if (op === null) return null; operands.push(op); } return pos === chars.length ? { layout, operands } : null; } async function main() { const idsText = (await get(IDS_URL)).toString("utf8"); const kanjidic = gunzipSync(await get(KANJIDIC_URL)).toString("utf8"); const dbVersion = /([^<]+)<\/database_version>/.exec(kanjidic)?.[1] ?? null; const created = /([^<]+)<\/date_of_creation>/.exec(kanjidic)?.[1] ?? null; const joyo: string[] = []; for (const m of kanjidic.matchAll(/\s*([^<]+)<\/literal>[\s\S]*?<\/character>/g)) { const grade = /(\d+)<\/grade>/.exec(m[0])?.[1]; if (grade && ["1", "2", "3", "4", "5", "6", "8"].includes(grade)) joyo.push(m[1]); } if (joyo.length !== 2136) throw new Error(`expected 2136 jōyō kanji, got ${joyo.length}`); // char → IDS alternatives (J or untagged) const idsOf = new Map(); for (const line of idsText.split("\n")) { if (!line || line.startsWith("#")) continue; const [, ch, ...alts] = line.split("\t"); if (!ch) continue; const keep: string[] = []; for (const alt of alts) { const m = /^(.*?)(?:\[([A-Z]+)\])?$/.exec(alt.trim()); if (!m) continue; if (m[2] && !m[2].includes("J")) continue; keep.push(m[1]); } idsOf.set(ch, keep); } // nested sequence → character const charOfIds = new Map(); for (const [ch, alts] of idsOf) for (const a of alts) if (a !== ch && !charOfIds.has(a)) charOfIds.set(a, ch); const kanji: Record = {}; let dropped = 0; let atomic = 0; for (const ch of joyo) { const entries: Entry[] = []; for (const ids of idsOf.get(ch) ?? []) { const top = topLevel(ids); if (!top) continue; const parts = top.operands.map((op) => (Array.from(op).length === 1 ? op : charOfIds.get(op) ?? null)); if (parts.some((p) => p === null)) { dropped++; continue; } const e = { layout: top.layout, parts: parts as string[] }; if (!entries.some((x) => x.layout === e.layout && x.parts.join() === e.parts.join())) entries.push(e); } if (entries.length) kanji[ch] = entries; else atomic++; } const out = { source: { ids: { url: IDS_URL, commit: IDS_COMMIT, licence: "GPL-2.0 (CHISE IDS terms)" }, joyo: { url: KANJIDIC_URL, databaseVersion: dbVersion, dateOfCreation: created, licence: "CC BY-SA 4.0 (EDRDG)" }, built: new Date().toISOString().slice(0, 10), counts: { joyo: joyo.length, decomposed: Object.keys(kanji).length, atomicOrUnresolved: atomic, droppedAlternatives: dropped }, }, kanji, }; const path = join(__dirname, "..", "content", "kanji", "ids.json"); writeFileSync(path, JSON.stringify(out) + "\n"); console.log(`wrote ${path}`, out.source.counts, `kanjidic ${dbVersion} (${created})`); } main().catch((e) => { console.error(e); process.exit(1); });