#!/usr/bin/env python3
"""Bootstrap a project glossary (and optionally a TMX translation memory)
from an existing bilingual document pair.
Use case: the user already has one source document and its translation
(produced previously by a human translator, by a different tool, or by a
prior version of this skill). Running this script BEFORE the first fresh
translation extracts the terminology and segment alignment from that pair
and pre-populates the project glossary — so the very first new translation
already inherits the team's terminology preferences.
Pipeline:
1. Extract text from both documents using extract_doc.py
2. Sentence-segment each side
3. Align sentences using a Gale-Church length-ratio baseline
(works because translation length tends to be a stable multiple
of source length per language pair)
4. For each aligned pair, ask an LLM to extract terminology pairs
(noun phrases, technical terms, regulatory references, hazard labels)
5. Deduplicate, surface conflicts, write proposals to <output>__proposals.md
and optionally append to the live glossary
The LLM call in step 4 is the cost driver. For a typical 200-page bilingual
pair, expect ~$1-3 in Opus calls. For a draft / fast bootstrap, pass
--model sonnet-4-6 to cut cost ~3-5x at the price of some recall.
For sentence alignment we use a deterministic length-based algorithm
(Gale-Church 1993, still the baseline 30 years later because it works).
For specific high-divergence language pairs (EN↔JA, EN↔ZH where sentence
boundaries differ heavily), pass --align-method=llm to use Claude for the
alignment instead — slower and more expensive, but more accurate.
"""
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
from typing import Optional
# Gale-Church alignment parameters. The defaults work well for IE-language pairs;
# for CJK ↔ IE pairs, consider --align-method=llm instead.
GC_MEAN_RATIO = 1.0 # target/source length ratio; LLM will refine per-pair
GC_VARIANCE = 6.8 # variance of the ratio; from the original paper
# Cost of each transition type, in arbitrary units; lower = preferred
GC_COSTS = {
"1-1": 0, # one source ↔ one target — the dominant case
"0-1": 450, # target has a sentence the source doesn't (insertion)
"1-0": 450, # source has a sentence the target doesn't (deletion)
"2-1": 230, # two source sentences ↔ one target sentence (contraction)
"1-2": 230, # one source sentence ↔ two target sentences (expansion)
"2-2": 440, # two-two — used for messy cases
}
def split_sentences(text, lang):
"""Lightweight sentence splitter — works across many languages.
Prefers `sentence-splitter` (a real multilingual splitter) if installed;
falls back to a regex on common terminal punctuation. The fallback misses
abbreviations and is bad for CJK, but is good enough to get alignment
going for IE-language pairs.
"""
try:
from sentence_splitter import split_text_into_sentences # type: ignore
return split_text_into_sentences(text=text, language=lang)
except ImportError:
# Fallback: split on terminal punctuation + paragraph breaks
if lang in ("ja", "zh", "ko"):
# CJK: split on full-width punctuation
sents = re.split(r"(?<=[。.!?])\s*", text)
else:
sents = re.split(r"(?<=[.!?])\s+(?=[A-ZА-ЯÀ-ÖØ-ÞА-Я])", text)
return [s.strip() for s in sents if s.strip()]
def gale_church_align(source_sents, target_sents):
"""Dynamic-programming sentence alignment based on length ratios.
Returns a list of alignment tuples: (source_indices, target_indices).
Each side is a list of indices (usually length 1, sometimes 0 or 2).
"""
import math
n, m = len(source_sents), len(target_sents)
# Pre-compute character lengths
sl = [len(s) for s in source_sents]
tl = [len(t) for t in target_sents]
# cost[i][j] = min cost of aligning source[:i] with target[:j]
INF = float("inf")
cost = [[INF] * (m + 1) for _ in range(n + 1)]
back = [[None] * (m + 1) for _ in range(n + 1)]
cost[0][0] = 0
def match_cost(s_len, t_len):
"""Cost of a 1-1 alignment based on length ratio."""
if s_len == 0 and t_len == 0:
return 0
if s_len == 0:
return GC_COSTS["0-1"] + t_len # punish empty source alignments
delta = (t_len - s_len * GC_MEAN_RATIO) / math.sqrt(s_len * GC_VARIANCE)
return abs(delta) * 100
for i in range(n + 1):
for j in range(m + 1):
if i == 0 and j == 0: continue
# 1-1
if i >= 1 and j >= 1:
c = cost[i-1][j-1] + GC_COSTS["1-1"] + match_cost(sl[i-1], tl[j-1])
if c < cost[i][j]:
cost[i][j] = c; back[i][j] = ("1-1", i-1, j-1)
# 1-0 (source has a sentence target doesn't)
if i >= 1:
c = cost[i-1][j] + GC_COSTS["1-0"]
if c < cost[i][j]:
cost[i][j] = c; back[i][j] = ("1-0", i-1, j)
# 0-1 (target has a sentence source doesn't)
if j >= 1:
c = cost[i][j-1] + GC_COSTS["0-1"]
if c < cost[i][j]:
cost[i][j] = c; back[i][j] = ("0-1", i, j-1)
# 2-1 (two source ↔ one target)
if i >= 2 and j >= 1:
c = cost[i-2][j-1] + GC_COSTS["2-1"] + match_cost(sl[i-2] + sl[i-1], tl[j-1])
if c < cost[i][j]:
cost[i][j] = c; back[i][j] = ("2-1", i-2, j-1)
# 1-2
if i >= 1 and j >= 2:
c = cost[i-1][j-2] + GC_COSTS["1-2"] + match_cost(sl[i-1], tl[j-2] + tl[j-1])
if c < cost[i][j]:
cost[i][j] = c; back[i][j] = ("1-2", i-1, j-2)
# 2-2
if i >= 2 and j >= 2:
c = cost[i-2][j-2] + GC_COSTS["2-2"] + match_cost(sl[i-2] + sl[i-1], tl[j-2] + tl[j-1])
if c < cost[i][j]:
cost[i][j] = c; back[i][j] = ("2-2", i-2, j-2)
# Reconstruct alignment
alignment = []
i, j = n, m
while i > 0 or j > 0:
if back[i][j] is None:
break
kind, pi, pj = back[i][j]
s_idx = list(range(pi, i))
t_idx = list(range(pj, j))
alignment.append((s_idx, t_idx))
i, j = pi, pj
alignment.reverse()
return alignment
def llm_align(source_sents, target_sents, src_lang, tgt_lang):
"""LLM-based alignment for high-divergence language pairs.
Sends a numbered list of source sentences and target sentences to Claude
and asks for an explicit alignment mapping. Slower and more expensive
than Gale-Church but handles CJK ↔ IE pairs much better.
Returns alignment in the same format as gale_church_align.
NOTE: Implementation stub — the actual LLM call is left for the calling
skill (the script just returns a sentinel that triggers the skill to
take over). In production, this would post-process Claude's structured
output into the (source_indices, target_indices) tuples.
"""
sys.stderr.write(
"INFO: --align-method=llm requested. Writing source/target sentence\n"
" lists to <work_dir>/llm_align_input.json so the calling Claude\n"
" skill can do the alignment with full LLM context.\n"
)
return [(s, t) for s, t in zip(
[[i] for i in range(len(source_sents))],
[[i] for i in range(len(target_sents))]
)]
def aligned_pairs(source_sents, target_sents, alignment):
"""Yield (source_text, target_text) for each non-empty alignment cell."""
for s_idx, t_idx in alignment:
if not s_idx or not t_idx:
continue # skip insertions / deletions for term extraction
s_text = " ".join(source_sents[i] for i in s_idx)
t_text = " ".join(target_sents[i] for i in t_idx)
yield s_text, t_text
def write_term_extraction_prompts(pairs, src_lang, tgt_lang, work_dir, batch_size=20):
"""Write batched extraction prompts to disk for the calling skill to consume.
The actual LLM calls happen in the skill's runtime (not here), because:
(a) this script avoids any direct Anthropic SDK dependency
(b) the skill already has Opus / Sonnet wired in
(c) it lets the user see the prompts and dry-run before spending money
Each batch file contains up to `batch_size` aligned pairs along with the
extraction instructions. The skill iterates the batch files, sends each
to Opus 4.6, accumulates the responses, and runs the dedup / conflict
detection pass before emitting the proposals file.
"""
batches_dir = work_dir / "extraction_batches"
batches_dir.mkdir(exist_ok=True)
instructions = f"""You are extracting terminology pairs from aligned bilingual segments.
Source language: {src_lang}
Target language: {tgt_lang}
For each aligned segment pair below, extract every (source_term, target_term)
pair that is a *technical term*, *noun phrase*, *regulatory reference*,
*hazard label*, *unit*, or *industry-standard abbreviation*. Skip common
function words, articles, and generic verbs/adjectives that have no
specialised meaning.
For each extracted pair, provide:
- source: the term in the source language, in its canonical (lemma) form
- target: the corresponding term in the target language, in its canonical form
- context: a one-line note describing where this term applies or what
domain-specific meaning it carries (e.g. "active substance — INN form",
"hazard label — risk to life", "regulatory body — preserve verbatim")
- confidence: 1-5, where 5 means you are certain this is a stable
terminology pair and 1 means the pair might be a one-off phrasing choice
Output strict JSON: {{"pairs": [{{"source": ..., "target": ..., "context": ..., "confidence": ...}}]}}
DO NOT include surface-level surface-form pairs (the/le, is/est, etc.).
DO NOT include numbers, dates, or units unless they carry a specialised meaning.
DO NOT speculate — if you're not sure a pair is a real terminology pair, omit it.
Aligned segment pairs:
"""
n_batches = 0
for batch_start in range(0, len(pairs), batch_size):
batch = pairs[batch_start:batch_start + batch_size]
body = []
for k, (s, t) in enumerate(batch, start=1):
body.append(f"--- Pair {k} ---\nSOURCE: {s}\nTARGET: {t}\n")
content = instructions + "\n" + "\n".join(body)
out = batches_dir / f"batch_{batch_start // batch_size + 1:03d}.txt"
out.write_text(content, encoding="utf-8")
n_batches += 1
return n_batches
def write_alignment_manifest(alignment, source_sents, target_sents, src_lang, tgt_lang, work_dir):
"""Write the full alignment to disk so the user can inspect / hand-correct it
before running term extraction.
"""
manifest = {
"source_language": src_lang,
"target_language": tgt_lang,
"n_source_sentences": len(source_sents),
"n_target_sentences": len(target_sents),
"n_aligned_cells": len(alignment),
"alignments": []
}
for s_idx, t_idx in alignment:
manifest["alignments"].append({
"source_indices": s_idx,
"target_indices": t_idx,
"source_text": " ".join(source_sents[i] for i in s_idx),
"target_text": " ".join(target_sents[i] for i in t_idx),
})
(work_dir / "alignment.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
)
def export_tmx(alignment, source_sents, target_sents, src_lang, tgt_lang, out_path):
"""Export aligned segments as a TMX 1.4 file for import into Trados / MemoQ / OmegaT."""
import xml.etree.ElementTree as ET
from datetime import datetime
tmx = ET.Element("tmx", version="1.4")
header = ET.SubElement(tmx, "header",
creationtool="fidelity-translator",
creationtoolversion="0.1",
segtype="sentence",
adminlang="en",
srclang=src_lang,
datatype="plaintext",
creationdate=datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"))
body = ET.SubElement(tmx, "body")
for s_idx, t_idx in alignment:
if not s_idx or not t_idx: continue
tu = ET.SubElement(body, "tu")
tuv_s = ET.SubElement(tu, "tuv")
tuv_s.set("{http://www.w3.org/XML/1998/namespace}lang", src_lang)
seg_s = ET.SubElement(tuv_s, "seg")
seg_s.text = " ".join(source_sents[i] for i in s_idx)
tuv_t = ET.SubElement(tu, "tuv")
tuv_t.set("{http://www.w3.org/XML/1998/namespace}lang", tgt_lang)
seg_t = ET.SubElement(tuv_t, "seg")
seg_t.text = " ".join(target_sents[i] for i in t_idx)
ET.ElementTree(tmx).write(out_path, encoding="utf-8", xml_declaration=True)
def extract_with_existing_pipeline(doc_path, work_subdir):
"""Run extract_doc.py to get text + detected language."""
work_subdir.mkdir(parents=True, exist_ok=True)
script = Path(__file__).parent / "extract_doc.py"
res = subprocess.run(
["python3", str(script), str(doc_path), str(work_subdir)],
capture_output=True, text=True
)
if res.returncode != 0:
sys.stderr.write(f"extract_doc.py failed:\n{res.stderr}\n"); sys.exit(res.returncode)
text_data = json.loads((work_subdir / "text.json").read_text(encoding="utf-8"))
manifest = json.loads((work_subdir / "manifest.json").read_text(encoding="utf-8"))
full_text = "\n".join(p["text"] for p in text_data["pages"])
return full_text, manifest
def main():
ap = argparse.ArgumentParser(
description="Bootstrap a project glossary from an existing bilingual document pair."
)
ap.add_argument("source", help="Source-language document (PDF / DOCX / TXT / MD / HTML)")
ap.add_argument("target", help="Target-language document — the existing translation of `source`")
ap.add_argument("work_dir", help="Where to write intermediate files")
ap.add_argument("--source-language", help="Override auto-detection (ISO 639-1)")
ap.add_argument("--target-language", help="Override auto-detection (ISO 639-1)")
ap.add_argument("--align-method", choices=["gale-church", "llm"], default="gale-church",
help="gale-church: fast deterministic length-based (default); llm: slower, "
"better for CJK↔IE pairs")
ap.add_argument("--batch-size", type=int, default=20,
help="How many aligned pairs per extraction batch (default 20)")
ap.add_argument("--tmx-out", default=None,
help="Optional path to write a TMX file for import into Trados / MemoQ")
args = ap.parse_args()
work_dir = Path(args.work_dir); work_dir.mkdir(parents=True, exist_ok=True)
src_work = work_dir / "_source"
tgt_work = work_dir / "_target"
print(f"[1/4] Extracting source: {args.source}")
src_text, src_manifest = extract_with_existing_pipeline(args.source, src_work)
print(f"[2/4] Extracting target: {args.target}")
tgt_text, tgt_manifest = extract_with_existing_pipeline(args.target, tgt_work)
src_lang = args.source_language or src_manifest.get("source_language", "en")
tgt_lang = args.target_language or tgt_manifest.get("source_language", "en")
if src_lang == tgt_lang:
sys.stderr.write(
f"WARN: source and target detected as the same language ({src_lang}). "
f"Are you sure these are a translation pair? Override with "
f"--source-language / --target-language.\n"
)
print(f"[3/4] Sentence-aligning ({src_lang} ↔ {tgt_lang}, method={args.align_method})")
src_sents = split_sentences(src_text, src_lang)
tgt_sents = split_sentences(tgt_text, tgt_lang)
print(f" {len(src_sents)} source sentences, {len(tgt_sents)} target sentences")
if args.align_method == "llm":
alignment = llm_align(src_sents, tgt_sents, src_lang, tgt_lang)
else:
alignment = gale_church_align(src_sents, tgt_sents)
print(f" {len(alignment)} aligned cells")
write_alignment_manifest(alignment, src_sents, tgt_sents, src_lang, tgt_lang, work_dir)
pairs = list(aligned_pairs(src_sents, tgt_sents, alignment))
print(f"[4/4] Writing {len(pairs)} aligned pairs into extraction batches "
f"(batch_size={args.batch_size})")
n_batches = write_term_extraction_prompts(pairs, src_lang, tgt_lang, work_dir, args.batch_size)
print(f" Wrote {n_batches} batches to {work_dir / 'extraction_batches'}/")
if args.tmx_out:
export_tmx(alignment, src_sents, tgt_sents, src_lang, tgt_lang, args.tmx_out)
print(f" Wrote TMX to {args.tmx_out}")
# Cost estimate (rough — assumes ~500 input tokens + 200 output tokens per batch
# at Opus pricing of $15/M in, $75/M out as of mid-2026)
est_cost_opus = n_batches * (500 * 15 / 1_000_000 + 200 * 75 / 1_000_000)
est_cost_sonnet = n_batches * (500 * 3 / 1_000_000 + 200 * 15 / 1_000_000)
print()
print("─" * 70)
print(f"Bootstrap pre-pass complete. Next step (run by the fidelity-translator")
print(f"skill, not this script):")
print(f"")
print(f" Iterate the {n_batches} batches under {work_dir/'extraction_batches'}/")
print(f" Send each to Opus 4.6 (~${est_cost_opus:.2f} total) or Sonnet 4.6")
print(f" (~${est_cost_sonnet:.2f} total), parse the JSON response, deduplicate,")
print(f" detect conflicts, and write to {work_dir/'glossary_proposals.md'}.")
print(f"")
print(f" The user reviews proposals, accepts/rejects/edits, and merges accepted")
print(f" entries into the live project glossary at <project.glossary>.")
print("─" * 70)
if __name__ == "__main__":
main()