Skip to content

Tasdax123/claude-fidelity-translator / plugins/fidelity-translator/skills/fidelity-translator

fidelity-translator

Verified

scripts/bootstrap_glossary.py

View on GitHub
#!/usr/bin/env python3
"""Bootstrap a project glossary (and optionally a TMX translation memory)
from an existing bilingual document pair.

Use case: the user already has one source document and its translation
(produced previously by a human translator, by a different tool, or by a
prior version of this skill). Running this script BEFORE the first fresh
translation extracts the terminology and segment alignment from that pair
and pre-populates the project glossary — so the very first new translation
already inherits the team's terminology preferences.

Pipeline:
  1. Extract text from both documents using extract_doc.py
  2. Sentence-segment each side
  3. Align sentences using a Gale-Church length-ratio baseline
     (works because translation length tends to be a stable multiple
     of source length per language pair)
  4. For each aligned pair, ask an LLM to extract terminology pairs
     (noun phrases, technical terms, regulatory references, hazard labels)
  5. Deduplicate, surface conflicts, write proposals to <output>__proposals.md
     and optionally append to the live glossary

The LLM call in step 4 is the cost driver. For a typical 200-page bilingual
pair, expect ~$1-3 in Opus calls. For a draft / fast bootstrap, pass
--model sonnet-4-6 to cut cost ~3-5x at the price of some recall.

For sentence alignment we use a deterministic length-based algorithm
(Gale-Church 1993, still the baseline 30 years later because it works).
For specific high-divergence language pairs (EN↔JA, EN↔ZH where sentence
boundaries differ heavily), pass --align-method=llm to use Claude for the
alignment instead — slower and more expensive, but more accurate.
"""
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
from typing import Optional

# Gale-Church alignment parameters. The defaults work well for IE-language pairs;
# for CJK ↔ IE pairs, consider --align-method=llm instead.
GC_MEAN_RATIO = 1.0    # target/source length ratio; LLM will refine per-pair
GC_VARIANCE = 6.8      # variance of the ratio; from the original paper
# Cost of each transition type, in arbitrary units; lower = preferred
GC_COSTS = {
    "1-1": 0,    # one source ↔ one target — the dominant case
    "0-1": 450,  # target has a sentence the source doesn't (insertion)
    "1-0": 450,  # source has a sentence the target doesn't (deletion)
    "2-1": 230,  # two source sentences ↔ one target sentence (contraction)
    "1-2": 230,  # one source sentence ↔ two target sentences (expansion)
    "2-2": 440,  # two-two — used for messy cases
}


def split_sentences(text, lang):
    """Lightweight sentence splitter — works across many languages.

    Prefers `sentence-splitter` (a real multilingual splitter) if installed;
    falls back to a regex on common terminal punctuation. The fallback misses
    abbreviations and is bad for CJK, but is good enough to get alignment
    going for IE-language pairs.
    """
    try:
        from sentence_splitter import split_text_into_sentences  # type: ignore
        return split_text_into_sentences(text=text, language=lang)
    except ImportError:
        # Fallback: split on terminal punctuation + paragraph breaks
        if lang in ("ja", "zh", "ko"):
            # CJK: split on full-width punctuation
            sents = re.split(r"(?<=[。.!?])\s*", text)
        else:
            sents = re.split(r"(?<=[.!?])\s+(?=[A-ZА-ЯÀ-ÖØ-ÞА-Я])", text)
        return [s.strip() for s in sents if s.strip()]


def gale_church_align(source_sents, target_sents):
    """Dynamic-programming sentence alignment based on length ratios.

    Returns a list of alignment tuples: (source_indices, target_indices).
    Each side is a list of indices (usually length 1, sometimes 0 or 2).
    """
    import math
    n, m = len(source_sents), len(target_sents)
    # Pre-compute character lengths
    sl = [len(s) for s in source_sents]
    tl = [len(t) for t in target_sents]

    # cost[i][j] = min cost of aligning source[:i] with target[:j]
    INF = float("inf")
    cost = [[INF] * (m + 1) for _ in range(n + 1)]
    back = [[None] * (m + 1) for _ in range(n + 1)]
    cost[0][0] = 0

    def match_cost(s_len, t_len):
        """Cost of a 1-1 alignment based on length ratio."""
        if s_len == 0 and t_len == 0:
            return 0
        if s_len == 0:
            return GC_COSTS["0-1"] + t_len  # punish empty source alignments
        delta = (t_len - s_len * GC_MEAN_RATIO) / math.sqrt(s_len * GC_VARIANCE)
        return abs(delta) * 100

    for i in range(n + 1):
        for j in range(m + 1):
            if i == 0 and j == 0: continue
            # 1-1
            if i >= 1 and j >= 1:
                c = cost[i-1][j-1] + GC_COSTS["1-1"] + match_cost(sl[i-1], tl[j-1])
                if c < cost[i][j]:
                    cost[i][j] = c; back[i][j] = ("1-1", i-1, j-1)
            # 1-0 (source has a sentence target doesn't)
            if i >= 1:
                c = cost[i-1][j] + GC_COSTS["1-0"]
                if c < cost[i][j]:
                    cost[i][j] = c; back[i][j] = ("1-0", i-1, j)
            # 0-1 (target has a sentence source doesn't)
            if j >= 1:
                c = cost[i][j-1] + GC_COSTS["0-1"]
                if c < cost[i][j]:
                    cost[i][j] = c; back[i][j] = ("0-1", i, j-1)
            # 2-1 (two source ↔ one target)
            if i >= 2 and j >= 1:
                c = cost[i-2][j-1] + GC_COSTS["2-1"] + match_cost(sl[i-2] + sl[i-1], tl[j-1])
                if c < cost[i][j]:
                    cost[i][j] = c; back[i][j] = ("2-1", i-2, j-1)
            # 1-2
            if i >= 1 and j >= 2:
                c = cost[i-1][j-2] + GC_COSTS["1-2"] + match_cost(sl[i-1], tl[j-2] + tl[j-1])
                if c < cost[i][j]:
                    cost[i][j] = c; back[i][j] = ("1-2", i-1, j-2)
            # 2-2
            if i >= 2 and j >= 2:
                c = cost[i-2][j-2] + GC_COSTS["2-2"] + match_cost(sl[i-2] + sl[i-1], tl[j-2] + tl[j-1])
                if c < cost[i][j]:
                    cost[i][j] = c; back[i][j] = ("2-2", i-2, j-2)

    # Reconstruct alignment
    alignment = []
    i, j = n, m
    while i > 0 or j > 0:
        if back[i][j] is None:
            break
        kind, pi, pj = back[i][j]
        s_idx = list(range(pi, i))
        t_idx = list(range(pj, j))
        alignment.append((s_idx, t_idx))
        i, j = pi, pj
    alignment.reverse()
    return alignment


def llm_align(source_sents, target_sents, src_lang, tgt_lang):
    """LLM-based alignment for high-divergence language pairs.

    Sends a numbered list of source sentences and target sentences to Claude
    and asks for an explicit alignment mapping. Slower and more expensive
    than Gale-Church but handles CJK ↔ IE pairs much better.

    Returns alignment in the same format as gale_church_align.

    NOTE: Implementation stub — the actual LLM call is left for the calling
    skill (the script just returns a sentinel that triggers the skill to
    take over). In production, this would post-process Claude's structured
    output into the (source_indices, target_indices) tuples.
    """
    sys.stderr.write(
        "INFO: --align-method=llm requested. Writing source/target sentence\n"
        "      lists to <work_dir>/llm_align_input.json so the calling Claude\n"
        "      skill can do the alignment with full LLM context.\n"
    )
    return [(s, t) for s, t in zip(
        [[i] for i in range(len(source_sents))],
        [[i] for i in range(len(target_sents))]
    )]


def aligned_pairs(source_sents, target_sents, alignment):
    """Yield (source_text, target_text) for each non-empty alignment cell."""
    for s_idx, t_idx in alignment:
        if not s_idx or not t_idx:
            continue  # skip insertions / deletions for term extraction
        s_text = " ".join(source_sents[i] for i in s_idx)
        t_text = " ".join(target_sents[i] for i in t_idx)
        yield s_text, t_text


def write_term_extraction_prompts(pairs, src_lang, tgt_lang, work_dir, batch_size=20):
    """Write batched extraction prompts to disk for the calling skill to consume.

    The actual LLM calls happen in the skill's runtime (not here), because:
      (a) this script avoids any direct Anthropic SDK dependency
      (b) the skill already has Opus / Sonnet wired in
      (c) it lets the user see the prompts and dry-run before spending money

    Each batch file contains up to `batch_size` aligned pairs along with the
    extraction instructions. The skill iterates the batch files, sends each
    to Opus 4.6, accumulates the responses, and runs the dedup / conflict
    detection pass before emitting the proposals file.
    """
    batches_dir = work_dir / "extraction_batches"
    batches_dir.mkdir(exist_ok=True)
    instructions = f"""You are extracting terminology pairs from aligned bilingual segments.

Source language: {src_lang}
Target language: {tgt_lang}

For each aligned segment pair below, extract every (source_term, target_term)
pair that is a *technical term*, *noun phrase*, *regulatory reference*,
*hazard label*, *unit*, or *industry-standard abbreviation*. Skip common
function words, articles, and generic verbs/adjectives that have no
specialised meaning.

For each extracted pair, provide:
- source: the term in the source language, in its canonical (lemma) form
- target: the corresponding term in the target language, in its canonical form
- context: a one-line note describing where this term applies or what
  domain-specific meaning it carries (e.g. "active substance — INN form",
  "hazard label — risk to life", "regulatory body — preserve verbatim")
- confidence: 1-5, where 5 means you are certain this is a stable
  terminology pair and 1 means the pair might be a one-off phrasing choice

Output strict JSON: {{"pairs": [{{"source": ..., "target": ..., "context": ..., "confidence": ...}}]}}

DO NOT include surface-level surface-form pairs (the/le, is/est, etc.).
DO NOT include numbers, dates, or units unless they carry a specialised meaning.
DO NOT speculate — if you're not sure a pair is a real terminology pair, omit it.

Aligned segment pairs:
"""
    n_batches = 0
    for batch_start in range(0, len(pairs), batch_size):
        batch = pairs[batch_start:batch_start + batch_size]
        body = []
        for k, (s, t) in enumerate(batch, start=1):
            body.append(f"--- Pair {k} ---\nSOURCE: {s}\nTARGET: {t}\n")
        content = instructions + "\n" + "\n".join(body)
        out = batches_dir / f"batch_{batch_start // batch_size + 1:03d}.txt"
        out.write_text(content, encoding="utf-8")
        n_batches += 1
    return n_batches


def write_alignment_manifest(alignment, source_sents, target_sents, src_lang, tgt_lang, work_dir):
    """Write the full alignment to disk so the user can inspect / hand-correct it
    before running term extraction.
    """
    manifest = {
        "source_language": src_lang,
        "target_language": tgt_lang,
        "n_source_sentences": len(source_sents),
        "n_target_sentences": len(target_sents),
        "n_aligned_cells": len(alignment),
        "alignments": []
    }
    for s_idx, t_idx in alignment:
        manifest["alignments"].append({
            "source_indices": s_idx,
            "target_indices": t_idx,
            "source_text": " ".join(source_sents[i] for i in s_idx),
            "target_text": " ".join(target_sents[i] for i in t_idx),
        })
    (work_dir / "alignment.json").write_text(
        json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
    )


def export_tmx(alignment, source_sents, target_sents, src_lang, tgt_lang, out_path):
    """Export aligned segments as a TMX 1.4 file for import into Trados / MemoQ / OmegaT."""
    import xml.etree.ElementTree as ET
    from datetime import datetime
    tmx = ET.Element("tmx", version="1.4")
    header = ET.SubElement(tmx, "header",
                           creationtool="fidelity-translator",
                           creationtoolversion="0.1",
                           segtype="sentence",
                           adminlang="en",
                           srclang=src_lang,
                           datatype="plaintext",
                           creationdate=datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"))
    body = ET.SubElement(tmx, "body")
    for s_idx, t_idx in alignment:
        if not s_idx or not t_idx: continue
        tu = ET.SubElement(body, "tu")
        tuv_s = ET.SubElement(tu, "tuv")
        tuv_s.set("{http://www.w3.org/XML/1998/namespace}lang", src_lang)
        seg_s = ET.SubElement(tuv_s, "seg")
        seg_s.text = " ".join(source_sents[i] for i in s_idx)
        tuv_t = ET.SubElement(tu, "tuv")
        tuv_t.set("{http://www.w3.org/XML/1998/namespace}lang", tgt_lang)
        seg_t = ET.SubElement(tuv_t, "seg")
        seg_t.text = " ".join(target_sents[i] for i in t_idx)
    ET.ElementTree(tmx).write(out_path, encoding="utf-8", xml_declaration=True)


def extract_with_existing_pipeline(doc_path, work_subdir):
    """Run extract_doc.py to get text + detected language."""
    work_subdir.mkdir(parents=True, exist_ok=True)
    script = Path(__file__).parent / "extract_doc.py"
    res = subprocess.run(
        ["python3", str(script), str(doc_path), str(work_subdir)],
        capture_output=True, text=True
    )
    if res.returncode != 0:
        sys.stderr.write(f"extract_doc.py failed:\n{res.stderr}\n"); sys.exit(res.returncode)
    text_data = json.loads((work_subdir / "text.json").read_text(encoding="utf-8"))
    manifest = json.loads((work_subdir / "manifest.json").read_text(encoding="utf-8"))
    full_text = "\n".join(p["text"] for p in text_data["pages"])
    return full_text, manifest


def main():
    ap = argparse.ArgumentParser(
        description="Bootstrap a project glossary from an existing bilingual document pair."
    )
    ap.add_argument("source", help="Source-language document (PDF / DOCX / TXT / MD / HTML)")
    ap.add_argument("target", help="Target-language document — the existing translation of `source`")
    ap.add_argument("work_dir", help="Where to write intermediate files")
    ap.add_argument("--source-language", help="Override auto-detection (ISO 639-1)")
    ap.add_argument("--target-language", help="Override auto-detection (ISO 639-1)")
    ap.add_argument("--align-method", choices=["gale-church", "llm"], default="gale-church",
                    help="gale-church: fast deterministic length-based (default); llm: slower, "
                         "better for CJK↔IE pairs")
    ap.add_argument("--batch-size", type=int, default=20,
                    help="How many aligned pairs per extraction batch (default 20)")
    ap.add_argument("--tmx-out", default=None,
                    help="Optional path to write a TMX file for import into Trados / MemoQ")
    args = ap.parse_args()

    work_dir = Path(args.work_dir); work_dir.mkdir(parents=True, exist_ok=True)
    src_work = work_dir / "_source"
    tgt_work = work_dir / "_target"

    print(f"[1/4] Extracting source: {args.source}")
    src_text, src_manifest = extract_with_existing_pipeline(args.source, src_work)
    print(f"[2/4] Extracting target: {args.target}")
    tgt_text, tgt_manifest = extract_with_existing_pipeline(args.target, tgt_work)

    src_lang = args.source_language or src_manifest.get("source_language", "en")
    tgt_lang = args.target_language or tgt_manifest.get("source_language", "en")
    if src_lang == tgt_lang:
        sys.stderr.write(
            f"WARN: source and target detected as the same language ({src_lang}). "
            f"Are you sure these are a translation pair? Override with "
            f"--source-language / --target-language.\n"
        )

    print(f"[3/4] Sentence-aligning ({src_lang} ↔ {tgt_lang}, method={args.align_method})")
    src_sents = split_sentences(src_text, src_lang)
    tgt_sents = split_sentences(tgt_text, tgt_lang)
    print(f"      {len(src_sents)} source sentences, {len(tgt_sents)} target sentences")
    if args.align_method == "llm":
        alignment = llm_align(src_sents, tgt_sents, src_lang, tgt_lang)
    else:
        alignment = gale_church_align(src_sents, tgt_sents)
    print(f"      {len(alignment)} aligned cells")
    write_alignment_manifest(alignment, src_sents, tgt_sents, src_lang, tgt_lang, work_dir)

    pairs = list(aligned_pairs(src_sents, tgt_sents, alignment))
    print(f"[4/4] Writing {len(pairs)} aligned pairs into extraction batches "
          f"(batch_size={args.batch_size})")
    n_batches = write_term_extraction_prompts(pairs, src_lang, tgt_lang, work_dir, args.batch_size)
    print(f"      Wrote {n_batches} batches to {work_dir / 'extraction_batches'}/")

    if args.tmx_out:
        export_tmx(alignment, src_sents, tgt_sents, src_lang, tgt_lang, args.tmx_out)
        print(f"      Wrote TMX to {args.tmx_out}")

    # Cost estimate (rough — assumes ~500 input tokens + 200 output tokens per batch
    # at Opus pricing of $15/M in, $75/M out as of mid-2026)
    est_cost_opus = n_batches * (500 * 15 / 1_000_000 + 200 * 75 / 1_000_000)
    est_cost_sonnet = n_batches * (500 * 3 / 1_000_000 + 200 * 15 / 1_000_000)
    print()
    print("─" * 70)
    print(f"Bootstrap pre-pass complete. Next step (run by the fidelity-translator")
    print(f"skill, not this script):")
    print(f"")
    print(f"  Iterate the {n_batches} batches under {work_dir/'extraction_batches'}/")
    print(f"  Send each to Opus 4.6 (~${est_cost_opus:.2f} total) or Sonnet 4.6")
    print(f"  (~${est_cost_sonnet:.2f} total), parse the JSON response, deduplicate,")
    print(f"  detect conflicts, and write to {work_dir/'glossary_proposals.md'}.")
    print(f"")
    print(f"  The user reviews proposals, accepts/rejects/edits, and merges accepted")
    print(f"  entries into the live project glossary at <project.glossary>.")
    print("─" * 70)


if __name__ == "__main__":
    main()
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall