Skip to content

Tasdax123/claude-fidelity-translator / plugins/fidelity-translator/skills/fidelity-translator

fidelity-translator

Verified

scripts/extract_doc.py

View on GitHub
#!/usr/bin/env python3
"""Extract text and images from a source document (PDF / DOCX / TXT / HTML / MD).

Branches by file extension. Writes:
  <work_dir>/text.json       — page-by-page text (DOCX/TXT collapse to one 'page')
  <work_dir>/images/         — extracted images, renamed with page/size metadata
  <work_dir>/manifest.json   — source, page count, detected language, chapters, image inventory

Language detection: script-ratio (Cyrillic / Han / Hangul / Hiragana-Katakana / Devanagari /
Arabic / Hebrew / Greek / Latin), with `langdetect` as a fallback when available.

Chapter detection is multilingual (English, Ukrainian, German, French, Spanish, Portuguese,
Italian, Chinese, Japanese, Korean, Russian) — see CHAPTER_PATTERNS.

Image filter: images below MIN_TRANSLATE_W × MIN_TRANSLATE_H are flagged `translate: False`
by default (almost always icons / bullet markers / stamps), but they are still embedded
when the markdown references them.
"""
import argparse, json, re, subprocess, sys, unicodedata
from pathlib import Path

MIN_TRANSLATE_W = 200
MIN_TRANSLATE_H = 150

# Multilingual chapter heading patterns. Order matters only for tie-breaking.
CHAPTER_PATTERNS = [
    (re.compile(r"^\s*CHAPTER\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "en"),
    (re.compile(r"^\s*РОЗДІЛ\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "uk"),
    (re.compile(r"^\s*ГЛАВА\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "ru"),
    (re.compile(r"^\s*KAPITEL\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "de"),
    (re.compile(r"^\s*CHAPITRE\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "fr"),
    (re.compile(r"^\s*CAP[IÍ]TULO\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "es-pt"),
    (re.compile(r"^\s*CAPITOLO\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "it"),
    (re.compile(r"^\s*第\s*(\d{1,3})\s*章", re.MULTILINE), "zh-ja"),
    (re.compile(r"^\s*제\s*(\d{1,3})\s*장", re.MULTILINE), "ko"),
]

# Script ranges for language detection (the keys are ISO 15924 / loose script names).
SCRIPT_RANGES = {
    "Cyrillic": (0x0400, 0x04FF),
    "Han":      (0x4E00, 0x9FFF),
    "Hiragana": (0x3040, 0x309F),
    "Katakana": (0x30A0, 0x30FF),
    "Hangul":   (0xAC00, 0xD7A3),
    "Arabic":   (0x0600, 0x06FF),
    "Hebrew":   (0x0590, 0x05FF),
    "Greek":    (0x0370, 0x03FF),
    "Devanagari": (0x0900, 0x097F),
    "Thai":     (0x0E00, 0x0E7F),
    "Latin":    (0x0041, 0x024F),
}


def run(cmd, **kw):
    res = subprocess.run(cmd, capture_output=True, text=True, **kw)
    if res.returncode != 0:
        sys.stderr.write(f"FAILED: {' '.join(cmd)}\n{res.stderr}\n")
        sys.exit(res.returncode)
    return res


def script_histogram(text, limit=20000):
    """Return {script_name: char_count} for the first `limit` chars."""
    hist = {k: 0 for k in SCRIPT_RANGES}
    for ch in text[:limit]:
        cp = ord(ch)
        for name, (lo, hi) in SCRIPT_RANGES.items():
            if lo <= cp <= hi:
                hist[name] += 1
                break
    return hist


def detect_language(text):
    """Best-effort source-language ID. Returns ISO 639-1 code + confidence + signal name."""
    hist = script_histogram(text)
    total = sum(hist.values()) or 1
    ratios = {k: v / total for k, v in hist.items()}
    # Script-first triage: an unambiguous non-Latin script wins immediately.
    if ratios["Hangul"] > 0.20:
        return "ko", round(ratios["Hangul"], 3), "Hangul ratio"
    if ratios["Hiragana"] + ratios["Katakana"] > 0.15:
        return "ja", round(ratios["Hiragana"] + ratios["Katakana"], 3), "kana ratio"
    if ratios["Han"] > 0.30 and ratios["Hiragana"] + ratios["Katakana"] < 0.05:
        return "zh", round(ratios["Han"], 3), "Han ratio (no kana)"
    if ratios["Cyrillic"] > 0.50:
        # Need to distinguish Ukrainian from Russian. Ukrainian has unique letters: і, ї, є, ґ.
        ua_markers = sum(1 for ch in text[:20000] if ch in "іїєґІЇЄҐ")
        if ua_markers > 5:
            return "uk", round(ratios["Cyrillic"], 3), "Cyrillic + UA markers"
        return "ru", round(ratios["Cyrillic"], 3), "Cyrillic ratio"
    if ratios["Arabic"] > 0.30:
        return "ar", round(ratios["Arabic"], 3), "Arabic ratio"
    if ratios["Hebrew"] > 0.30:
        return "he", round(ratios["Hebrew"], 3), "Hebrew ratio"
    if ratios["Greek"] > 0.30:
        return "el", round(ratios["Greek"], 3), "Greek ratio"
    if ratios["Devanagari"] > 0.30:
        return "hi", round(ratios["Devanagari"], 3), "Devanagari ratio"
    if ratios["Thai"] > 0.30:
        return "th", round(ratios["Thai"], 3), "Thai ratio"
    # Latin-script: fall back to langdetect if installed, else default to English.
    if ratios["Latin"] > 0.50:
        try:
            from langdetect import detect, detect_langs  # type: ignore
            langs = detect_langs(text[:5000])
            top = langs[0]
            return top.lang, round(top.prob, 3), "langdetect"
        except Exception:
            return "en", round(ratios["Latin"], 3), "Latin (no langdetect)"
    return "unknown", 0.0, "ambiguous"


def extract_text_from_pdf(pdf, work_dir, first=None, last=None):
    info = run(["pdfinfo", pdf]).stdout
    n_pages = int(re.search(r"Pages:\s+(\d+)", info).group(1))
    lo, hi = first or 1, last or n_pages
    pages = []
    for p in range(lo, hi + 1):
        out = run(["pdftotext", "-layout", "-f", str(p), "-l", str(p), pdf, "-"]).stdout
        pages.append({"page": p, "text": out})
    return {"pages": pages, "meta": {"source": pdf, "n_pages": n_pages, "range": [lo, hi]}}


def extract_images_from_pdf(pdf, work_dir, first=None, last=None):
    img_dir = Path(work_dir) / "images"
    img_dir.mkdir(parents=True, exist_ok=True)
    cmd = ["pdfimages", "-all", "-p"]
    if first is not None: cmd += ["-f", str(first)]
    if last is not None: cmd += ["-l", str(last)]
    cmd += [pdf, str(img_dir / "raw")]
    run(cmd)
    list_args = ["pdfimages", "-list"]
    if first is not None: list_args += ["-f", str(first)]
    if last is not None: list_args += ["-l", str(last)]
    list_args.append(pdf)
    listing = run(list_args).stdout.splitlines()
    images = []
    for line in listing[2:]:
        parts = line.split()
        if len(parts) < 5: continue
        try:
            page = int(parts[0]); num = int(parts[1])
            width = int(parts[3]); height = int(parts[4])
        except (ValueError, IndexError):
            continue
        images.append({"page": page, "num": num, "width": width, "height": height})
    by_pn = {(im["page"], im["num"]): im for im in images}
    renamed = []
    for raw in sorted(img_dir.glob("raw-*.*")):
        m = re.match(r"raw-(\d+)-(\d+)\.(\w+)", raw.name)
        if m:
            page = int(m.group(1)); num = int(m.group(2)); ext = m.group(3)
            meta = by_pn.get((page, num)) or {"page": page, "num": num, "width": 0, "height": 0}
        else:
            m2 = re.match(r"raw-(\d+)\.(\w+)", raw.name)
            if not m2: continue
            gidx = int(m2.group(1)); ext = m2.group(2)
            if gidx >= len(images): continue
            meta = images[gidx]
        new_name = f"p{meta['page']:03d}_i{meta['num']:03d}_{meta['width']}x{meta['height']}.{ext}"
        new_path = img_dir / new_name
        raw.rename(new_path)
        translate = meta["width"] >= MIN_TRANSLATE_W and meta["height"] >= MIN_TRANSLATE_H
        renamed.append({"page": meta["page"], "num": meta["num"],
                        "width": meta["width"], "height": meta["height"],
                        "path": str(new_path.relative_to(Path(work_dir))),
                        "translate": translate})
    return renamed


def extract_from_docx(docx, work_dir):
    """Extract paragraphs and images from a .docx via python-docx + zipfile.

    DOCX has no real 'pages' concept; we synthesise one page per ~3000 chars
    so the downstream chunker has something to grip onto.
    """
    try:
        from docx import Document  # type: ignore
    except ImportError:
        sys.stderr.write("python-docx not installed: pip install python-docx\n"); sys.exit(2)
    import zipfile, shutil
    doc = Document(docx)
    full_text = []
    for para in doc.paragraphs:
        full_text.append(para.text)
    text = "\n".join(full_text)
    # Synthesise pages of ~3000 chars
    pages = []
    chunk_size = 3000
    pos = 0; page_no = 1
    while pos < len(text):
        # Break on paragraph boundary if possible
        end = min(pos + chunk_size, len(text))
        if end < len(text):
            nl = text.rfind("\n", pos, end)
            if nl > pos + 1000: end = nl
        pages.append({"page": page_no, "text": text[pos:end]})
        pos = end; page_no += 1
    # Extract embedded images from the DOCX archive
    img_dir = Path(work_dir) / "images"; img_dir.mkdir(parents=True, exist_ok=True)
    images = []
    with zipfile.ZipFile(docx) as z:
        media = [n for n in z.namelist() if n.startswith("word/media/")]
        for idx, m in enumerate(media, start=1):
            ext = Path(m).suffix.lstrip(".")
            new_name = f"docx_i{idx:03d}.{ext}"
            with z.open(m) as src, (img_dir / new_name).open("wb") as dst:
                shutil.copyfileobj(src, dst)
            # We don't have width/height without parsing the image; flag for translation
            # by default and let the user opt out at runtime.
            images.append({"page": 1, "num": idx,
                           "width": 0, "height": 0,
                           "path": str((img_dir / new_name).relative_to(Path(work_dir))),
                           "translate": True})
    return {"pages": pages, "meta": {"source": docx, "n_pages": len(pages), "range": [1, len(pages)]}}, images


def extract_from_text(path, work_dir):
    text = Path(path).read_text(encoding="utf-8")
    # Paragraphs/pages: split on form-feed if present, else one page.
    if "\x0c" in text:
        pages = [{"page": i+1, "text": t} for i, t in enumerate(text.split("\x0c"))]
    else:
        pages = [{"page": 1, "text": text}]
    return {"pages": pages, "meta": {"source": path, "n_pages": len(pages), "range": [1, len(pages)]}}, []


def detect_chapters(pages):
    boundaries = []; seen = {}
    matched_lang = None
    for pattern, lang in CHAPTER_PATTERNS:
        for entry in pages:
            for m in pattern.finditer(entry["text"]):
                num = int(m.group(1))
                if num not in seen:
                    seen[num] = entry["page"]
                    boundaries.append({"chapter": num, "start_page": entry["page"], "lang": lang})
                    matched_lang = lang
        if seen:
            break  # First pattern that matches wins; don't double-count
    boundaries.sort(key=lambda b: b["start_page"])
    for i, b in enumerate(boundaries):
        b["end_page"] = boundaries[i+1]["start_page"] - 1 if i+1 < len(boundaries) else (pages[-1]["page"] if pages else b["start_page"])
    if not boundaries and pages:
        boundaries = [{"chapter": 0, "start_page": pages[0]["page"],
                       "end_page": pages[-1]["page"], "lang": None}]
    return boundaries, matched_lang


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("source")
    ap.add_argument("work_dir")
    ap.add_argument("--pages", help="START-END, 1-indexed (PDF only)")
    ap.add_argument("--target-language", help="ISO 639-1 target language code (default: en, unless source is en)")
    args = ap.parse_args()
    work_dir = Path(args.work_dir); work_dir.mkdir(parents=True, exist_ok=True)
    src = args.source
    ext = Path(src).suffix.lower()

    if ext == ".pdf":
        first = last = None
        if args.pages:
            a, b = args.pages.split("-"); first, last = int(a), int(b)
        text = extract_text_from_pdf(src, work_dir, first, last)
        # Check for empty text — sign of a scanned PDF
        total_chars = sum(len(p["text"].strip()) for p in text["pages"])
        if total_chars < 50 * len(text["pages"]):
            sys.stderr.write(
                "ERROR: PDF appears to be scanned (no extractable text layer).\n"
                "Run OCR first (tesseract / paddleOCR / ABBYY) and re-submit.\n"
            )
            sys.exit(3)
        images = extract_images_from_pdf(src, work_dir, first, last)
    elif ext == ".docx":
        text, images = extract_from_docx(src, work_dir)
    elif ext in (".txt", ".md", ".html", ".htm"):
        text, images = extract_from_text(src, work_dir)
    else:
        sys.stderr.write(f"ERROR: unsupported extension {ext}\n"); sys.exit(2)

    (work_dir / "text.json").write_text(json.dumps(text, ensure_ascii=False, indent=2))
    chapters, chapter_lang = detect_chapters(text["pages"])
    full_text = "".join(p["text"] for p in text["pages"])
    src_lang, conf, signal = detect_language(full_text)

    # Target language: explicit arg > 'en' default unless source IS English > prompt
    if args.target_language:
        tgt_lang = args.target_language
    elif src_lang != "en":
        tgt_lang = "en"
    else:
        tgt_lang = None  # caller must prompt

    translate_count = sum(1 for i in images if i["translate"])
    manifest = {
        "source": src,
        "page_range": text["meta"]["range"],
        "n_pages": text["meta"]["n_pages"],
        "source_language": src_lang,
        "source_language_confidence": conf,
        "source_language_signal": signal,
        "target_language": tgt_lang,
        "chapters": chapters,
        "chapter_pattern_lang": chapter_lang,
        "images": images,
        "images_total": len(images),
        "images_to_translate": translate_count,
        "image_size_threshold": {"min_width": MIN_TRANSLATE_W, "min_height": MIN_TRANSLATE_H},
    }
    (work_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
    tgt_display = tgt_lang or "(unset — please specify)"
    print(f"Extracted {len(text['pages'])} pages, {len(images)} images "
          f"({translate_count} flagged), {len(chapters)} chapters detected.")
    print(f"Source language: {src_lang} ({signal}, conf={conf}) → target: {tgt_display}.")


if __name__ == "__main__":
    main()
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall