#!/usr/bin/env python3
"""Extract text and images from a source document (PDF / DOCX / TXT / HTML / MD).
Branches by file extension. Writes:
<work_dir>/text.json — page-by-page text (DOCX/TXT collapse to one 'page')
<work_dir>/images/ — extracted images, renamed with page/size metadata
<work_dir>/manifest.json — source, page count, detected language, chapters, image inventory
Language detection: script-ratio (Cyrillic / Han / Hangul / Hiragana-Katakana / Devanagari /
Arabic / Hebrew / Greek / Latin), with `langdetect` as a fallback when available.
Chapter detection is multilingual (English, Ukrainian, German, French, Spanish, Portuguese,
Italian, Chinese, Japanese, Korean, Russian) — see CHAPTER_PATTERNS.
Image filter: images below MIN_TRANSLATE_W × MIN_TRANSLATE_H are flagged `translate: False`
by default (almost always icons / bullet markers / stamps), but they are still embedded
when the markdown references them.
"""
import argparse, json, re, subprocess, sys, unicodedata
from pathlib import Path
MIN_TRANSLATE_W = 200
MIN_TRANSLATE_H = 150
# Multilingual chapter heading patterns. Order matters only for tie-breaking.
CHAPTER_PATTERNS = [
(re.compile(r"^\s*CHAPTER\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "en"),
(re.compile(r"^\s*РОЗДІЛ\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "uk"),
(re.compile(r"^\s*ГЛАВА\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "ru"),
(re.compile(r"^\s*KAPITEL\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "de"),
(re.compile(r"^\s*CHAPITRE\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "fr"),
(re.compile(r"^\s*CAP[IÍ]TULO\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "es-pt"),
(re.compile(r"^\s*CAPITOLO\s+(\d{1,3})\b", re.MULTILINE | re.IGNORECASE), "it"),
(re.compile(r"^\s*第\s*(\d{1,3})\s*章", re.MULTILINE), "zh-ja"),
(re.compile(r"^\s*제\s*(\d{1,3})\s*장", re.MULTILINE), "ko"),
]
# Script ranges for language detection (the keys are ISO 15924 / loose script names).
SCRIPT_RANGES = {
"Cyrillic": (0x0400, 0x04FF),
"Han": (0x4E00, 0x9FFF),
"Hiragana": (0x3040, 0x309F),
"Katakana": (0x30A0, 0x30FF),
"Hangul": (0xAC00, 0xD7A3),
"Arabic": (0x0600, 0x06FF),
"Hebrew": (0x0590, 0x05FF),
"Greek": (0x0370, 0x03FF),
"Devanagari": (0x0900, 0x097F),
"Thai": (0x0E00, 0x0E7F),
"Latin": (0x0041, 0x024F),
}
def run(cmd, **kw):
res = subprocess.run(cmd, capture_output=True, text=True, **kw)
if res.returncode != 0:
sys.stderr.write(f"FAILED: {' '.join(cmd)}\n{res.stderr}\n")
sys.exit(res.returncode)
return res
def script_histogram(text, limit=20000):
"""Return {script_name: char_count} for the first `limit` chars."""
hist = {k: 0 for k in SCRIPT_RANGES}
for ch in text[:limit]:
cp = ord(ch)
for name, (lo, hi) in SCRIPT_RANGES.items():
if lo <= cp <= hi:
hist[name] += 1
break
return hist
def detect_language(text):
"""Best-effort source-language ID. Returns ISO 639-1 code + confidence + signal name."""
hist = script_histogram(text)
total = sum(hist.values()) or 1
ratios = {k: v / total for k, v in hist.items()}
# Script-first triage: an unambiguous non-Latin script wins immediately.
if ratios["Hangul"] > 0.20:
return "ko", round(ratios["Hangul"], 3), "Hangul ratio"
if ratios["Hiragana"] + ratios["Katakana"] > 0.15:
return "ja", round(ratios["Hiragana"] + ratios["Katakana"], 3), "kana ratio"
if ratios["Han"] > 0.30 and ratios["Hiragana"] + ratios["Katakana"] < 0.05:
return "zh", round(ratios["Han"], 3), "Han ratio (no kana)"
if ratios["Cyrillic"] > 0.50:
# Need to distinguish Ukrainian from Russian. Ukrainian has unique letters: і, ї, є, ґ.
ua_markers = sum(1 for ch in text[:20000] if ch in "іїєґІЇЄҐ")
if ua_markers > 5:
return "uk", round(ratios["Cyrillic"], 3), "Cyrillic + UA markers"
return "ru", round(ratios["Cyrillic"], 3), "Cyrillic ratio"
if ratios["Arabic"] > 0.30:
return "ar", round(ratios["Arabic"], 3), "Arabic ratio"
if ratios["Hebrew"] > 0.30:
return "he", round(ratios["Hebrew"], 3), "Hebrew ratio"
if ratios["Greek"] > 0.30:
return "el", round(ratios["Greek"], 3), "Greek ratio"
if ratios["Devanagari"] > 0.30:
return "hi", round(ratios["Devanagari"], 3), "Devanagari ratio"
if ratios["Thai"] > 0.30:
return "th", round(ratios["Thai"], 3), "Thai ratio"
# Latin-script: fall back to langdetect if installed, else default to English.
if ratios["Latin"] > 0.50:
try:
from langdetect import detect, detect_langs # type: ignore
langs = detect_langs(text[:5000])
top = langs[0]
return top.lang, round(top.prob, 3), "langdetect"
except Exception:
return "en", round(ratios["Latin"], 3), "Latin (no langdetect)"
return "unknown", 0.0, "ambiguous"
def extract_text_from_pdf(pdf, work_dir, first=None, last=None):
info = run(["pdfinfo", pdf]).stdout
n_pages = int(re.search(r"Pages:\s+(\d+)", info).group(1))
lo, hi = first or 1, last or n_pages
pages = []
for p in range(lo, hi + 1):
out = run(["pdftotext", "-layout", "-f", str(p), "-l", str(p), pdf, "-"]).stdout
pages.append({"page": p, "text": out})
return {"pages": pages, "meta": {"source": pdf, "n_pages": n_pages, "range": [lo, hi]}}
def extract_images_from_pdf(pdf, work_dir, first=None, last=None):
img_dir = Path(work_dir) / "images"
img_dir.mkdir(parents=True, exist_ok=True)
cmd = ["pdfimages", "-all", "-p"]
if first is not None: cmd += ["-f", str(first)]
if last is not None: cmd += ["-l", str(last)]
cmd += [pdf, str(img_dir / "raw")]
run(cmd)
list_args = ["pdfimages", "-list"]
if first is not None: list_args += ["-f", str(first)]
if last is not None: list_args += ["-l", str(last)]
list_args.append(pdf)
listing = run(list_args).stdout.splitlines()
images = []
for line in listing[2:]:
parts = line.split()
if len(parts) < 5: continue
try:
page = int(parts[0]); num = int(parts[1])
width = int(parts[3]); height = int(parts[4])
except (ValueError, IndexError):
continue
images.append({"page": page, "num": num, "width": width, "height": height})
by_pn = {(im["page"], im["num"]): im for im in images}
renamed = []
for raw in sorted(img_dir.glob("raw-*.*")):
m = re.match(r"raw-(\d+)-(\d+)\.(\w+)", raw.name)
if m:
page = int(m.group(1)); num = int(m.group(2)); ext = m.group(3)
meta = by_pn.get((page, num)) or {"page": page, "num": num, "width": 0, "height": 0}
else:
m2 = re.match(r"raw-(\d+)\.(\w+)", raw.name)
if not m2: continue
gidx = int(m2.group(1)); ext = m2.group(2)
if gidx >= len(images): continue
meta = images[gidx]
new_name = f"p{meta['page']:03d}_i{meta['num']:03d}_{meta['width']}x{meta['height']}.{ext}"
new_path = img_dir / new_name
raw.rename(new_path)
translate = meta["width"] >= MIN_TRANSLATE_W and meta["height"] >= MIN_TRANSLATE_H
renamed.append({"page": meta["page"], "num": meta["num"],
"width": meta["width"], "height": meta["height"],
"path": str(new_path.relative_to(Path(work_dir))),
"translate": translate})
return renamed
def extract_from_docx(docx, work_dir):
"""Extract paragraphs and images from a .docx via python-docx + zipfile.
DOCX has no real 'pages' concept; we synthesise one page per ~3000 chars
so the downstream chunker has something to grip onto.
"""
try:
from docx import Document # type: ignore
except ImportError:
sys.stderr.write("python-docx not installed: pip install python-docx\n"); sys.exit(2)
import zipfile, shutil
doc = Document(docx)
full_text = []
for para in doc.paragraphs:
full_text.append(para.text)
text = "\n".join(full_text)
# Synthesise pages of ~3000 chars
pages = []
chunk_size = 3000
pos = 0; page_no = 1
while pos < len(text):
# Break on paragraph boundary if possible
end = min(pos + chunk_size, len(text))
if end < len(text):
nl = text.rfind("\n", pos, end)
if nl > pos + 1000: end = nl
pages.append({"page": page_no, "text": text[pos:end]})
pos = end; page_no += 1
# Extract embedded images from the DOCX archive
img_dir = Path(work_dir) / "images"; img_dir.mkdir(parents=True, exist_ok=True)
images = []
with zipfile.ZipFile(docx) as z:
media = [n for n in z.namelist() if n.startswith("word/media/")]
for idx, m in enumerate(media, start=1):
ext = Path(m).suffix.lstrip(".")
new_name = f"docx_i{idx:03d}.{ext}"
with z.open(m) as src, (img_dir / new_name).open("wb") as dst:
shutil.copyfileobj(src, dst)
# We don't have width/height without parsing the image; flag for translation
# by default and let the user opt out at runtime.
images.append({"page": 1, "num": idx,
"width": 0, "height": 0,
"path": str((img_dir / new_name).relative_to(Path(work_dir))),
"translate": True})
return {"pages": pages, "meta": {"source": docx, "n_pages": len(pages), "range": [1, len(pages)]}}, images
def extract_from_text(path, work_dir):
text = Path(path).read_text(encoding="utf-8")
# Paragraphs/pages: split on form-feed if present, else one page.
if "\x0c" in text:
pages = [{"page": i+1, "text": t} for i, t in enumerate(text.split("\x0c"))]
else:
pages = [{"page": 1, "text": text}]
return {"pages": pages, "meta": {"source": path, "n_pages": len(pages), "range": [1, len(pages)]}}, []
def detect_chapters(pages):
boundaries = []; seen = {}
matched_lang = None
for pattern, lang in CHAPTER_PATTERNS:
for entry in pages:
for m in pattern.finditer(entry["text"]):
num = int(m.group(1))
if num not in seen:
seen[num] = entry["page"]
boundaries.append({"chapter": num, "start_page": entry["page"], "lang": lang})
matched_lang = lang
if seen:
break # First pattern that matches wins; don't double-count
boundaries.sort(key=lambda b: b["start_page"])
for i, b in enumerate(boundaries):
b["end_page"] = boundaries[i+1]["start_page"] - 1 if i+1 < len(boundaries) else (pages[-1]["page"] if pages else b["start_page"])
if not boundaries and pages:
boundaries = [{"chapter": 0, "start_page": pages[0]["page"],
"end_page": pages[-1]["page"], "lang": None}]
return boundaries, matched_lang
def main():
ap = argparse.ArgumentParser()
ap.add_argument("source")
ap.add_argument("work_dir")
ap.add_argument("--pages", help="START-END, 1-indexed (PDF only)")
ap.add_argument("--target-language", help="ISO 639-1 target language code (default: en, unless source is en)")
args = ap.parse_args()
work_dir = Path(args.work_dir); work_dir.mkdir(parents=True, exist_ok=True)
src = args.source
ext = Path(src).suffix.lower()
if ext == ".pdf":
first = last = None
if args.pages:
a, b = args.pages.split("-"); first, last = int(a), int(b)
text = extract_text_from_pdf(src, work_dir, first, last)
# Check for empty text — sign of a scanned PDF
total_chars = sum(len(p["text"].strip()) for p in text["pages"])
if total_chars < 50 * len(text["pages"]):
sys.stderr.write(
"ERROR: PDF appears to be scanned (no extractable text layer).\n"
"Run OCR first (tesseract / paddleOCR / ABBYY) and re-submit.\n"
)
sys.exit(3)
images = extract_images_from_pdf(src, work_dir, first, last)
elif ext == ".docx":
text, images = extract_from_docx(src, work_dir)
elif ext in (".txt", ".md", ".html", ".htm"):
text, images = extract_from_text(src, work_dir)
else:
sys.stderr.write(f"ERROR: unsupported extension {ext}\n"); sys.exit(2)
(work_dir / "text.json").write_text(json.dumps(text, ensure_ascii=False, indent=2))
chapters, chapter_lang = detect_chapters(text["pages"])
full_text = "".join(p["text"] for p in text["pages"])
src_lang, conf, signal = detect_language(full_text)
# Target language: explicit arg > 'en' default unless source IS English > prompt
if args.target_language:
tgt_lang = args.target_language
elif src_lang != "en":
tgt_lang = "en"
else:
tgt_lang = None # caller must prompt
translate_count = sum(1 for i in images if i["translate"])
manifest = {
"source": src,
"page_range": text["meta"]["range"],
"n_pages": text["meta"]["n_pages"],
"source_language": src_lang,
"source_language_confidence": conf,
"source_language_signal": signal,
"target_language": tgt_lang,
"chapters": chapters,
"chapter_pattern_lang": chapter_lang,
"images": images,
"images_total": len(images),
"images_to_translate": translate_count,
"image_size_threshold": {"min_width": MIN_TRANSLATE_W, "min_height": MIN_TRANSLATE_H},
}
(work_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
tgt_display = tgt_lang or "(unset — please specify)"
print(f"Extracted {len(text['pages'])} pages, {len(images)} images "
f"({translate_count} flagged), {len(chapters)} chapters detected.")
print(f"Source language: {src_lang} ({signal}, conf={conf}) → target: {tgt_display}.")
if __name__ == "__main__":
main()