Skip to content

scdenney/open-science-skills / codex/vlm-ocr

vlm-ocr

Verified

OCR scanned or image-only corpora with vision-language models in four phases: `triage` inventories the scans, identifies script and language, and shortlists models by script, hardware, and data constraints; `evaluate` picks a system by measured CER/WER against ground truth, using born-digital twins, a second reader, and decision rules set in advance; `run` builds the production pipeline; and `clean` removes model artifacts, audits what every cleaning rule deletes, corrects with rule-based and LLM passes, and records provenance. Born-digital documents with a text layer go to $doc-to-markdown; runs on Leiden's ALICE cluster add $vlm-ocr-alice.

FileKindSize
SKILL.mdskill md76.5 KB
agents/openai.yamlconfig221 B
references/model-routing.mdmarkdown11.3 KB
references/prompt-and-schema.mdmarkdown13.3 KB
references/prompt-templates-and-schema.mdmarkdown9.0 KB
scripts/ocr_client.pyexecutablescript15.5 KB
scripts/triage.pyexecutablescript14.8 KB

scripts/triage.py

View on GitHub
#!/usr/bin/env python3
"""
Inventory a corpus of scans before choosing an OCR model.

`scan` reads every PDF (and every folder of page images) under the given
paths and writes triage.csv with one row per document: page count, a content
hash for spotting duplicate scans, whether a text layer exists and whether
it sits on top of page-sized images (a scan with an earlier OCR layer) or
is born-digital, /Rotate flags, landscape pages (often two-page spreads
that need splitting), page size, the scan resolution, the script
mix of any text layer, and a Latin-diacritics language hint. It renders
sample body pages to samples/ and draws contact sheets (twelve documents per
sheet) to sheets/, which are what a person or a vision model looks at to
name the script, language, era and layout. A text layer drawn invisibly
(render mode 3, or zero opacity) is an earlier OCR, not born-digital text.

`profile` reports the script mix and language of OCR output (.txt, or
.jsonl with a "text" field). Run it on a pilot read of the sample pages to
name the language of scans without a text layer, and on a finished run to
find pages whose script does not match the rest of the book.

    python triage.py scan corpus/ --out triage/ [--lid]
    python triage.py profile pilot/*.jsonl --lid

Needs: pymupdf. --lid needs fasttext (pip: fasttext-wheel, with numpy<2) and
huggingface_hub for GlotLID v3 (cis-lmu/glotlid, Apache-2.0), or falls back to
lingua-language-detector.
"""

import argparse
import csv
import hashlib
import json
import re
import statistics
import unicodedata
from collections import Counter
from pathlib import Path

import pymupdf as fitz

IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"}  # PyMuPDF cannot read WebP
# Letters that mark a Latin-script language. A hint, not an identification.
DIACRITICS = {
    "Polish": "ąćęłńśźż", "Czech": "ěřůťďň", "Slovak": "ľĺŕô", "Hungarian": "őű",
    "German": "äöüß", "French": "àâçèêëîïœù", "Spanish": "ñ¿¡", "Portuguese": "ãõ",
    "Romanian": "ășțşţ", "Turkish": "ğış", "Vietnamese": "ơưđạảấầẩẫậắằẳẵặẹẻẽếềểễệỉịọỏốồổỗộớờởỡợụủứừửữựỳỵỷỹ",
    "Croatian/Serbian/Bosnian": "đ", "Lithuanian": "ėįų", "Latvian": "āēģīķļņ",
    "Nordic": "åæø", "Icelandic": "ðþ",
}


MULTIWORD = {"OLD", "LINEAR", "CANADIAN", "NEW", "INSCRIPTIONAL", "MEROITIC", "CAUCASIAN", "PSALTER"}


def script_of(ch: str) -> str:
    """Approximate script from the Unicode character name (the standard library
    has no Script property). Good for the common scripts this is used for."""
    name = unicodedata.name(ch, "")
    if not name:
        return "OTHER"
    words = name.split()
    if words[0] == "CJK":
        return "HAN"
    return " ".join(words[:2]) if words[0] in MULTIWORD and len(words) > 1 else words[0]


def script_profile(text: str) -> dict[str, float]:
    letters = [c for c in text if c.isalpha()]
    if not letters:
        return {}
    counts = Counter(script_of(c) for c in letters)
    return {s: round(n / len(letters), 3) for s, n in counts.most_common(4)}


def diacritic_hint(text: str) -> str:
    low = text.lower()
    latin = sum(1 for c in low if c.isalpha() and script_of(c) == "LATIN")
    if latin < 200:
        return ""
    scores = {lang: sum(low.count(c) for c in chars) / latin for lang, chars in DIACRITICS.items()}
    lang, best = max(scores.items(), key=lambda kv: kv[1])
    return f"{lang} ({best:.1%})" if best >= 0.002 else "no distinctive diacritics"


def lid(text: str) -> str:
    """Top languages of a page-length text: GlotLID v3 if installed, else lingua."""
    global _LID
    text = " ".join(text.split())[:5000]
    if "_LID" not in globals():
        try:
            import fasttext
            from huggingface_hub import hf_hub_download
            _LID = ("glotlid", fasttext.load_model(hf_hub_download("cis-lmu/glotlid", "model.bin")))
        except Exception:
            try:
                from lingua import LanguageDetectorBuilder
                _LID = ("lingua", LanguageDetectorBuilder.from_all_languages().build())
            except ImportError:
                _LID = (None, None)
    kind, model = _LID
    if kind == "glotlid":
        labels, probs = model.predict(text, k=3)
        return "; ".join(f"{l.removeprefix('__label__')} {p:.2f}" for l, p in zip(labels, probs) if p >= 0.05)
    if kind == "lingua":
        out = model.compute_language_confidence_values(text)[:3]
        return "; ".join(f"{c.language.name.title()} {c.value:.2f}" for c in out if c.value >= 0.05)
    return "no LID package (pip install fasttext huggingface_hub, or lingua-language-detector)"


def sample_pages(n: int, k: int) -> list[int]:
    """k pages spread over the middle of the book (covers, front matter and
    indexes skipped; they are sparse or in another language)."""
    if n <= k:
        return list(range(1, n + 1))
    lo, hi = max(1, round(n * 0.2)), min(n, round(n * 0.8))
    return sorted({lo + round(i * (hi - lo) / max(1, k - 1)) for i in range(k)})


def sha16(path: Path) -> str:
    h = hashlib.sha256()
    files = [path] if path.is_file() else image_files(path)
    for f in files:
        with open(f, "rb") as fh:
            for block in iter(lambda: fh.read(1 << 20), b""):
                h.update(block)
    return h.hexdigest()[:16]


def image_files(folder: Path) -> list[Path]:
    def natural(p: Path):
        return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", p.name)]
    return sorted((p for p in folder.iterdir() if p.suffix.lower() in IMAGE_SUFFIXES), key=natural)


def find_documents(paths: list[str], exclude: Path) -> list[Path]:
    """PDFs (any case of .pdf) and folders of page images, skipping the output folder."""
    exclude = exclude.resolve()
    docs = []
    for p in map(Path, paths):
        if p.is_file() and p.suffix.lower() == ".pdf":
            docs.append(p)
        elif p.is_dir():
            files = [f for f in sorted(p.rglob("*")) if f.is_file()
                     and exclude not in f.resolve().parents]
            docs += [f for f in files if f.suffix.lower() == ".pdf"]
            docs += sorted({f.parent for f in files if f.suffix.lower() in IMAGE_SUFFIXES})
    return docs


def safe(name: str) -> str:
    return re.sub(r"[^\w.,()-]+", "_", name)


def unique_ids(docs: list[Path]) -> list[str]:
    """File stems, made unique with the parent folder name and then a counter."""
    ids = [safe(p.stem if p.is_file() else p.name) for p in docs]
    ids = [f"{safe(p.parent.name)}__{d}" if ids.count(d) > 1 else d for p, d in zip(docs, ids)]
    seen: dict[str, int] = {}
    for i, d in enumerate(ids):
        seen[d] = seen.get(d, 0) + 1
        if seen[d] > 1:
            ids[i] = f"{d}__{seen[d]}"
    return ids



def scan_pdf(path: Path, k: int, samples_dir: Path, doc_id: str, dpi: int):
    doc = fitz.open(path)
    n = doc.page_count
    chars, rotated, landscape, sizes, texts = [], 0, 0, [], []
    hidden = visible = 0
    for page in doc:
        t = page.get_text()
        if t.strip() and page.number % 10 == 0:  # every tenth page is enough
            for span in page.get_texttrace():
                n_chars = len(span["chars"])
                if span["type"] == 3 or span["opacity"] == 0:
                    hidden += n_chars
                else:
                    visible += n_chars
        chars.append(len(t.strip()))
        texts.append(t)
        rotated += page.rotation != 0
        landscape += page.rect.width > 1.15 * page.rect.height  # two-page spreads?
        sizes.append((page.rect.width / 72 * 25.4, page.rect.height / 72 * 25.4))
    picks = sample_pages(n, k)
    scan_dpis, full_image, pngs = [], 0, []
    for p in picks:
        page = doc[p - 1]
        area = abs(page.rect)
        best = None
        for img in page.get_images(full=True):
            for r in page.get_image_rects(img[0]):
                if best is None or abs(r) > abs(best[1]):
                    best = (img[2], r)
        if best and abs(best[1]) >= 0.8 * area:
            full_image += 1
            scan_dpis.append(round(best[0] / (best[1].width / 72)))
        png = samples_dir / f"{doc_id}_p{p}.png"
        page.get_pixmap(dpi=dpi).save(png)
        pngs.append(png)
    med = statistics.median(chars) if chars else 0
    if med < 50:
        kind = "scan"
    elif hidden > visible or full_image >= len(picks) / 2:
        kind = "scan with OCR layer" if hidden > visible else "scan with text layer"
    else:
        kind = "born-digital" if med >= 300 else "mixed or sparse text layer"
    text = "\n".join(texts)
    return {
        "n_pages": n, "kind": kind, "text_chars_median": int(med),
        "text_pages_share": round(sum(c >= 200 for c in chars) / max(1, n), 2),
        "rotated_pages": rotated, "landscape_pages": landscape,
        "page_mm": "{:.0f}x{:.0f}".format(*map(statistics.median, zip(*sizes))) if sizes else "",
        "scan_dpi": int(statistics.median(scan_dpis)) if scan_dpis else "",
        "sample_pages": ";".join(map(str, picks)),
        "text_script": json.dumps(script_profile(text)) if med >= 50 else "",
        "text_lang_hint": diacritic_hint(text) if med >= 50 else "",
        "_text": text if med >= 50 else "",
    }, pngs


def scan_images(folder: Path, k: int, samples_dir: Path, doc_id: str, dpi: int):
    files = image_files(folder)
    picks = sample_pages(len(files), k)
    pngs = []
    multi = 0
    for f in files:
        if f.suffix.lower() in {".tif", ".tiff"}:
            with fitz.open(f) as d:
                multi += d.page_count > 1
    for p in picks:
        pix = fitz.Pixmap(str(files[p - 1]))
        if pix.n - pix.alpha >= 4:  # CMYK
            pix = fitz.Pixmap(fitz.csRGB, pix)
        scale = dpi / (pix.xres or 300)
        if scale < 1:
            pix = fitz.Pixmap(pix, int(pix.width * scale), int(pix.height * scale), None)
        png = samples_dir / f"{doc_id}_p{p}.png"
        pix.save(png)
        pngs.append(png)
    first = fitz.Pixmap(str(files[0])) if files else None
    return {"n_pages": len(files), "kind": "scan (images)" + (f", {multi} multi-page TIFFs: split them" if multi else ""), "text_chars_median": 0,
            "text_pages_share": 0, "rotated_pages": 0, "landscape_pages": "", "page_mm": "",
            "scan_dpi": first.xres if first and first.xres else "",
            "sample_pages": ";".join(map(str, picks)), "text_script": "", "text_lang_hint": "",
            "_text": ""}, pngs


def contact_sheets(entries: list[tuple[str, Path]], out: Path, per_sheet: int = 12):
    """Grid of one sample page per document, labelled with its id."""
    cols, cell_w, cell_h, label = 4, 300, 420, 14
    for s in range(0, len(entries), per_sheet):
        chunk = entries[s:s + per_sheet]
        rows = (len(chunk) + cols - 1) // cols
        sheet = fitz.open()
        page = sheet.new_page(width=cols * cell_w, height=rows * (cell_h + label))
        for i, (doc_id, png) in enumerate(chunk):
            x, y = (i % cols) * cell_w, (i // cols) * (cell_h + label)
            page.insert_text((x + 4, y + 11), doc_id[:48], fontsize=9)
            page.insert_image(fitz.Rect(x + 2, y + label, x + cell_w - 2, y + label + cell_h),
                              filename=str(png), keep_proportion=True)
        page.get_pixmap(dpi=110).save(out / f"sheet_{s // per_sheet + 1:03d}.png")


def cmd_scan(args):
    out = Path(args.out)
    (out / "samples").mkdir(parents=True, exist_ok=True)
    (out / "sheets").mkdir(exist_ok=True)
    docs = find_documents(args.paths, out)
    rows, firsts, seen = [], [], {}
    for path, doc_id in zip(docs, unique_ids(docs)):
        info, pngs = (scan_pdf if path.is_file() else scan_images)(
            path, args.samples, out / "samples", doc_id, args.dpi)
        digest = sha16(path)
        row = {"doc_id": doc_id, "path": str(path), "mb": round(
            (path.stat().st_size if path.is_file() else sum(f.stat().st_size for f in image_files(path))) / 1e6, 1),
            "sha256_16": digest, "duplicate_of": seen.get(digest, "")}
        seen.setdefault(digest, doc_id)
        text = info.pop("_text")
        row.update(info)
        if args.lid and text:
            row["text_lid"] = lid(text)
        rows.append(row)
        if pngs:
            firsts.append((doc_id, pngs[len(pngs) // 2]))
        print(f"{doc_id}: {row['n_pages']} pages, {row['kind']}"
              f"{', dup of ' + row['duplicate_of'] if row['duplicate_of'] else ''}"
              f"{', landscape ' + str(row['landscape_pages']) if row['landscape_pages'] else ''}"
              f"{', LID ' + row['text_lid'] if row.get('text_lid') else ''}", flush=True)
    fields = list(dict.fromkeys(k for r in rows for k in r))
    with open(out / "triage.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        w.writerows(rows)
    contact_sheets(firsts, out / "sheets")
    kinds = Counter(r["kind"] for r in rows)
    print(f"\n{len(rows)} documents, {sum(r['n_pages'] for r in rows):,} pages: "
          + ", ".join(f"{k} {v}" for k, v in kinds.items())
          + f"; {sum(1 for r in rows if r['duplicate_of'])} exact duplicates")
    print(f"Wrote {out / 'triage.csv'}, samples/ and sheets/ (look at the sheets).")


def cmd_profile(args):
    for f in map(Path, args.files):
        if f.suffix == ".jsonl":
            recs = [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines() if l.strip()]
            pages = {r.get("page", i): r.get("text", "") for i, r in enumerate(recs)}  # last wins
            text = "\n".join(pages.values())
            prof = {p: script_profile(t) for p, t in pages.items() if len(t) >= 200}
            main = Counter(next(iter(v), "") for v in prof.values()).most_common(1)
            odd = [p for p, v in prof.items() if main and next(iter(v), "") != main[0][0]]
        else:
            text, odd = f.read_text(encoding="utf-8"), []
        hint = diacritic_hint(text)
        extra = f" | lid {lid(text)}" if args.lid else ""
        print(f"{f.name}: {script_profile(text)} | {hint or '-'}{extra}"
              + (f" | pages with another main script: {sorted(odd)[:15]}" if odd else ""))


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[1])
    sub = ap.add_subparsers(dest="cmd", required=True)
    s = sub.add_parser("scan")
    s.add_argument("paths", nargs="+")
    s.add_argument("--out", required=True)
    s.add_argument("--samples", type=int, default=3, help="sample pages per document")
    s.add_argument("--dpi", type=int, default=100, help="dpi of the sample renders")
    s.add_argument("--lid", action="store_true", help="language of the text layer (GlotLID, else lingua)")
    s.set_defaults(func=cmd_scan)
    p = sub.add_parser("profile")
    p.add_argument("files", nargs="+")
    p.add_argument("--lid", action="store_true")
    p.set_defaults(func=cmd_profile)
    args = ap.parse_args()
    args.func(args)


if __name__ == "__main__":
    main()
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall