Skip to content

guillaumemeyer/watermarks-remover / skills/clean-user-facing-text

clean-user-facing-text

Flagged

Clean and finalize authorized natural-language text intended for readers by auditing suspicious invisible Unicode and rewriting prose while preserving facts, meaning, and the writer's voice. Use when the user asks to clean, humanize, polish, or finalize articles, manuscripts, reports, documentation, emails, product copy, UI text, Markdown, or HTML prose, or when a project rule or instruction file explicitly requires this workflow. Don't use for code-only tasks or undisclosed authorship evasion; leave code, commands, identifiers, paths, APIs, formulas, citations, required disclosures, and verbatim quotations unchanged.

FileKindSize
SKILL.mdskill md13.6 KB
references/detectors.mdmarkdown8.1 KB
references/responsible-use.mdmarkdown536 B
references/watermark-notes.mdmarkdown1.2 KB
references/writing-in-your-voice.mdmarkdown2.5 KB
scripts/clean_text.pyexecutablescript3.0 KB
scripts/common.pyscript20.9 KB
scripts/inspect_text.pyexecutablescript4.4 KB
scripts/score_stylometry.pyscript19.5 KB
scripts/text_unicode.pyexecutablescript24.5 KB

scripts/text_unicode.py

View on GitHub
"""Layer A: invisible Unicode / homoglyph space detection and cleaning."""

from __future__ import annotations

import unicodedata
from collections import Counter
from dataclasses import dataclass, field
from difflib import SequenceMatcher

# Format / invisible controls commonly used for steganography or broken pastes.
STRIP_CODEPOINTS: frozenset[int] = frozenset(
    {
        0x00AD,  # soft hyphen
        0x034F,  # combining grapheme joiner
        0x061C,  # Arabic letter mark
        0x115F,  # Hangul choseong filler
        0x1160,  # Hangul jungseong filler
        0x17B4,  # Khmer vowel inherent AQ
        0x17B5,  # Khmer vowel inherent AA
        0x180B,  # Mongolian free variation selector-1
        0x180C,
        0x180D,
        0x180E,  # Mongolian vowel separator
        0x180F,  # Mongolian free variation selector-4 (Unicode 14)
        0x200B,  # zero width space
        0x200C,  # zero width non-joiner
        0x200D,  # zero width joiner
        0x200E,  # LRM
        0x200F,  # RLM
        0x202A,  # LRE
        0x202B,  # RLE
        0x202C,  # PDF
        0x202D,  # LRO
        0x202E,  # RLO
        0x2060,  # word joiner
        0x2061,  # function application
        0x2062,  # invisible times
        0x2063,  # invisible separator
        0x2064,  # invisible plus
        0x2066,  # LRI
        0x2067,  # RLI
        0x2068,  # FSI
        0x2069,  # PDI
        0x206A,  # inhibit symmetric swapping
        0x206B,
        0x206C,
        0x206D,
        0x206E,
        0x206F,
        0xFEFF,  # BOM / ZWNBSP
        0xFE00,  # variation selectors
        0xFE01,
        0xFE02,
        0xFE03,
        0xFE04,
        0xFE05,
        0xFE06,
        0xFE07,
        0xFE08,
        0xFE09,
        0xFE0A,
        0xFE0B,
        0xFE0C,
        0xFE0D,
        0xFE0E,
        0xFE0F,
        0x3164,  # Hangul filler (blank compatibility jamo)
        0xFFA0,  # halfwidth Hangul filler
        0xFFF9,  # interlinear annotation
        0xFFFA,
        0xFFFB,
    }
)

# Spaces that look like (or substitute for) U+0020.
SPACE_HOMOGLYPHS: dict[int, str] = {
    0x00A0: " ",  # no-break space
    0x1680: " ",  # Ogham space mark
    0x2000: " ",  # en quad
    0x2001: " ",  # em quad
    0x2002: " ",  # en space
    0x2003: " ",  # em space
    0x2004: " ",  # three-per-em space
    0x2005: " ",  # four-per-em space
    0x2006: " ",  # six-per-em space
    0x2007: " ",  # figure space
    0x2008: " ",  # punctuation space
    0x2009: " ",  # thin space
    0x200A: " ",  # hair space
    0x202F: " ",  # narrow no-break space
    0x205F: " ",  # medium mathematical space
    0x3000: " ",  # ideographic space
}

# Optional confusable Latin lookalikes (aggressive mode only).
LATIN_CONFUSABLES: dict[int, str] = {
    0x0410: "A",  # Cyrillic
    0x0412: "B",
    0x0415: "E",
    0x041A: "K",
    0x041C: "M",
    0x041D: "H",
    0x041E: "O",
    0x0420: "P",
    0x0421: "C",
    0x0422: "T",
    0x0425: "X",
    0x0430: "a",
    0x0435: "e",
    0x043E: "o",
    0x0440: "p",
    0x0441: "c",
    0x0443: "y",
    0x0445: "x",
    0x0456: "i",
    0xFF21: "A",  # fullwidth
    0xFF22: "B",
    0xFF23: "C",
    0xFF24: "D",
    0xFF25: "E",
    0xFF26: "F",
    0xFF27: "G",
    0xFF28: "H",
    0xFF29: "I",
    0xFF2A: "J",
    0xFF2B: "K",
    0xFF2C: "L",
    0xFF2D: "M",
    0xFF2E: "N",
    0xFF2F: "O",
    0xFF30: "P",
    0xFF31: "Q",
    0xFF32: "R",
    0xFF33: "S",
    0xFF34: "T",
    0xFF35: "U",
    0xFF36: "V",
    0xFF37: "W",
    0xFF38: "X",
    0xFF39: "Y",
    0xFF3A: "Z",
    0xFF41: "a",
    0xFF42: "b",
    0xFF43: "c",
    0xFF44: "d",
    0xFF45: "e",
    0xFF46: "f",
    0xFF47: "g",
    0xFF48: "h",
    0xFF49: "i",
    0xFF4A: "j",
    0xFF4B: "k",
    0xFF4C: "l",
    0xFF4D: "m",
    0xFF4E: "n",
    0xFF4F: "o",
    0xFF50: "p",
    0xFF51: "q",
    0xFF52: "r",
    0xFF53: "s",
    0xFF54: "t",
    0xFF55: "u",
    0xFF56: "v",
    0xFF57: "w",
    0xFF58: "x",
    0xFF59: "y",
    0xFF5A: "z",
}

# Variation selectors beyond FE0x (VS17-VS256 in Supplementary Special-purpose)
_VS_SUPPLEMENT = range(0xE0100, 0xE01F0)

# Unassigned code points with Other_Default_Ignorable_Code_Point=Yes: reserved
# for future default-ignorable characters, so conformant renderers display them
# invisibly today and normalisation preserves them. They have no legitimate use
# in interchange text (conformance clause C7), which makes them ideal covert
# carriers. Kept as explicit ranges, never a category-Cn rule: unicodedata is
# pinned per Python build, so a Cn rule would destroy freshly assigned real
# characters. Re-check these ranges on Unicode version bumps: assignment turns
# a strip entry into a potential preserve-in-context case, exactly as happened
# when U+180F became Mongolian FVS4 in Unicode 14.
_RESERVED_IGNORABLE_CPS: frozenset[int] = frozenset({0x2065, 0xE0000})
_RESERVED_IGNORABLE_RANGES: tuple[range, ...] = (
    range(0xFFF0, 0xFFF9),
    range(0xE0080, 0xE0100),
    range(0xE01F0, 0xE1000),
)


def _is_reserved_ignorable(cp: int) -> bool:
    if cp in _RESERVED_IGNORABLE_CPS:
        return True
    return any(cp in r for r in _RESERVED_IGNORABLE_RANGES)


# The 66 Unicode noncharacters: U+FDD0..U+FDEF plus U+nFFFE/U+nFFFF at the
# end of every plane. Permanently reserved for internal use and prohibited in
# interchange text (TUS 23.7), so any occurrence in interchange is contraband.
# Rendered as nothing or tofu, preserved by normalisation, and permanently
# unassignable, so stripping them carries no future-Unicode risk.
def _is_noncharacter(cp: int) -> bool:
    return 0xFDD0 <= cp <= 0xFDEF or (cp & 0xFFFE) == 0xFFFE


# Bidi / directional format controls (subset of strip set, finer inspect labels)
_BIDI_CPS: frozenset[int] = frozenset(
    {
        0x061C,
        0x200E,
        0x200F,
        0x202A,
        0x202B,
        0x202C,
        0x202D,
        0x202E,
        0x2066,
        0x2067,
        0x2068,
        0x2069,
    }
)

# Directional marks and isolates are legitimate in mixed RTL/LTR prose. Inspect
# them, but preserve them during the default clean. Paired LRE/RLE embeddings
# (see _valid_bidi_embedding_indices) are preserved too; overrides and
# unpaired embeddings remain destructive by default because they can reorder
# unrelated spans.
_PRESERVABLE_BIDI_CPS: frozenset[int] = frozenset(
    {
        0x061C,
        0x200E,
        0x200F,
        0x2066,
        0x2067,
        0x2068,
        0x2069,
    }
)

# Visible-layout format controls: Egyptian hieroglyph quadrat controls,
# Duployan shorthand overlap/step controls, and musical beam/tie/slur/phrase
# controls are Cf but visibly govern how their script renders. Next to their
# own script they are document body, not carriers; floating between unrelated
# text they stay strip-class. Context ranges include each script's own block
# (and the controls themselves) so control sequences survive intact.
_LAYOUT_CF_CONTROLS: tuple[tuple[range, range], ...] = (
    (range(0x13430, 0x13440), range(0x13000, 0x14400)),  # Egyptian hieroglyphs
    (range(0x1BCA0, 0x1BCA4), range(0x1BC00, 0x1BCA4)),  # Duployan shorthand
    (range(0x1D173, 0x1D17B), range(0x1D100, 0x1D200)),  # musical symbols
)


def _layout_cf_script(cp: int) -> range | None:
    for controls, script in _LAYOUT_CF_CONTROLS:
        if cp in controls:
            return script
    return None


# Zero-width family (common edit-based carriers)
_ZW_FAMILY: frozenset[int] = frozenset({0x200B, 0x200C, 0x200D, 0x2060, 0xFEFF, 0x180E})


def _is_private_use(cp: int) -> bool:
    """BMP and supplementary private-use planes (Co: no portable meaning)."""
    return 0xE000 <= cp <= 0xF8FF or 0xF0000 <= cp <= 0xFFFFD or 0x100000 <= cp <= 0x10FFFD


def _is_strip_cp(cp: int) -> bool:
    if cp in STRIP_CODEPOINTS:
        return True
    if cp in _VS_SUPPLEMENT:
        return True
    # Tag characters used in some stego schemes (U+E0001-U+E007F)
    if 0xE0001 <= cp <= 0xE007F:
        return True
    if _is_noncharacter(cp):
        return True
    if _is_reserved_ignorable(cp):
        return True
    return bool(_is_private_use(cp))


def _strip_kind(cp: int) -> str:
    """Finer-grained inspect kind for strip-class codepoints."""
    if 0xE0001 <= cp <= 0xE007F:
        return "tag_chars"
    if _is_noncharacter(cp):
        return "noncharacter"
    if _is_reserved_ignorable(cp):
        return "reserved_ignorable"
    if cp in _VS_SUPPLEMENT or 0xFE00 <= cp <= 0xFE0F or cp in _MONGOLIAN_FVS:
        return "variation_selector"
    if cp in _BIDI_CPS:
        return "bidi"
    if cp in _ZW_FAMILY:
        return "zwj_family"
    if _is_private_use(cp):
        return "private_use"
    return "strip"


# Emoji presentation glue: zero-width joiner and text/emoji variation
# selectors. These are invisible carriers when free-floating, but after an
# emoji base they are part of the visible sequence (⚖️, 👨‍👩‍👧, ❤️‍🔥) and
# stripping them visibly alters the text.
EMOJI_GLUE_CODEPOINTS: frozenset[int] = frozenset({0x200D, 0xFE0E, 0xFE0F})


def _is_emoji_glue(cp: int) -> bool:
    return cp in EMOJI_GLUE_CODEPOINTS


def _is_emoji_base(cp: int) -> bool:
    """Return True for characters that can start or continue an emoji sequence."""
    if 0x1F000 <= cp <= 0x1FAFF:
        return True
    if 0x2190 <= cp <= 0x25FF:  # arrows, technical symbols, enclosed symbols
        return True
    if 0x2600 <= cp <= 0x27BF:  # misc symbols / dingbats / arrows
        return True
    if 0x2B00 <= cp <= 0x2BFF:  # misc symbols and arrows
        return True
    # Emoji=Yes singletons outside the ranges above (general punctuation,
    # letterlike symbols, supplemental arrows-B): !!, !?, i, curved up/down
    # arrows. Without these a VS16 after them is stripped, visibly turning
    # the emoji presentation back into the text glyph.
    if cp in (0x203C, 0x2049, 0x2139, 0x2934, 0x2935):
        return True
    if cp in (0x00A9, 0x00AE, 0x2122, 0x3030, 0x303D, 0x3297, 0x3299):
        return True
    # keycap bases
    return cp in (0x0023, 0x002A) or 0x0030 <= cp <= 0x0039


# ZWNJ/ZWJ are orthographic inside complex scripts (Persian می‌روم, Devanagari
# क्‍ष); flag emoji are an emoji base followed by tag chars (🏴󠁧󠁢󠁳󠁣󠁴󠁿); and a
# handful of Cf codepoints are normal Arabic/Syriac orthography, not carriers.
# So are Mongolian free variation selectors (choose a glyph of the preceding
# letter), Khmer inherent vowels (invisible but phonemic), and Hangul fillers
# (hold a jamo slot in a partial syllable). Each is only meaningful directly
# after a base from its own script; isolated instances are contraband.
_SCRIPT_JOINERS: frozenset[int] = frozenset({0x200C, 0x200D})
_TAG_RANGE = range(0xE0020, 0xE0080)
_ORTHOGRAPHIC_CF: frozenset[int] = frozenset(
    {0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x06DD, 0x070F, 0x08E2, 0x110BD, 0x110CD}
)
_MONGOLIAN_FVS: frozenset[int] = frozenset({0x180B, 0x180C, 0x180D, 0x180F})
_KHMER_VOWELS: frozenset[int] = frozenset({0x17B4, 0x17B5})
_HANGUL_FILLERS: frozenset[int] = frozenset({0x115F, 0x1160, 0x3164, 0xFFA0})
_SCRIPT_GLUE: frozenset[int] = _MONGOLIAN_FVS | _KHMER_VOWELS | _HANGUL_FILLERS


def _joining_script(cp: int) -> str | None:
    """Return a broad script group where ZWJ/ZWNJ can be orthographic."""
    for start, end, name in (
        (0x0600, 0x08FF, "arabic"),
        (0x0900, 0x0DFF, "indic"),
        (0x0F00, 0x109F, "south-asian"),
        (0x1780, 0x17FF, "khmer"),
        (0x1800, 0x18AF, "mongolian"),
    ):
        if start <= cp <= end and unicodedata.category(chr(cp))[0] in ("L", "M"):
            return name
    return None


def _is_cjk_ideograph(cp: int) -> bool:
    return (
        0x3400 <= cp <= 0x4DBF
        or 0x4E00 <= cp <= 0x9FFF
        or 0xF900 <= cp <= 0xFAFF
        or 0x20000 <= cp <= 0x323AF
    )


def _is_mongolian_base(cp: int) -> bool:
    return 0x1800 <= cp <= 0x18AF


def _is_variation_selector(cp: int) -> bool:
    return cp in _VS_SUPPLEMENT or 0xFE00 <= cp <= 0xFE0F or cp in _MONGOLIAN_FVS


def _valid_flag_tag_indices(text: str) -> set[int]:
    """Indices in complete subdivision-flag tag sequences."""
    valid: set[int] = set()
    i = 0
    while i < len(text):
        if ord(text[i]) != 0x1F3F4:  # waving black flag
            i += 1
            continue
        j = i + 1
        while j < len(text) and 0xE0020 <= ord(text[j]) <= 0xE007E:
            j += 1
        if j > i + 1 and j < len(text) and ord(text[j]) == 0xE007F:
            valid.update(range(i + 1, j + 1))
            i = j + 1
        else:
            i += 1
    return valid


def _valid_bidi_embedding_indices(text: str) -> set[int]:
    """Indices belonging to complete LRE/RLE ... PDF pairs, excluding overrides."""
    valid: set[int] = set()
    stack: list[tuple[int, int]] = []
    for index, char in enumerate(text):
        cp = ord(char)
        if cp in (0x202A, 0x202B, 0x202D, 0x202E):
            stack.append((cp, index))
        elif cp == 0x202C:
            if not stack:
                continue
            opener, opener_index = stack.pop()
            if opener in (0x202A, 0x202B):
                valid.update((opener_index, index))
    return valid


def _is_mongolian_letter(cp: int) -> bool:
    return 0x1800 <= cp <= 0x18AF and unicodedata.category(chr(cp))[0] == "L"


def _is_khmer_letter(cp: int) -> bool:
    return 0x1780 <= cp <= 0x17FF and unicodedata.category(chr(cp))[0] == "L"


def _is_hangul_jamo(cp: int) -> bool:
    # Conjoining jamo plus the compatibility and halfwidth presentation forms,
    # so each filler can follow letters of its own form (U+115F/U+1160 after
    # conjoining jamo, U+3164 after compatibility jamo, U+FFA0 after halfwidth).
    return (
        0x1100 <= cp <= 0x11FF
        or 0xA960 <= cp <= 0xA97C  # Hangul Jamo Extended-A
        or 0xD7B0 <= cp <= 0xD7C6  # Hangul Jamo Extended-B
        or 0x3131 <= cp <= 0x318E  # Hangul Compatibility Jamo (incl. U+3164)
        or 0xFFA1 <= cp <= 0xFFDC  # halfwidth Hangul jamo letters
    )


def _is_glue(cp: int) -> bool:
    """Load-bearing invisible char: emoji glue, script joiner, flag tag char,
    or same-script filler/selector (Mongolian FVS, Khmer vowel, Hangul filler)."""
    return (
        _is_emoji_glue(cp)
        or _is_variation_selector(cp)
        or cp in _SCRIPT_JOINERS
        or cp in _TAG_RANGE
        or cp in _SCRIPT_GLUE
    )


def _decide(
    ch: str,
    prev_kept: str | None,
    prev_input: str | None,
    next_input: str | None,
    *,
    valid_flag_tag: bool,
    valid_bidi_embedding: bool,
    normalize_spaces: bool,
    treat_confusables: bool,
    strip_emoji_glue: bool,
    strip_bidi: bool,
) -> tuple[str, str, str | None]:
    """Classify one input char for both inspect and clean.

    Returns ``(action, out_char, kind)`` where action is ``keep``, ``strip``
    or ``replace``; out_char is the surviving character for keep/replace; and
    kind is the inspect classification (None when not suspicious).
    """
    cp = ord(ch)
    if valid_bidi_embedding and not strip_bidi:
        return ("keep", ch, None)
    if cp in _PRESERVABLE_BIDI_CPS and not strip_bidi:
        return ("keep", ch, None)
    if prev_input is not None and not strip_emoji_glue:
        prev_cp = ord(prev_input)
        if cp in _VS_SUPPLEMENT and _is_cjk_ideograph(prev_cp):
            return ("keep", ch, None)
        if cp in _MONGOLIAN_FVS and _is_mongolian_base(prev_cp):
            return ("keep", ch, None)
        if 0xFE00 <= cp <= 0xFE0D and _is_cjk_ideograph(prev_cp):
            return ("keep", ch, None)
    if _is_emoji_glue(cp) and not strip_emoji_glue:
        if cp in (0xFE0E, 0xFE0F) and prev_input is not None and _is_emoji_base(ord(prev_input)):
            return ("keep", ch, None)
        if (
            cp == 0x200D
            and prev_kept is not None
            and next_input is not None
            and _is_emoji_base(ord(prev_kept))
            and _is_emoji_base(ord(next_input))
        ):
            return ("keep", ch, None)
    if not strip_emoji_glue:
        if cp in _SCRIPT_JOINERS and prev_input is not None and next_input is not None:
            prev_script = _joining_script(ord(prev_input))
            next_script = _joining_script(ord(next_input))
            if prev_script is not None and prev_script == next_script:
                return ("keep", ch, None)
        if cp in _TAG_RANGE and valid_flag_tag:
            return ("keep", ch, None)
        if cp in _MONGOLIAN_FVS and prev_kept is not None and _is_mongolian_letter(ord(prev_kept)):
            return ("keep", ch, None)
        if cp in _KHMER_VOWELS and prev_kept is not None and _is_khmer_letter(ord(prev_kept)):
            return ("keep", ch, None)
        if cp in _HANGUL_FILLERS and prev_kept is not None and _is_hangul_jamo(ord(prev_kept)):
            return ("keep", ch, None)
        if cp in _ORTHOGRAPHIC_CF:
            return ("keep", ch, None)
        script = _layout_cf_script(cp)
        if script is not None and (
            (prev_input is not None and ord(prev_input) in script)
            or (next_input is not None and ord(next_input) in script)
        ):
            return ("keep", ch, None)
    if _is_strip_cp(cp):
        return ("strip", "", _strip_kind(cp))
    if normalize_spaces and cp in SPACE_HOMOGLYPHS:
        return ("replace", SPACE_HOMOGLYPHS[cp], "space")
    if treat_confusables and cp in LATIN_CONFUSABLES:
        return ("replace", LATIN_CONFUSABLES[cp], "confusable")
    if unicodedata.category(ch) == "Cf" and cp not in SPACE_HOMOGLYPHS:
        return ("strip", "", "other_cf")
    return ("keep", ch, None)


def _char_label(ch: str) -> str:
    cp = ord(ch)
    name = unicodedata.name(ch, "UNKNOWN")
    cat = unicodedata.category(ch)
    return f"U+{cp:04X} {name} ({cat})"


def _hit_confidence(kind: str) -> str:
    """Layer A hits are edit-based carriers; space homoglyphs are weaker context."""
    return "informational" if kind == "space" else "probable"


@dataclass
class CharHit:
    codepoint: int
    char: str
    label: str
    count: int
    kind: str  # strip | bidi | tag_chars | variation_selector | zwj_family | private_use | noncharacter | reserved_ignorable | space | confusable | other_cf
    samples: list[int] = field(default_factory=list)  # character offsets


@dataclass
class TextInspectReport:
    length: int
    suspicious_total: int
    hits: list[CharHit]
    notes: list[str] = field(default_factory=list)

    def to_dict(self) -> dict:
        return {
            "length": self.length,
            "suspicious_total": self.suspicious_total,
            "hits": [
                {
                    "codepoint": f"U+{h.codepoint:04X}",
                    "label": h.label,
                    "count": h.count,
                    "kind": h.kind,
                    "confidence": _hit_confidence(h.kind),
                    "sample_offsets": h.samples[:10],
                }
                for h in self.hits
            ],
            "notes": self.notes,
        }


def inspect_text(
    text: str,
    *,
    aggressive: bool = False,
    strip_emoji_glue: bool = False,
) -> TextInspectReport:
    buckets: dict[tuple[int, str], list[int]] = {}
    prev_kept: str | None = None
    valid_flag_tags = _valid_flag_tag_indices(text)
    valid_bidi_embeddings = _valid_bidi_embedding_indices(text)
    for i, ch in enumerate(text):
        action, out_char, kind = _decide(
            ch,
            prev_kept,
            text[i - 1] if i > 0 else None,
            text[i + 1] if i + 1 < len(text) else None,
            valid_flag_tag=i in valid_flag_tags,
            valid_bidi_embedding=i in valid_bidi_embeddings,
            normalize_spaces=True,
            treat_confusables=aggressive,
            strip_emoji_glue=strip_emoji_glue,
            strip_bidi=True,
        )
        if kind is None:
            # Kept; glue (emoji/script joiner/tag) does not advance the
            # "previous kept" base so ZWJ chains and flag runs stay bound.
            if not _is_glue(ord(ch)):
                prev_kept = out_char
            continue
        key = (ord(ch), kind)
        buckets.setdefault(key, []).append(i)
        if action == "replace":
            prev_kept = out_char
        # strip: prev_kept unchanged

    hits: list[CharHit] = []
    total = 0
    for (cp, kind), offsets in sorted(buckets.items(), key=lambda x: (-len(x[1]), x[0][0])):
        ch = chr(cp)
        hits.append(
            CharHit(
                codepoint=cp,
                char=ch,
                label=_char_label(ch),
                count=len(offsets),
                kind=kind,
                samples=offsets[:10],
            )
        )
        total += len(offsets)

    notes = [
        "Layer A only: invisible/format Unicode and space homoglyphs (edit-based carriers).",
        "Statistical (token-sampling) watermarks are not detectable here; use Layer B rewrite.",
        "Inspect kinds: strip, bidi, tag_chars, variation_selector, zwj_family, private_use, space, confusable, other_cf.",
        "Load-bearing invisibles are preserved by default during cleaning: emoji glue, CJK/Mongolian variation selectors, script joiners, complete flag tag sequences, same-script fillers/selectors (Mongolian FVS, Khmer inherent vowels, Hangul jamo fillers), RTL directional marks/paired embeddings, orthographic Arabic/Syriac Cf marks, and visible-layout format controls next to their own script (Egyptian hieroglyph quadrat, Duployan shorthand, musical beaming). Inspection still reports bidi controls. Use explicit strip flags only after review.",
    ]
    if not hits:
        notes.append(
            "No deterministic Layer A (invisible Unicode/format) carriers detected; "
            "statistical and pixel-domain marks are out of scope here."
        )
    return TextInspectReport(length=len(text), suspicious_total=total, hits=hits, notes=notes)


def clean_text(
    text: str,
    *,
    nfkc: bool = False,
    aggressive_homoglyphs: bool = False,
    normalize_spaces: bool = True,
    strip_emoji_glue: bool = False,
    strip_bidi: bool = False,
) -> tuple[str, dict]:
    """Return cleaned text and a stats dict."""
    removed: Counter[str] = Counter()
    replaced: Counter[str] = Counter()
    out_chars: list[str] = []
    prev_kept: str | None = None
    valid_flag_tags = _valid_flag_tag_indices(text)
    valid_bidi_embeddings = _valid_bidi_embedding_indices(text)

    for i, ch in enumerate(text):
        action, out_char, _kind = _decide(
            ch,
            prev_kept,
            text[i - 1] if i > 0 else None,
            text[i + 1] if i + 1 < len(text) else None,
            valid_flag_tag=i in valid_flag_tags,
            valid_bidi_embedding=i in valid_bidi_embeddings,
            normalize_spaces=normalize_spaces,
            treat_confusables=aggressive_homoglyphs,
            strip_emoji_glue=strip_emoji_glue,
            strip_bidi=strip_bidi,
        )
        if action == "keep":
            out_chars.append(out_char)
            # Glue (emoji/script joiner/tag) does not advance the "previous
            # kept" base, so ZWJ chains (❤️‍🔥) and flag runs stay bound.
            if not _is_glue(ord(ch)):
                prev_kept = out_char
        elif action == "replace":
            out_chars.append(out_char)
            replaced[_char_label(ch)] += 1
            prev_kept = out_char
        else:  # strip
            removed[_char_label(ch)] += 1
            # prev_kept unchanged

    result = "".join(out_chars)
    nfkc_changed = False
    if nfkc:
        before = result
        result = unicodedata.normalize("NFKC", result)
        if result != before:
            nfkc_changed = True
            changed_inputs = sum(
                end - start
                for operation, start, end, _new_start, _new_end in SequenceMatcher(
                    None, before, result, autojunk=False
                ).get_opcodes()
                if operation != "equal"
            )
            replaced["NFKC_normalize"] += changed_inputs or 1

    # Collapse runs of spaces only if we introduced space replacements? Keep conservative: no.

    stats = {
        "input_length": len(text),
        "output_length": len(result),
        "removed": dict(removed),
        "replaced": dict(replaced),
        "removed_count": sum(removed.values()),
        "replaced_count": sum(replaced.values()),
        "nfkc_changed": nfkc_changed,
    }
    return result, stats


def human_report(report: TextInspectReport) -> str:
    lines = [
        f"Length: {report.length} chars",
        f"Suspicious: {report.suspicious_total}",
    ]
    if report.hits:
        lines.append("Hits:")
        for h in report.hits:
            lines.append(
                f"  [{h.kind}/{_hit_confidence(h.kind)}] {h.label} x{h.count} @ {h.samples[:5]}"
            )
    for n in report.notes:
        lines.append(f"Note: {n}")
    return "\n".join(lines)
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall