Skip to content

oliverweissl/context-garden / skills/weeder

weeder

Verified

Measure and reduce a Skill's token footprint (always-loaded description + on-trigger SKILL.md body) without regressing routing accuracy or losing instructions. Use when a SKILL.md feels oversized, repeats rules, or has a vague description that might mis-route.

scripts/we_dupes.py

View on GitHub
"""Near-duplicate detection (normalized-word Jaccard, plus shared word runs at
sentence level): a rule repeated in five places collapses to one finding."""

from __future__ import annotations

import re
from itertools import combinations

from we_text import _CODE_FENCE_RE, split_into_original_sentences, strip_code
from we_tokens import estimate_tokens

MIN_WORDS = 8
MIN_SENTENCE_WORDS = 6
DEFAULT_THRESHOLD = 0.6
DEFAULT_SENTENCE_THRESHOLD = 0.45
MIN_SHARED_PHRASE_WORDS = 6
_WORD_RE = re.compile(r"[a-z0-9]+")


def _normalize_words(text: str) -> set[str]:
    return set(_WORD_RE.findall(text.lower()))


def _word_seq(text: str) -> list[str]:
    return _WORD_RE.findall(text.lower())


def longest_common_run(a: list[str], b: list[str]) -> int:
    """Longest common contiguous word run. Jaccard is diluted by differing
    elaboration; a long shared run catches an embedded restated rule."""
    if not a or not b:
        return 0
    prev = [0] * (len(b) + 1)
    best = 0
    for i in range(1, len(a) + 1):
        curr = [0] * (len(b) + 1)
        for j in range(1, len(b) + 1):
            if a[i - 1] == b[j - 1]:
                curr[j] = prev[j - 1] + 1
                best = max(best, curr[j])
        prev = curr
    return best


def split_paragraphs(text: str, source: str) -> list[dict]:
    """Fenced code is dropped: usage examples legitimately repeat the same
    command prefix and would be false positives. Inline code stays in `text`
    (excluded only from similarity words) so `text` is a verbatim span
    apply-suggestion can find in the original file."""
    cleaned = _CODE_FENCE_RE.sub("\n\n", text)
    paras = [p.strip() for p in re.split(r"\n\s*\n", cleaned) if p.strip()]
    out = []
    for p in paras:
        prose = strip_code(p)
        if len(prose.split()) < MIN_WORDS:
            continue
        out.append({"text": p, "source": source, "words": _normalize_words(prose)})
    return out


def split_sentences(text: str, source: str) -> list[dict]:
    """Catches a rule restated inside different surrounding prose, which
    dilutes below the paragraph threshold. `text` keeps inline code verbatim;
    similarity words don't."""
    out = []
    for s in split_into_original_sentences(text):
        prose = strip_code(s)
        if len(prose.split()) < MIN_SENTENCE_WORDS:
            continue
        out.append(
            {"text": s, "source": source, "words": _normalize_words(prose), "seq": _word_seq(prose)}
        )
    return out


def jaccard(a: set, b: set) -> float:
    if not a or not b:
        return 0.0
    union = len(a | b)
    return len(a & b) / union if union else 0.0


def find_duplicate_groups(
    paragraphs: list[dict],
    threshold: float = DEFAULT_THRESHOLD,
    phrase_min_words: int | None = None,
) -> list[dict]:
    """`phrase_min_words`: also count a pair as duplicate when it shares a word
    run this long despite low Jaccard. Use for sentences; leave None for
    paragraphs, where a whole-block match is the target."""
    n = len(paragraphs)
    parent = list(range(n))

    def find(x):
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    def union(x, y):
        rx, ry = find(x), find(y)
        if rx != ry:
            parent[ry] = rx

    max_sim = {}
    for i, j in combinations(range(n), 2):
        sim = jaccard(paragraphs[i]["words"], paragraphs[j]["words"])
        is_dup = sim >= threshold
        if not is_dup and phrase_min_words is not None:
            run = longest_common_run(paragraphs[i].get("seq", []), paragraphs[j].get("seq", []))
            if run >= phrase_min_words:
                is_dup = True
                sim = max(
                    sim, run / max(1, min(len(paragraphs[i]["seq"]), len(paragraphs[j]["seq"])))
                )
        if is_dup:
            union(i, j)
            max_sim[i] = max(max_sim.get(i, 0.0), sim)
            max_sim[j] = max(max_sim.get(j, 0.0), sim)

    groups: dict[int, list[int]] = {}
    for i in range(n):
        groups.setdefault(find(i), []).append(i)

    return [
        {
            "members": [
                {"text": paragraphs[i]["text"], "source": paragraphs[i]["source"]} for i in idxs
            ],
            "max_similarity": round(max(max_sim.get(i, 0.0) for i in idxs), 3),
            "duplicate_tokens_estimate": _estimate_dup_tokens(paragraphs, idxs),
        }
        for idxs in groups.values()
        if len(idxs) > 1
    ]


def _estimate_dup_tokens(paragraphs: list[dict], idxs: list[int]) -> int:
    """Tokens that would be saved by keeping only the first occurrence."""
    sizes = sorted((estimate_tokens(paragraphs[i]["text"]) for i in idxs), reverse=True)
    return sum(sizes[1:])  # keep the largest as canonical, the rest are "duplicate"


def find_duplicates_in_skill(
    parsed: dict,
    threshold: float = DEFAULT_THRESHOLD,
    sentence_threshold: float = DEFAULT_SENTENCE_THRESHOLD,
) -> dict:
    """Scans SKILL.md sections and reference files together, so cross-file
    repeats are caught. Returns `paragraphs` and `sentences` groups."""
    paragraphs, sentences = [], []
    for s in parsed["sections"]:
        label = f"SKILL.md#{s['heading'] or 'intro'}"
        paragraphs.extend(split_paragraphs(s["content"], label))
        sentences.extend(split_sentences(s["content"], label))
    for name, text in parsed["references"].items():
        paragraphs.extend(split_paragraphs(text, f"references/{name}"))
        sentences.extend(split_sentences(text, f"references/{name}"))

    paragraph_groups = find_duplicate_groups(paragraphs, threshold)
    sentence_groups = find_duplicate_groups(
        sentences, sentence_threshold, phrase_min_words=MIN_SHARED_PHRASE_WORDS
    )
    # a sentence group whose members are already fully covered by a
    # reported paragraph group is redundant noise -- drop it
    paragraph_texts = {" ".join(m["text"].split()) for g in paragraph_groups for m in g["members"]}
    sentence_groups = [
        g
        for g in sentence_groups
        if not all(
            any(" ".join(m["text"].split()) in p for p in paragraph_texts) for m in g["members"]
        )
    ]
    return {"paragraphs": paragraph_groups, "sentences": sentence_groups}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall