oliverweissl/context-garden / skills/weeder
weeder
VerifiedMeasure and reduce a Skill's token footprint (always-loaded description + on-trigger SKILL.md body) without regressing routing accuracy or losing instructions. Use when a SKILL.md feels oversized, repeats rules, or has a vague description that might mis-route.
| File | Kind | Size |
|---|---|---|
| .gitignore | other | 32 B |
| SKILL.md | skill md | 3.1 KB |
| agents/weeder.md | markdown | 1.2 KB |
| bin/weederexecutable | script | 292 B |
| references/classification.md | markdown | 4.9 KB |
| references/llm-assist.md | markdown | 2.1 KB |
| references/routing-heuristic.md | markdown | 4.1 KB |
| scripts/cg_mode.py | script | 5.4 KB |
| scripts/we_apply_suggestion.py | script | 5.7 KB |
| scripts/we_audit.py | script | 5.6 KB |
| scripts/we_config.py | script | 2.5 KB |
| scripts/we_constraints.py | script | 5.1 KB |
| scripts/we_dupes.py | script | 6.1 KB |
| scripts/we_filler.py | script | 4.9 KB |
| scripts/we_optimize.py | script | 4.3 KB |
| scripts/we_parse.py | script | 5.0 KB |
| scripts/we_routing.py | script | 4.5 KB |
| scripts/we_suggest.py | script | 3.8 KB |
| scripts/we_text.py | script | 2.1 KB |
| scripts/we_tokens.py | script | 4.1 KB |
| scripts/weeder.pyexecutable | script | 12.0 KB |
| tests/fixtures/answer_lot.json | config | 875 B |
| tests/fixtures/filler_sentences.json | config | 2.6 KB |
| tests/fixtures/routing_examples.json | config | 489 B |
| tests/smoke_test.shexecutable | script | 14.3 KB |
scripts/we_dupes.py
View on GitHub"""Near-duplicate detection (normalized-word Jaccard, plus shared word runs at
sentence level): a rule repeated in five places collapses to one finding."""
from __future__ import annotations
import re
from itertools import combinations
from we_text import _CODE_FENCE_RE, split_into_original_sentences, strip_code
from we_tokens import estimate_tokens
MIN_WORDS = 8
MIN_SENTENCE_WORDS = 6
DEFAULT_THRESHOLD = 0.6
DEFAULT_SENTENCE_THRESHOLD = 0.45
MIN_SHARED_PHRASE_WORDS = 6
_WORD_RE = re.compile(r"[a-z0-9]+")
def _normalize_words(text: str) -> set[str]:
return set(_WORD_RE.findall(text.lower()))
def _word_seq(text: str) -> list[str]:
return _WORD_RE.findall(text.lower())
def longest_common_run(a: list[str], b: list[str]) -> int:
"""Longest common contiguous word run. Jaccard is diluted by differing
elaboration; a long shared run catches an embedded restated rule."""
if not a or not b:
return 0
prev = [0] * (len(b) + 1)
best = 0
for i in range(1, len(a) + 1):
curr = [0] * (len(b) + 1)
for j in range(1, len(b) + 1):
if a[i - 1] == b[j - 1]:
curr[j] = prev[j - 1] + 1
best = max(best, curr[j])
prev = curr
return best
def split_paragraphs(text: str, source: str) -> list[dict]:
"""Fenced code is dropped: usage examples legitimately repeat the same
command prefix and would be false positives. Inline code stays in `text`
(excluded only from similarity words) so `text` is a verbatim span
apply-suggestion can find in the original file."""
cleaned = _CODE_FENCE_RE.sub("\n\n", text)
paras = [p.strip() for p in re.split(r"\n\s*\n", cleaned) if p.strip()]
out = []
for p in paras:
prose = strip_code(p)
if len(prose.split()) < MIN_WORDS:
continue
out.append({"text": p, "source": source, "words": _normalize_words(prose)})
return out
def split_sentences(text: str, source: str) -> list[dict]:
"""Catches a rule restated inside different surrounding prose, which
dilutes below the paragraph threshold. `text` keeps inline code verbatim;
similarity words don't."""
out = []
for s in split_into_original_sentences(text):
prose = strip_code(s)
if len(prose.split()) < MIN_SENTENCE_WORDS:
continue
out.append(
{"text": s, "source": source, "words": _normalize_words(prose), "seq": _word_seq(prose)}
)
return out
def jaccard(a: set, b: set) -> float:
if not a or not b:
return 0.0
union = len(a | b)
return len(a & b) / union if union else 0.0
def find_duplicate_groups(
paragraphs: list[dict],
threshold: float = DEFAULT_THRESHOLD,
phrase_min_words: int | None = None,
) -> list[dict]:
"""`phrase_min_words`: also count a pair as duplicate when it shares a word
run this long despite low Jaccard. Use for sentences; leave None for
paragraphs, where a whole-block match is the target."""
n = len(paragraphs)
parent = list(range(n))
def find(x):
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def union(x, y):
rx, ry = find(x), find(y)
if rx != ry:
parent[ry] = rx
max_sim = {}
for i, j in combinations(range(n), 2):
sim = jaccard(paragraphs[i]["words"], paragraphs[j]["words"])
is_dup = sim >= threshold
if not is_dup and phrase_min_words is not None:
run = longest_common_run(paragraphs[i].get("seq", []), paragraphs[j].get("seq", []))
if run >= phrase_min_words:
is_dup = True
sim = max(
sim, run / max(1, min(len(paragraphs[i]["seq"]), len(paragraphs[j]["seq"])))
)
if is_dup:
union(i, j)
max_sim[i] = max(max_sim.get(i, 0.0), sim)
max_sim[j] = max(max_sim.get(j, 0.0), sim)
groups: dict[int, list[int]] = {}
for i in range(n):
groups.setdefault(find(i), []).append(i)
return [
{
"members": [
{"text": paragraphs[i]["text"], "source": paragraphs[i]["source"]} for i in idxs
],
"max_similarity": round(max(max_sim.get(i, 0.0) for i in idxs), 3),
"duplicate_tokens_estimate": _estimate_dup_tokens(paragraphs, idxs),
}
for idxs in groups.values()
if len(idxs) > 1
]
def _estimate_dup_tokens(paragraphs: list[dict], idxs: list[int]) -> int:
"""Tokens that would be saved by keeping only the first occurrence."""
sizes = sorted((estimate_tokens(paragraphs[i]["text"]) for i in idxs), reverse=True)
return sum(sizes[1:]) # keep the largest as canonical, the rest are "duplicate"
def find_duplicates_in_skill(
parsed: dict,
threshold: float = DEFAULT_THRESHOLD,
sentence_threshold: float = DEFAULT_SENTENCE_THRESHOLD,
) -> dict:
"""Scans SKILL.md sections and reference files together, so cross-file
repeats are caught. Returns `paragraphs` and `sentences` groups."""
paragraphs, sentences = [], []
for s in parsed["sections"]:
label = f"SKILL.md#{s['heading'] or 'intro'}"
paragraphs.extend(split_paragraphs(s["content"], label))
sentences.extend(split_sentences(s["content"], label))
for name, text in parsed["references"].items():
paragraphs.extend(split_paragraphs(text, f"references/{name}"))
sentences.extend(split_sentences(text, f"references/{name}"))
paragraph_groups = find_duplicate_groups(paragraphs, threshold)
sentence_groups = find_duplicate_groups(
sentences, sentence_threshold, phrase_min_words=MIN_SHARED_PHRASE_WORDS
)
# a sentence group whose members are already fully covered by a
# reported paragraph group is redundant noise -- drop it
paragraph_texts = {" ".join(m["text"].split()) for g in paragraph_groups for m in g["members"]}
sentence_groups = [
g
for g in sentence_groups
if not all(
any(" ".join(m["text"].split()) in p for p in paragraph_texts) for m in g["members"]
)
]
return {"paragraphs": paragraph_groups, "sentences": sentence_groups}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |