Skip to content

hypnguyen1209/offensive-claude / skills/vulnerability-analysis

vulnerability-analysis

Unverified

Use when auditing source code for vulnerabilities — drive CodeQL/Semgrep/Joern to taint untrusted data source-to-sink across injection, memory safety, deserialization/prototype-pollution, secrets/crypto/authz/race, and supply-chain risks

scripts/vocab_gate.py

View on GitHub
#!/usr/bin/env python3
"""vocab_gate.py - executable vocabulary-grounding (graphify Step 0): mechanizes "never name-guess".

The CLAUDE.md rule is "ground every claim and never name-guess" - if you reference a function/symbol,
it must actually exist in the target. This makes that checkable: it extracts the real identifier
vocabulary from the in-scope target, then GATES a set of candidate tokens (the symbols an audit query
wants to reference) against it. A token that is not in the target is INVENTED -> the gate fails
("select from this list; do not invent tokens; if nothing matches, stop").

Fail-CLOSED: an unreadable target yields an EMPTY vocabulary, so every token reads as invented and
the gate fails rather than waving tokens through on a vacuous vocabulary.

  extract  <root> [--lang py,js,php,java] [--defs-only] [--out vocab.json]
           -> the target's identifier vocabulary (all identifiers, minus language keywords;
              --defs-only restricts to function/class/method definition names)
  check    --vocab vocab.json (--tokens a,b,c | --query "free text")
           -> partitions tokens into grounded / invented; exit 1 if ANY token is invented

NOTE: the caller is responsible for confirming <root> is in scope (scope_guard.py) before extraction.
"""
from __future__ import annotations

import argparse
import json
import os
import re
import sys
from pathlib import Path
from typing import Optional

_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
_SKIP_DIRS = {"node_modules", ".git", "vendor", "venv", ".venv", "dist", "build",
              "__pycache__", "target", ".mypy_cache", ".pytest_cache"}
_EXTS = {
    "py": (".py",), "js": (".js", ".ts", ".jsx", ".tsx", ".mjs"),
    "php": (".php", ".phtml", ".inc"), "java": (".java", ".kt"),
}
# language keywords / builtins are NOT symbols of the target - excluding them stops a query from
# "grounding" on the word `return` or `function`. A shared conservative set across the langs.
_KEYWORDS = {
    "if", "else", "elif", "for", "while", "return", "def", "class", "function", "var", "let",
    "const", "import", "from", "as", "in", "is", "and", "or", "not", "new", "try", "catch",
    "except", "finally", "raise", "throw", "with", "yield", "await", "async", "public", "private",
    "protected", "static", "void", "int", "string", "bool", "boolean", "float", "double", "char",
    "true", "false", "null", "none", "self", "this", "super", "then", "case", "switch", "break",
    "continue", "default", "do", "extends", "implements", "interface", "package", "using", "namespace",
    "echo", "print", "require", "require_once", "include", "include_once", "lambda", "pass", "global",
}
_DEF_RX = {
    "py": [re.compile(r"^\s*(?:async\s+)?def\s+(\w+)"), re.compile(r"^\s*class\s+(\w+)")],
    "js": [re.compile(r"function\s+(\w+)"), re.compile(r"(\w+)\s*[:=]\s*(?:async\s+)?\([^)]*\)\s*=>"),
           re.compile(r"class\s+(\w+)")],
    "php": [re.compile(r"function\s+(\w+)"), re.compile(r"class\s+(\w+)")],
    "java": [re.compile(r"(?:class|interface|enum)\s+(\w+)"),
             re.compile(r"(?:public|private|protected|static|\s)+[\w<>\[\],\s]+\s+(\w+)\s*\(")],
}


def _iter_files(root: Path, langs: list):
    exts = tuple(e for L in langs for e in _EXTS[L])
    for dp, dn, fn in os.walk(root):
        dn[:] = [d for d in dn if d not in _SKIP_DIRS]
        for f in fn:
            if f.endswith(exts):
                yield Path(dp) / f


def _lang_for(path: Path) -> Optional[str]:
    for L, exts in _EXTS.items():
        if path.name.endswith(exts):
            return L
    return None


def extract_vocab(root, langs: list, *, defs_only: bool = False) -> set:
    """Return the set of identifier tokens in the target. Fail-closed: unreadable files contribute
    nothing (an empty vocab => every token later reads as invented)."""
    root = Path(root)
    vocab: set = set()
    for path in _iter_files(root, langs):
        try:
            text = path.read_text(encoding="utf-8", errors="replace")
        except OSError:
            continue
        if defs_only:
            L = _lang_for(path)
            for line in text.splitlines():
                for rx in _DEF_RX.get(L, []):
                    m = rx.search(line)
                    if m:
                        vocab.add(m.group(1))
        else:
            for tok in _IDENT.findall(text):
                if tok.lower() not in _KEYWORDS:
                    vocab.add(tok)
    return vocab


def check_tokens(tokens, vocab: set) -> dict:
    """Partition tokens into grounded / invented against the vocabulary (case-sensitive: a symbol's
    exact spelling matters)."""
    grounded, invented = [], []
    for t in tokens:
        (grounded if t in vocab else invented).append(t)
    return {"grounded": grounded, "invented": invented, "ok": not invented}


def _tokens_from_query(query: str) -> list:
    seen, out = set(), []
    for tok in _IDENT.findall(query or ""):
        if tok.lower() in _KEYWORDS or tok in seen:
            continue
        seen.add(tok)
        out.append(tok)
    return out


def main(argv: Optional[list] = None) -> int:
    p = argparse.ArgumentParser(description="Vocabulary-grounding gate: never name-guess.")
    sub = p.add_subparsers(dest="cmd", required=True)
    pe = sub.add_parser("extract", help="extract the target identifier vocabulary")
    pe.add_argument("root")
    pe.add_argument("--lang", default="py,js,php,java")
    pe.add_argument("--defs-only", action="store_true")
    pe.add_argument("--out")
    pc = sub.add_parser("check", help="gate candidate tokens against a vocabulary")
    pc.add_argument("--vocab", required=True, help="vocab.json produced by extract")
    pc.add_argument("--tokens", help="comma-separated candidate tokens")
    pc.add_argument("--query", help="free text; identifiers are extracted and checked")
    pc.add_argument("--json", action="store_true")
    args = p.parse_args(argv)

    if args.cmd == "extract":
        langs = [x.strip() for x in args.lang.split(",") if x.strip() in _EXTS]
        if not langs:
            print("error: no valid --lang", file=sys.stderr)
            return 2
        vocab = sorted(extract_vocab(args.root, langs, defs_only=args.defs_only))
        payload = {"root": str(args.root), "count": len(vocab), "vocab": vocab}
        if args.out:
            Path(args.out).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
            print(f"extracted {len(vocab)} tokens -> {args.out}")
        else:
            print(json.dumps(payload, indent=2))
        return 0

    # check
    try:
        data = json.loads(Path(args.vocab).read_text(encoding="utf-8"))
        vocab = set(data.get("vocab", []) if isinstance(data, dict) else data)
    except (OSError, ValueError, json.JSONDecodeError) as exc:
        print(f"error: --vocab: {exc}", file=sys.stderr)
        return 2
    if args.tokens:
        tokens = [t.strip() for t in args.tokens.split(",") if t.strip()]
    elif args.query:
        tokens = _tokens_from_query(args.query)
    else:
        print("error: pass --tokens or --query", file=sys.stderr)
        return 2
    result = check_tokens(tokens, vocab)
    if args.json:
        print(json.dumps(result, indent=2))
    else:
        if result["ok"]:
            print(f"ok: all {len(tokens)} token(s) grounded in the target vocabulary")
        else:
            print(f"INVENTED (not in target): {', '.join(result['invented'])}")
            print("-> select only from the target vocabulary; if nothing matches, stop.")
    return 0 if result["ok"] else 1


if __name__ == "__main__":
    raise SystemExit(main())
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall