Skip to content

hypnguyen1209/offensive-claude / skills/vulnerability-analysis

vulnerability-analysis

Unverified

Use when auditing source code for vulnerabilities — drive CodeQL/Semgrep/Joern to taint untrusted data source-to-sink across injection, memory safety, deserialization/prototype-pollution, secrets/crypto/authz/race, and supply-chain risks

scripts/finding_dedup.py

View on GitHub
#!/usr/bin/env python3
"""finding_dedup.py - content-hash dedup for finding records (claude-mem idea).

The same vulnerability surfaces repeatedly: phase re-runs, `/engage.pickup` resumes, and each
rebuttal iteration can re-emit the same finding. Recording it N times inflates the report and skews
any count-based metric. This assigns each finding a stable content hash and drops duplicates.

  content_hash = sha256(scope \\0 title \\0 sink \\0 cwe)[:16]   (fields lowercased + stripped)

WINDOW RULE (important): dedup is WITHIN-PHASE / WITHIN-RUN. A finding legitimately re-confirmed in a
*later kill-chain phase* is NOT a duplicate - re-confirmation is signal, not noise. So the dedup key
is (content_hash, phase): identical content in the SAME phase collapses to the first occurrence;
the same content re-observed in a DIFFERENT phase is kept. A finding with no `phase` field is
treated as one default bucket (pure within-run dedup).

CLI:
  finding_dedup.py --findings findings.json [--out deduped.json] [--json]
  exit 0 always (dedup is advisory hygiene, not a gate); 2 on error.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Optional


def _norm(value) -> str:
    return str(value if value is not None else "").strip().lower()


def content_hash(finding: dict) -> str:
    """Stable 16-hex content hash over the identity fields (scope, title, sink, cwe)."""
    if not isinstance(finding, dict):
        parts = ["", "", "", ""]
    else:
        parts = [_norm(finding.get("scope")), _norm(finding.get("title")),
                 _norm(finding.get("sink")), _norm(finding.get("cwe"))]
    blob = "\0".join(parts).encode("utf-8")
    return hashlib.sha256(blob).hexdigest()[:16]


def _phase(finding: dict) -> str:
    return _norm(finding.get("phase")) if isinstance(finding, dict) else ""


@dataclass
class DedupResult:
    kept: list
    dropped: list          # list of {"index", "content_hash", "phase", "duplicate_of"}

    def to_dict(self) -> dict:
        return {"kept": self.kept, "dropped": self.dropped,
                "kept_count": len(self.kept), "dropped_count": len(self.dropped)}


def dedup(findings: list) -> DedupResult:
    """Collapse (content_hash, phase) duplicates, keeping the first occurrence of each."""
    seen: dict = {}                 # (hash, phase) -> index of the kept finding
    kept, dropped = [], []
    for i, f in enumerate(findings):
        key = (content_hash(f), _phase(f))
        if key in seen:
            dropped.append({"index": i, "content_hash": key[0], "phase": key[1],
                            "duplicate_of": seen[key]})
        else:
            seen[key] = i
            kept.append(f)
    return DedupResult(kept, dropped)


def main(argv: Optional[list] = None) -> int:
    p = argparse.ArgumentParser(description="Content-hash dedup for finding records (within-phase).")
    p.add_argument("--findings", required=True)
    p.add_argument("--out", help="write the deduped finding list here")
    p.add_argument("--json", action="store_true")
    args = p.parse_args(argv)
    try:
        data = json.loads(Path(args.findings).read_text(encoding="utf-8"))
        if isinstance(data, dict):
            data = data.get("findings", [])
        if not isinstance(data, list):
            raise ValueError("findings file must be a JSON list (or {findings:[...]})")
    except (OSError, ValueError, json.JSONDecodeError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return 2
    result = dedup(data)
    if args.out:
        try:
            Path(args.out).write_text(json.dumps(result.kept, indent=2) + "\n", encoding="utf-8")
        except OSError as exc:
            print(f"error writing --out: {exc}", file=sys.stderr)
            return 2
    if args.json:
        print(json.dumps(result.to_dict(), indent=2))
    else:
        print(f"kept {len(result.kept)}, dropped {len(result.dropped)} duplicate(s)")
        for d in result.dropped:
            print(f"  drop #{d['index']} (dup of #{d['duplicate_of']}) hash={d['content_hash']} phase={d['phase']!r}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall