Skip to content

me2resh/apexyard / .claude/skills/eval-agents

eval-agents

Unverified

Score a review agent (Rex/Hakim/Tariq) against a labeled PR corpus — ground-truth defect-set overlap, never a prose rubric. Headline metric is approve-precision.

FileKindSize
SKILL.mdskill md28.8 KB
lib/mine_disagreements.pyexecutablescript23.1 KB
lib/snapshot-diff.shexecutablescript3.3 KB
lib/source-snapshot.shexecutablescript6.7 KB
lib/validate-corpus.shexecutablescript8.7 KB
tests/fixtures/mine-disagreements/edge-cases.jsonconfig4.9 KB
tests/fixtures/mine-disagreements/merged-prs.jsonconfig10.8 KB
tests/smoke.shexecutablescript3.4 KB
tests/test-snapshot-diff.shexecutablescript3.2 KB
tests/test_mine_disagreements.pyscript17.0 KB
tests/test_mine_disagreements.shexecutablescript5.6 KB
tests/test_source_snapshot.shexecutablescript11.8 KB

tests/test-snapshot-diff.sh

View on GitHub
#!/usr/bin/env bash
# test-snapshot-diff.sh — smoke tests for the /eval-agents contamination
# backstop (snapshot-diff.sh, #833).
# Run: .claude/skills/eval-agents/tests/test-snapshot-diff.sh
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SNAPSHOT_DIFF="$SCRIPT_DIR/../lib/snapshot-diff.sh"

pass=0
fail=0

check() {
  local desc="$1" expect_rc="$2"; shift 2
  "$@" >/tmp/snapshot-diff-test.out 2>&1
  local rc=$?
  if [ "$rc" -eq "$expect_rc" ]; then
    echo "✓ $desc"
    pass=$((pass + 1))
  else
    echo "✗ $desc (expected exit $expect_rc, got $rc)"
    cat /tmp/snapshot-diff-test.out
    fail=$((fail + 1))
  fi
}

check_output_contains() {
  local desc="$1" needle="$2"
  if grep -qF "$needle" /tmp/snapshot-diff-test.out; then
    echo "✓ $desc"
    pass=$((pass + 1))
  else
    echo "✗ $desc (output did not contain: $needle)"
    cat /tmp/snapshot-diff-test.out
    fail=$((fail + 1))
  fi
}

WORKDIR="$(mktemp -d)"
trap 'rm -rf "$WORKDIR"' EXIT

TARGET="$WORKDIR/reviews"
SNAP="$WORKDIR/snap.manifest"

mkdir -p "$TARGET"
echo "existing content" > "$TARGET/keep.txt"

# 1. check with no prior snapshot is a usage error, not a false negative.
check "check without a prior snapshot fails cleanly" 2 bash "$SNAPSHOT_DIFF" check "$TARGET" "$SNAP"

# 2. snapshot succeeds and check right after reports no contamination.
check "snapshot succeeds" 0 bash "$SNAPSHOT_DIFF" snapshot "$TARGET" "$SNAP"
check "check immediately after snapshot is clean" 0 bash "$SNAPSHOT_DIFF" check "$TARGET" "$SNAP"

# 3. a new file (simulating a silent marker write) is flagged NEW.
echo "contamination" > "$TARGET/pr-99-rex.approved"
check "new file is flagged as contamination" 1 bash "$SNAPSHOT_DIFF" check "$TARGET" "$SNAP"
check_output_contains "new-file report names the path" "NEW:     pr-99-rex.approved"
rm -f "$TARGET/pr-99-rex.approved"

# 4. back to clean after the extra file is removed (re-snapshot first).
check "re-snapshot after cleanup" 0 bash "$SNAPSHOT_DIFF" snapshot "$TARGET" "$SNAP"
check "check is clean again" 0 bash "$SNAPSHOT_DIFF" check "$TARGET" "$SNAP"

# 5. a changed file (same path, different content) is flagged CHANGED, not NEW.
echo "mutated content" > "$TARGET/keep.txt"
check "changed file is flagged as contamination" 1 bash "$SNAPSHOT_DIFF" check "$TARGET" "$SNAP"
check_output_contains "changed-file report names the path" "CHANGED: keep.txt"

# 6. a deleted file is flagged DELETED.
check "re-snapshot before delete test" 0 bash "$SNAPSHOT_DIFF" snapshot "$TARGET" "$SNAP"
rm -f "$TARGET/keep.txt"
check "deleted file is flagged as contamination" 1 bash "$SNAPSHOT_DIFF" check "$TARGET" "$SNAP"
check_output_contains "deleted-file report names the path" "DELETED: keep.txt"

# 7. snapshotting a directory that doesn't exist yet is not itself an error
#    (a fresh session may have no .claude/session/reviews/ at all).
FRESH="$WORKDIR/does-not-exist-yet"
FRESH_SNAP="$WORKDIR/fresh.manifest"
check "snapshot of a missing dir creates it and succeeds" 0 bash "$SNAPSHOT_DIFF" snapshot "$FRESH" "$FRESH_SNAP"
check "check of an untouched fresh dir is clean" 0 bash "$SNAPSHOT_DIFF" check "$FRESH" "$FRESH_SNAP"

echo
echo "snapshot-diff test: $pass passed, $fail failed"
[ "$fail" -eq 0 ]
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall