ericrisco/rsc-harness / skills/agent-eval
agent-eval
VerifiedUse when measuring whether an LLM or agent system actually got better and gating merges on it: golden sets, fixing an inflated LLM-as-judge, scoring RAG (faithfulness, contextual recall) or agent trajectories (tool correctness, completion), or picking an eval framework. NOT building the agent loop, tools or RAG plumbing (that is `building-agents`).
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 12.6 KB |
| evals/README.md | markdown | 847 B |
| evals/cases.yaml | config | 3.3 KB |
| references/judge-design.md | markdown | 5.1 KB |
| references/runner-and-gate.md | markdown | 5.8 KB |
| scripts/verify.shexecutable | script | 5.8 KB |
scripts/verify.sh
View on GitHub#!/usr/bin/env bash
set -euo pipefail
# verify.sh — agent-eval skill gate. Run from your PROJECT root.
#
# What it does (read-only, idempotent, NEVER calls an LLM or network):
# 1. Discovers *.jsonl golden sets (skips vendor dirs); checks every line parses
# and carries the required keys id, input, expected. Hard fail on a bad line.
# 2. If an eval-report.json exists, validates its shape (a "passed" boolean and a
# "metrics" object). Hard fail only on malformed JSON / missing shape.
# 3. Optional lint, advisory only: ruff + mypy on example Python, markdownlint on docs.
#
# Exit non-zero ONLY on a malformed JSONL line, a missing required key, or a malformed
# eval-report.json. Every optional lint and every missing tool is a yellow WARN/skip.
# An empty or clean target exits 0. Stock macOS bash 3.2 (no mapfile / associative arrays).
YELLOW=$'\033[33m'; GREEN=$'\033[32m'; RED=$'\033[31m'; NC=$'\033[0m'
EXIT=0
skip() { printf '%s[skip]%s %s\n' "$YELLOW" "$NC" "$*"; }
note() { printf '%s[warn]%s %s\n' "$YELLOW" "$NC" "$*"; }
ok() { printf '%s[ok]%s %s\n' "$GREEN" "$NC" "$*"; }
err() { printf '%s[fail]%s %s\n' "$RED" "$NC" "$*"; EXIT=1; }
ROOT="$(pwd)"
# Need a JSON-aware tool for the structural checks. Prefer python3, fall back to jq.
JSON_TOOL=""
if command -v python3 >/dev/null 2>&1; then JSON_TOOL="python3"
elif command -v jq >/dev/null 2>&1; then JSON_TOOL="jq"; fi
# --- 1. golden-set JSONL validation -----------------------------------------
JSONL_FILES=()
while IFS= read -r -d '' f; do
JSONL_FILES+=("$f")
done < <(
find "$ROOT" \
\( -path '*/node_modules/*' -o -path '*/.git/*' -o -path '*/vendor/*' \
-o -path '*/.venv/*' -o -path '*/dist/*' \) -prune -o \
-type f -name '*.jsonl' -print0 2>/dev/null
)
if [ "${#JSONL_FILES[@]}" -eq 0 ]; then
skip "no *.jsonl golden sets found under $ROOT — nothing to validate"
elif [ -z "$JSON_TOOL" ]; then
skip "neither python3 nor jq found — cannot validate JSONL, skipping"
else
for f in "${JSONL_FILES[@]}"; do
if [ "$JSON_TOOL" = "python3" ]; then
msg="$(python3 - "$f" <<'PY'
import json, sys
path = sys.argv[1]
req = {"id", "input", "expected"}
bad = []
with open(path, encoding="utf-8") as fh:
for n, line in enumerate(fh, 1):
s = line.strip()
if not s:
continue
try:
obj = json.loads(s)
except ValueError as e:
bad.append(f"line {n}: invalid JSON ({e})"); continue
if not isinstance(obj, dict):
bad.append(f"line {n}: not a JSON object"); continue
missing = req - obj.keys()
if missing:
bad.append(f"line {n}: missing keys {sorted(missing)}")
print("\n".join(bad), end="")
PY
)" || msg="parser crashed on $f"
if [ -n "$msg" ]; then
err "$f:"; printf ' %s\n' "$msg"
else
ok "$f valid (id/input/expected present on every line)"
fi
else
# jq path: each line must be an object with the three keys.
if jq -e 'has("id") and has("input") and has("expected")' "$f" >/dev/null 2>&1; then
ok "$f valid (jq: required keys present)"
else
err "$f: a line is not an object or is missing id/input/expected (jq)"
fi
fi
done
fi
# --- 2. eval-report.json shape ------------------------------------------------
REPORTS=()
while IFS= read -r -d '' f; do
REPORTS+=("$f")
done < <(
find "$ROOT" \
\( -path '*/node_modules/*' -o -path '*/.git/*' -o -path '*/.venv/*' \) -prune -o \
-type f -name 'eval-report.json' -print0 2>/dev/null
)
if [ "${#REPORTS[@]}" -eq 0 ]; then
skip "no eval-report.json found — gate output not validated"
elif [ -z "$JSON_TOOL" ]; then
skip "no python3/jq — cannot validate eval-report.json shape"
else
for f in "${REPORTS[@]}"; do
if [ "$JSON_TOOL" = "python3" ]; then
if python3 - "$f" <<'PY'
import json, sys
try:
r = json.load(open(sys.argv[1], encoding="utf-8"))
except ValueError as e:
print(e); sys.exit(1)
ok = isinstance(r, dict) and isinstance(r.get("passed"), bool) \
and isinstance(r.get("metrics"), dict)
sys.exit(0 if ok else 1)
PY
then ok "$f shape ok (passed: bool, metrics: object)"
else err "$f: malformed or missing 'passed'(bool)/'metrics'(object)"; fi
else
if jq -e 'type=="object" and (.passed|type=="boolean") and (.metrics|type=="object")' "$f" >/dev/null 2>&1; then
ok "$f shape ok (jq)"
else err "$f: malformed or missing passed/metrics (jq)"; fi
fi
done
fi
# --- 3. optional lint (advisory only) ----------------------------------------
PY_FILES=()
while IFS= read -r -d '' f; do
PY_FILES+=("$f")
done < <(
find "$ROOT" \
\( -path '*/node_modules/*' -o -path '*/.git/*' -o -path '*/.venv/*' \) -prune -o \
-type f -name '*.py' -print0 2>/dev/null
)
if [ "${#PY_FILES[@]}" -gt 0 ]; then
if command -v ruff >/dev/null 2>&1; then
if ruff check "${PY_FILES[@]}" >/dev/null 2>&1; then ok "ruff clean"; else note "ruff reported lint findings (advisory)"; fi
else skip "ruff not installed — skipping Python lint"; fi
if command -v mypy >/dev/null 2>&1; then
mypy "${PY_FILES[@]}" >/dev/null 2>&1 && ok "mypy clean" || note "mypy reported type findings (advisory)"
else skip "mypy not installed — skipping type check"; fi
else
skip "no *.py files — skipping ruff/mypy"
fi
MD_FILES=()
while IFS= read -r -d '' f; do
MD_FILES+=("$f")
done < <(
find "$ROOT" \
\( -path '*/node_modules/*' -o -path '*/.git/*' \) -prune -o \
-type f -name '*.md' -print0 2>/dev/null
)
if [ "${#MD_FILES[@]}" -gt 0 ] && command -v markdownlint >/dev/null 2>&1; then
markdownlint "${MD_FILES[@]}" >/dev/null 2>&1 && ok "markdownlint clean" || note "markdownlint findings (advisory)"
else
skip "markdownlint not installed or no docs — skipping doc lint"
fi
printf '\n'
if [ "$EXIT" -eq 0 ]; then ok "verify.sh passed"; else err "verify.sh found failures"; fi
exit "$EXIT"
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 470k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |