richfrem/agent-plugins-skills / plugins/agent-orchestration/.apm/skills/evaluating-agent-outcomes
evaluating-agent-outcomes
VerifiedRepository-independent evaluation of delegated agent work: assesses a delivery against a work contract (criterion-level pass/fail/unverified, failure conditions, budget, authorization), checks evidence validity, selects proportionate verification, measures cost with missing data kept unknown, classifies human interventions, and records and retrieves specific lessons. Use when orchestrating-agent-work needs findings after a worker returns, or to gate a benchmark before model calls. Not an orchestrator and never grants authority.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 3.6 KB |
| assets/rubrics/pr-review.json | config | 194 B |
| evals/evals.json | config | 724 B |
| references/gates.md | markdown | 1.5 KB |
| references/scoring-and-accounting.md | markdown | 1.9 KB |
| references/work-contract.md | markdown | 2.7 KB |
| scripts/accounting.py | script | 17.2 KB |
| scripts/eval_cli.py | script | 3.5 KB |
| scripts/eval_core.py | script | 16.3 KB |
| scripts/lessons.py | script | 5.6 KB |
| scripts/outcome_eval.py | script | 8.6 KB |
| scripts/scoring.py | script | 4.7 KB |
| scripts/usage_measure.py | script | 16.1 KB |
| tests/fixtures/agy_usage.json | config | 2.4 KB |
| tests/fixtures/claude_usage.txt | markdown | 718 B |
| tests/fixtures/claude_usage_resetless.txt | markdown | 215 B |
| tests/fixtures/sanctuary-contribution-v1.json | config | 794 B |
| tests/test_eval.py | script | 8.0 KB |
| tests/test_scoring_accounting.py | script | 5.7 KB |
scripts/eval_cli.py
View on GitHub#!/usr/bin/env python3
"""CLI for eval_core gates: select | fixtures | pilot | validity | intervention | repair | invest | summary."""
import argparse, json, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import eval_core as c
def _j(p):
return json.loads(Path(p).read_text())
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--root", default="."); ap.add_argument("--store")
sub = ap.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("select"); s.add_argument("--deterministic", action="store_true"); s.add_argument("--consequential", action="store_true")
s.add_argument("--uncertain", action="store_true"); s.add_argument("--unaffordable-preregistered", action="store_true")
f = sub.add_parser("fixtures"); f.add_argument("file"); f.add_argument("--cwd", default=".")
p = sub.add_parser("pilot"); p.add_argument("file"); p.add_argument("--pattern", required=True); p.add_argument("--min-adherence", type=float, default=0.8)
p.add_argument("--confirmatory-ids")
v = sub.add_parser("validity"); v.add_argument("file", help='JSON {"planned","valid","arms", optional "claim":{"type",...}}'); v.add_argument("--min-adherence", type=float, default=0.8)
i = sub.add_parser("intervention"); i.add_argument("--category", required=True, choices=c.INTERVENTION_CATEGORIES)
i.add_argument("--reason", required=True); i.add_argument("--observed-event", required=True); i.add_argument("--mission")
r = sub.add_parser("repair"); r.add_argument("action", choices=["record", "applied", "report"])
for n in ("category", "evidence", "repair", "verification", "owner", "outcome", "unit"):
r.add_argument("--" + n)
r.add_argument("--cause", default="suspected", choices=["suspected", "confirmed"]); r.add_argument("--cost-tokens", type=int)
n_ = sub.add_parser("invest"); n_.add_argument("file")
sub.add_parser("summary")
a = ap.parse_args(argv)
kw = {"root": a.root, "store": a.store}
if a.cmd == "select": out = c.select_method(a.deterministic, a.consequential, a.uncertain, a.unaffordable_preregistered)
elif a.cmd == "fixtures": out = c.run_fixtures(_j(a.file), a.cwd)
elif a.cmd == "pilot": out = c.pilot_adherence(_j(a.file), a.pattern, a.min_adherence, _j(a.confirmatory_ids) if a.confirmatory_ids else ())
elif a.cmd == "validity":
d = _j(a.file)
out = c.assess_claim(d["planned"], d["valid"], d.get("arms", {}), d["claim"], a.min_adherence) if d.get("claim") is not None \
else c.validity(d["planned"], d["valid"], d.get("arms", {}), a.min_adherence)
elif a.cmd == "intervention": out = c.record_intervention(a.category, a.reason, a.observed_event, a.mission, **kw)
elif a.cmd == "repair":
if a.action == "record": out = c.record_repair(a.category, a.evidence, a.repair, a.verification, a.owner, a.cause, repair_cost_tokens=a.cost_tokens, **kw)
elif a.action == "applied": out = c.record_application(a.category, a.outcome, a.unit, **kw)
else: out = c.repair_report(**kw)
elif a.cmd == "invest": out = c.investment(_j(a.file))
else: out = {"interventions": c.intervention_summary(**kw), "repairs": c.repair_report(**kw)}
print(json.dumps(out, indent=1, sort_keys=True))
bad = (a.cmd == "fixtures" and not out["passed"]) or (a.cmd == "pilot" and not out["ready"]) or (a.cmd == "validity" and out["max_claim"] not in ("BEHAVIORAL_CLAIM_ALLOWED", "CLAIM_SUPPORTED"))
return 1 if bad else 0
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |