Skip to content

richfrem/agent-plugins-skills / plugins/agent-orchestration/.apm/skills/evaluating-agent-outcomes

evaluating-agent-outcomes

Verified

Repository-independent evaluation of delegated agent work: assesses a delivery against a work contract (criterion-level pass/fail/unverified, failure conditions, budget, authorization), checks evidence validity, selects proportionate verification, measures cost with missing data kept unknown, classifies human interventions, and records and retrieves specific lessons. Use when orchestrating-agent-work needs findings after a worker returns, or to gate a benchmark before model calls. Not an orchestrator and never grants authority.

scripts/eval_cli.py

View on GitHub
#!/usr/bin/env python3
"""CLI for eval_core gates: select | fixtures | pilot | validity | intervention | repair | invest | summary."""
import argparse, json, sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import eval_core as c


def _j(p):
    return json.loads(Path(p).read_text())


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--root", default="."); ap.add_argument("--store")
    sub = ap.add_subparsers(dest="cmd", required=True)
    s = sub.add_parser("select"); s.add_argument("--deterministic", action="store_true"); s.add_argument("--consequential", action="store_true")
    s.add_argument("--uncertain", action="store_true"); s.add_argument("--unaffordable-preregistered", action="store_true")
    f = sub.add_parser("fixtures"); f.add_argument("file"); f.add_argument("--cwd", default=".")
    p = sub.add_parser("pilot"); p.add_argument("file"); p.add_argument("--pattern", required=True); p.add_argument("--min-adherence", type=float, default=0.8)
    p.add_argument("--confirmatory-ids")
    v = sub.add_parser("validity"); v.add_argument("file", help='JSON {"planned","valid","arms", optional "claim":{"type",...}}'); v.add_argument("--min-adherence", type=float, default=0.8)
    i = sub.add_parser("intervention"); i.add_argument("--category", required=True, choices=c.INTERVENTION_CATEGORIES)
    i.add_argument("--reason", required=True); i.add_argument("--observed-event", required=True); i.add_argument("--mission")
    r = sub.add_parser("repair"); r.add_argument("action", choices=["record", "applied", "report"])
    for n in ("category", "evidence", "repair", "verification", "owner", "outcome", "unit"):
        r.add_argument("--" + n)
    r.add_argument("--cause", default="suspected", choices=["suspected", "confirmed"]); r.add_argument("--cost-tokens", type=int)
    n_ = sub.add_parser("invest"); n_.add_argument("file")
    sub.add_parser("summary")
    a = ap.parse_args(argv)
    kw = {"root": a.root, "store": a.store}
    if a.cmd == "select": out = c.select_method(a.deterministic, a.consequential, a.uncertain, a.unaffordable_preregistered)
    elif a.cmd == "fixtures": out = c.run_fixtures(_j(a.file), a.cwd)
    elif a.cmd == "pilot": out = c.pilot_adherence(_j(a.file), a.pattern, a.min_adherence, _j(a.confirmatory_ids) if a.confirmatory_ids else ())
    elif a.cmd == "validity":
        d = _j(a.file)
        out = c.assess_claim(d["planned"], d["valid"], d.get("arms", {}), d["claim"], a.min_adherence) if d.get("claim") is not None \
            else c.validity(d["planned"], d["valid"], d.get("arms", {}), a.min_adherence)
    elif a.cmd == "intervention": out = c.record_intervention(a.category, a.reason, a.observed_event, a.mission, **kw)
    elif a.cmd == "repair":
        if a.action == "record": out = c.record_repair(a.category, a.evidence, a.repair, a.verification, a.owner, a.cause, repair_cost_tokens=a.cost_tokens, **kw)
        elif a.action == "applied": out = c.record_application(a.category, a.outcome, a.unit, **kw)
        else: out = c.repair_report(**kw)
    elif a.cmd == "invest": out = c.investment(_j(a.file))
    else: out = {"interventions": c.intervention_summary(**kw), "repairs": c.repair_report(**kw)}
    print(json.dumps(out, indent=1, sort_keys=True))
    bad = (a.cmd == "fixtures" and not out["passed"]) or (a.cmd == "pilot" and not out["ready"]) or (a.cmd == "validity" and out["max_claim"] not in ("BEHAVIORAL_CLAIM_ALLOWED", "CLAIM_SUPPORTED"))
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall