richfrem/agent-plugins-skills / plugins/agent-orchestration/.apm/skills/evaluating-agent-outcomes
evaluating-agent-outcomes
VerifiedRepository-independent evaluation of delegated agent work: assesses a delivery against a work contract (criterion-level pass/fail/unverified, failure conditions, budget, authorization), checks evidence validity, selects proportionate verification, measures cost with missing data kept unknown, classifies human interventions, and records and retrieves specific lessons. Use when orchestrating-agent-work needs findings after a worker returns, or to gate a benchmark before model calls. Not an orchestrator and never grants authority.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 3.6 KB |
| assets/rubrics/pr-review.json | config | 194 B |
| evals/evals.json | config | 724 B |
| references/gates.md | markdown | 1.5 KB |
| references/scoring-and-accounting.md | markdown | 1.9 KB |
| references/work-contract.md | markdown | 2.7 KB |
| scripts/accounting.py | script | 17.2 KB |
| scripts/eval_cli.py | script | 3.5 KB |
| scripts/eval_core.py | script | 16.3 KB |
| scripts/lessons.py | script | 5.6 KB |
| scripts/outcome_eval.py | script | 8.6 KB |
| scripts/scoring.py | script | 4.7 KB |
| scripts/usage_measure.py | script | 16.1 KB |
| tests/fixtures/agy_usage.json | config | 2.4 KB |
| tests/fixtures/claude_usage.txt | markdown | 718 B |
| tests/fixtures/claude_usage_resetless.txt | markdown | 215 B |
| tests/fixtures/sanctuary-contribution-v1.json | config | 794 B |
| tests/test_eval.py | script | 8.0 KB |
| tests/test_scoring_accounting.py | script | 5.7 KB |
scripts/usage_measure.py
View on GitHub#!/usr/bin/env python3
"""Generic usage MEASUREMENT (stdlib only): zero-token vendor usage snapshots/diffs, transcript and rollout token readers,
secret scrubbing and catalog-priced dollar cost. Extracted verbatim from Sanctuary's usage_probe.py (MISSION-055); the
single authoritative implementation. Policy (floors, ceilings, predictors, ledgers) stays with the consumer.
Catalogs come from the installed cli-agent skill (env AGENT_CLI_REFERENCES, legacy SANCTUARY_CLI_AGENT_REFERENCES).
"""
import json
import os
import re
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
SECRET_PATTERNS = [
re.compile(r"sk-[A-Za-z0-9_\-]{16,}"),
re.compile(r"Bearer\s+[A-Za-z0-9._\-]{16,}"),
re.compile(r"eyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{6,}"),
re.compile(r"(?i)(api[_-]?key|access[_-]?token|refresh[_-]?token|secret)\"?\s*[:=]\s*\"?[A-Za-z0-9._\-]{16,}"),
re.compile(r"(?i)hf_[A-Za-z0-9]{20,}"),
]
def assert_no_secrets(obj: Any) -> None:
"""Raise ValueError if any credential-looking string appears in obj (serialised)."""
blob = json.dumps(obj, default=str)
for pat in SECRET_PATTERNS:
if pat.search(blob):
raise ValueError(f"credential-like pattern in usage record ({pat.pattern[:24]}...)")
def parse_claude_usage(text: str) -> Dict[str, Any]:
"""Parse headless `claude -p /usage` text into session (5-hour) and weekly percent used."""
out: Dict[str, Any] = {"session_pct_used": None, "session_resets": None,
"week_pct_used": None, "week_resets": None}
# The reset suffix is optional: the CLI omits "· resets ..." for some windows (observed for
# "Current session: 0% used"). The displayed percentage is kept as displayed; an absent
# reset is recorded as None, never as a zero or an imputed value.
m = re.search(r"Current session:\s*(\d+)%\s*used(?:\s*[·\-]\s*resets\s*(.+))?", text)
if m:
out["session_pct_used"] = int(m.group(1))
out["session_resets"] = m.group(2).strip() if m.group(2) else None
m = re.search(r"Current week \(all models\):\s*(\d+)%\s*used(?:\s*[·\-]\s*resets\s*(.+))?", text)
if m:
out["week_pct_used"] = int(m.group(1))
out["week_resets"] = m.group(2).strip() if m.group(2) else None
return out
def parse_agy_usage(obj: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse `agy -p /usage --output-format json` into one row per quota bucket."""
rows: List[Dict[str, Any]] = []
groups = (((obj or {}).get("command") or {}).get("data") or {}).get("groups") or []
for g in groups:
for b in g.get("buckets") or []:
frac = b.get("remaining_fraction")
if frac is None:
continue
window = b.get("window") if b.get("window") in ("weekly", "5h") else "other"
rows.append({"group": g.get("name", ""), "bucket": b.get("name", ""), "window": window,
"remaining_pct": round(float(frac) * 100.0, 3), "resets_at": b.get("reset_time", "")})
return rows
def reported_tokens(backend: str, obj: Dict[str, Any]) -> int:
"""Tokens the CLI itself reports for a headless call (used to assert probes cost 0)."""
u = (obj or {}).get("usage") or {}
if backend == "agy":
return int(u.get("total_tokens", 0) or 0)
if backend == "claude":
return int(u.get("input_tokens", 0) or 0) + int(u.get("output_tokens", 0) or 0)
return 0
def read_codex_latest(sessions_dir: Optional[Path] = None) -> Optional[Dict[str, Any]]:
"""Newest rollout that has a token_count event; returns window percents and session totals.
Scans newest-first because TUI-only sessions write rollouts with no token_count. Reads only
rollout-*.jsonl under sessions_dir; never opens auth files.
"""
sessions_dir = sessions_dir or Path.home() / ".codex" / "sessions"
files = sorted(sessions_dir.glob("**/rollout-*.jsonl"), reverse=True)
for f in files[:40]:
last = None
try:
with open(f, "r", encoding="utf-8") as fh:
for line in fh:
if '"token_count"' not in line:
continue
try:
rec = json.loads(line)
except ValueError:
continue
if (rec.get("payload") or {}).get("type") == "token_count":
last = rec
except OSError:
continue
if last is None:
continue
p = last["payload"]
rl = p.get("rate_limits") or {}
tot = ((p.get("info") or {}).get("total_token_usage")) or {}
return {
"five_hour_pct_used": (rl.get("primary") or {}).get("used_percent"),
"weekly_pct_used": (rl.get("secondary") or {}).get("used_percent"),
"five_hour_resets_at": (rl.get("primary") or {}).get("resets_at"),
"weekly_resets_at": (rl.get("secondary") or {}).get("resets_at"),
"session_total_tokens": tot.get("total_tokens"),
"source_timestamp": last.get("timestamp"),
"source_file": f.name,
}
return None
def _parse_ts(ts: str) -> Optional[datetime]:
try:
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
except (ValueError, AttributeError):
return None
def claude_tokens_between(project_dir: Path, start: str, end: str,
own_session: Optional[str] = None) -> Dict[str, Any]:
"""Sum assistant-message token usage in [start, end] across Claude transcripts.
concurrent=True when a transcript other than own_session wrote messages in the interval
(window-percent deltas are then shared with that activity and must be treated as noisy).
"""
t0, t1 = _parse_ts(start), _parse_ts(end)
tot = {"input_tokens": 0, "output_tokens": 0, "cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0, "cache_write_1h_tokens": 0}
active = set()
for f in project_dir.glob("*.jsonl"):
sid = f.stem
try:
with open(f, "r", encoding="utf-8") as fh:
for line in fh:
if '"usage"' not in line:
continue
try:
rec = json.loads(line)
except ValueError:
continue
ts = _parse_ts(rec.get("timestamp", ""))
usage = (rec.get("message") or {}).get("usage")
if not ts or not usage or not (t0 <= ts <= t1):
continue
active.add(sid)
if own_session is None or sid == own_session:
for k in tot:
tot[k] += int(usage.get(k, 0) or 0)
tot["cache_write_1h_tokens"] += int((usage.get("cache_creation") or {}).get("ephemeral_1h_input_tokens", 0) or 0)
except OSError:
continue
# Without a known own session, two or more active transcripts means shared activity.
others = active - {own_session} if own_session else (active if len(active) > 1 else set())
return {**tot, "concurrent": bool(others), "other_active_sessions": len(others)}
def parse_agy_json_usage(raw: str) -> Optional[Dict[str, int]]:
"""Token usage from `agy -p ... --output-format json` output; None for text output, failures or no usage block.
total_tokens excludes cache_read_tokens (it is input + output + thinking)."""
try:
d = json.loads(raw)
if d.get("status") != "SUCCESS":
return None
u = d["usage"]
return {"total_tokens": int(u["total_tokens"]), "input_tokens": int(u.get("input_tokens", 0)),
"output_tokens": int(u.get("output_tokens", 0)), "cache_read_tokens": int(u.get("cache_read_tokens", 0))}
except (ValueError, KeyError, TypeError, AttributeError):
return None
def codex_tokens_between(sessions_dir: Optional[Path], start: str, end: str) -> Dict[str, Any]:
"""Codex tokens used in [start, end]: per rollout, the cumulative totals at the end minus the totals at start.
Returns total_tokens plus the split input_tokens / cached_input_tokens / output_tokens (codex's input_tokens
includes the cached part). A session that began inside the interval counts from zero. concurrent=True when more
than one rollout recorded token usage in the interval (window percents are then shared and must be treated as noisy).
"""
sessions_dir = sessions_dir or Path.home() / ".codex" / "sessions"
t0, t1 = _parse_ts(start), _parse_ts(end)
keys = ("total_tokens", "input_tokens", "cached_input_tokens", "output_tokens")
sums = {k: 0 for k in keys}
active = 0
for f in sessions_dir.glob("**/rollout-*.jsonl"):
before = None
last_in = None
try:
with open(f, "r", encoding="utf-8") as fh:
for line in fh:
if '"token_count"' not in line:
continue
try:
rec = json.loads(line)
except ValueError:
continue
p = rec.get("payload") or {}
ts = _parse_ts(rec.get("timestamp", ""))
use = (p.get("info") or {}).get("total_token_usage") or {}
if p.get("type") != "token_count" or ts is None or use.get("total_tokens") is None:
continue
if ts < t0:
before = use
elif ts <= t1:
last_in = use
except OSError:
continue
if last_in is not None:
for k in keys:
sums[k] += max(0, (last_in.get(k) or 0) - ((before or {}).get(k) or 0))
active += 1
return {**sums, "concurrent": active > 1, "active_sessions": active}
def _run(argv: List[str], timeout: int = 60) -> str:
return subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=True).stdout
def snapshot(backends: List[str], runner: Callable[..., str] = _run, now: Optional[str] = None,
codex_dir: Optional[Path] = None) -> Dict[str, Any]:
"""Collect a usage snapshot. Every collector is zero-token; failures degrade to 'unavailable'."""
now = now or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
snap: Dict[str, Any] = {"timestamp": now, "backends": {}}
for b in backends:
try:
if b == "claude":
data = parse_claude_usage(runner(["claude", "-p", "/usage"]))
data["status"] = "ok" if data["session_pct_used"] is not None else "unparsed"
elif b == "agy":
raw = json.loads(runner(["agy", "-p", "/usage", "--output-format", "json"]))
tokens = reported_tokens("agy", raw)
rows = parse_agy_usage(raw)
data = {"status": "ok" if rows else "unparsed", "buckets": rows, "probe_tokens": tokens}
if tokens:
data["status"] = "probe_cost_nonzero"
elif b == "codex":
r = read_codex_latest(codex_dir)
data = {"status": "ok", **r} if r else {"status": "unavailable"}
else:
data = {"status": "unsupported"}
except Exception as exc: # noqa: BLE001 - probes must never crash the orchestrator
data = {"status": "unavailable", "error": type(exc).__name__}
snap["backends"][b] = data
assert_no_secrets(snap)
return snap
def diff(before: Dict[str, Any], after: Dict[str, Any]) -> Dict[str, Any]:
"""Window deltas between two snapshots, per backend."""
out: Dict[str, Any] = {"from": before["timestamp"], "to": after["timestamp"]}
bb, ab = before["backends"], after["backends"]
if "claude" in bb and "claude" in ab:
c0, c1 = bb["claude"], ab["claude"]
def d(k):
return None if c0.get(k) is None or c1.get(k) is None else c1[k] - c0[k]
out["claude"] = {"session_pct_delta": d("session_pct_used"), "week_pct_delta": d("week_pct_used")}
if "agy" in bb and "agy" in ab:
idx = {(r["group"], r["window"]): r for r in bb["agy"].get("buckets", [])}
rows = []
for r in ab["agy"].get("buckets", []):
prev = idx.get((r["group"], r["window"]))
if prev:
rows.append({"group": r["group"], "window": r["window"],
"remaining_pct_delta": round(r["remaining_pct"] - prev["remaining_pct"], 3)})
out["agy"] = rows
if "codex" in bb and "codex" in ab:
x0, x1 = bb["codex"], ab["codex"]
def dc(k):
return None if x0.get(k) is None or x1.get(k) is None else round(x1[k] - x0[k], 3)
out["codex"] = {"five_hour_pct_delta": dc("five_hour_pct_used"), "weekly_pct_delta": dc("weekly_pct_used"),
"stale": x0.get("source_timestamp") == x1.get("source_timestamp")}
return out
CATALOG_DIRS = [
Path.home() / ".claude" / "skills" / "cli-agent" / "references",
Path.home() / ".agents" / "skills" / "cli-agent" / "references",
Path.home() / ".gemini" / "antigravity-cli" / "skills" / "cli-agent" / "references",
]
def find_catalog_dir() -> Optional[Path]:
"""The installed cli-agent skill's model catalogs (single source of truth for prices and capabilities)."""
env = os.environ.get("AGENT_CLI_REFERENCES") or os.environ.get("SANCTUARY_CLI_AGENT_REFERENCES")
for d in ([Path(env)] if env else []) + CATALOG_DIRS:
if (d / "claude-models.json").exists():
return d
return None
def load_prices(catalog_dir: Optional[Path] = None) -> Dict[str, Any]:
"""Per-model prices (USD per million tokens) read from the cli-agent catalogs; nothing is duplicated here.
Keys are exact cli_ids from claude/codex/agy catalogs (direct API prices); Copilot entries are keyed
'copilot:<cli_id>' and carry credits_per_mtok. A model absent from the catalog is simply absent:
callers get None, never a price borrowed from a similarly named model.
"""
catalog_dir = catalog_dir or find_catalog_dir()
out: Dict[str, Any] = {}
if catalog_dir is None:
return out
for fname, prefix in (("claude-models.json", ""), ("codex-models.json", ""), ("agy-models.json", ""), ("copilot-models.json", "copilot:")):
try:
cat = json.loads((catalog_dir / fname).read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
meta = cat.get("_meta", {})
for m in cat.get("models", []):
pr = m.get("pricing_usd_per_1m") or {}
entry = {"input_per_mtok": pr.get("input"), "output_per_mtok": pr.get("output"),
"cached_input_per_mtok": pr.get("cached_input"), "cache_write_per_mtok": pr.get("cache_write"),
"cache_write_1h_per_mtok": pr.get("cache_write_1h"),
"credits_per_mtok": m.get("credits_per_1m"), "status": m.get("status"),
"catalog": fname, "catalog_updated": meta.get("updated")}
out.setdefault(prefix + m["cli_id"], entry)
return out
def cost_usd(model: str, tokens_in: int, tokens_out: int, prices: Optional[Dict[str, Any]] = None,
cache_read: int = 0, cache_write: int = 0, cache_write_1h: int = 0) -> Optional[float]:
"""Dollar cost for an EXACT catalog model id; None when absent or any needed price is null (never guessed).
tokens_in excludes cache reads/writes, which are priced at their own catalog rates: cache_write is the
5-minute TTL write, cache_write_1h the 1-hour TTL write (Claude Code uses the 1-hour TTL).
"""
prices = load_prices() if prices is None else prices
p = prices.get(model)
if not p or p.get("input_per_mtok") is None or p.get("output_per_mtok") is None:
return None
total = tokens_in / 1e6 * p["input_per_mtok"] + tokens_out / 1e6 * p["output_per_mtok"]
for tokens, key in ((cache_read, "cached_input_per_mtok"), (cache_write, "cache_write_per_mtok"),
(cache_write_1h, "cache_write_1h_per_mtok")):
if tokens:
if p.get(key) is None:
return None
total += tokens / 1e6 * p[key]
return round(total, 6)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |