TheGoat395/Codex-Skills / skills/agent-evaluation-operations
agent-evaluation-operations
VerifiedEvaluate agent and skill behavior or routing.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 5.7 KB |
| agents/openai.yaml | config | 256 B |
| references/fixture-execution.md | markdown | 2.0 KB |
| references/fixtures/draft-send.json | config | 351 B |
| references/regression-corpus.json | config | 19.9 KB |
| references/skill-behavior-evaluation.md | markdown | 3.1 KB |
| scripts/analyze_skill_collisions.pyexecutable | script | 5.0 KB |
| scripts/observe_draft_fixture.py | script | 1.1 KB |
| scripts/test_observe_draft_fixture.py | script | 841 B |
| scripts/validate_regression_corpus.pyexecutable | script | 4.6 KB |
scripts/validate_regression_corpus.py
View on GitHub#!/usr/bin/env python3
"""Validate the bundled behavioral regression corpus without external packages."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
REQUIRED_LIST_FIELDS = (
"allowed_resources",
"expected_behaviors",
"prohibited_behaviors",
"evidence",
"failure_evidence",
"side_effects_allowed",
"applicable_skills",
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"corpus",
nargs="?",
type=Path,
default=Path(__file__).resolve().parents[1] / "references" / "regression-corpus.json",
)
parser.add_argument("--expected-count", type=int, default=None)
args = parser.parse_args()
try:
data = json.loads(args.corpus.read_text(encoding="utf-8"))
if not isinstance(data, dict):
raise ValueError("corpus root must be an object")
except (OSError, ValueError) as exc:
print(f"ERROR: invalid corpus: {exc}", file=sys.stderr)
return 2
expected = args.expected_count if args.expected_count is not None else data.get("expected_case_count")
if data.get("schema_version") != "1.1":
errors = ["schema_version must be 1.1"]
else:
errors = []
scorer_contracts = data.get("scorer_contracts")
if not isinstance(scorer_contracts, dict) or not scorer_contracts:
errors.append("scorer_contracts must be a non-empty object")
scorer_contracts = {}
for name, contract in scorer_contracts.items():
if not isinstance(name, str) or not name.strip() or not isinstance(contract, dict):
errors.append("each scorer contract needs a name and object")
continue
for field in ("deterministic_checks", "rubric_checks", "critical_failure"):
value = contract.get(field)
if not isinstance(value, str) or not value.strip():
errors.append(f"scorer {name}: {field} must be a non-empty string")
cases = data.get("cases")
if not isinstance(cases, list):
errors.append("top-level cases must be a list")
cases = []
if type(expected) is not int or expected < 1:
errors.append("positive expected_case_count or --expected-count is required")
elif len(cases) != expected:
errors.append(f"expected {expected} cases, found {len(cases)}")
seen: set[str] = set()
for index, case in enumerate(cases, start=1):
prefix = f"case {index}"
if not isinstance(case, dict):
errors.append(f"{prefix}: must be an object")
continue
case_id = case.get("id")
if not isinstance(case_id, str) or not case_id.strip():
errors.append(f"{prefix}: missing id")
elif case_id in seen:
errors.append(f"{prefix}: duplicate id {case_id}")
else:
seen.add(case_id)
for field in ("title", "category", "prompt"):
if not isinstance(case.get(field), str) or not case[field].strip():
errors.append(f"{prefix}: {field} must be a non-empty string")
if not isinstance(case.get("scorer"), str) or not case["scorer"].strip():
errors.append(f"{prefix}: scorer must be a non-empty string")
elif case["scorer"] not in scorer_contracts:
errors.append(f"{prefix}: scorer has no matching scorer_contract")
threshold = case.get("acceptance_threshold")
if not isinstance(threshold, dict):
errors.append(f"{prefix}: acceptance_threshold must be an object")
else:
if threshold.get("required_behaviors") != "all":
errors.append(f"{prefix}: required_behaviors must be all")
if threshold.get("required_evidence") != "all":
errors.append(f"{prefix}: required_evidence must be all")
if type(threshold.get("prohibited_behaviors_allowed")) is not int or threshold["prohibited_behaviors_allowed"] != 0:
errors.append(f"{prefix}: prohibited_behaviors_allowed must be 0")
for field in REQUIRED_LIST_FIELDS:
value = case.get(field)
if not isinstance(value, list) or not value or not all(isinstance(item, str) and item.strip() for item in value):
errors.append(f"{prefix}: {field} must be a non-empty string list")
if errors:
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
return 1
print(f"PASS structure only: {len(cases)} unique regression cases in {args.corpus}; no model behavior executed")
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |