hassan-mohiddin/freeflow / .deprecated/skill-tooling-pre-rewrite/skills/evaluate-skill
evaluate-skill
VerifiedUse when evaluating, comparing, or making readiness claims about agent-skill behavior.
scripts/lib/plan.mjs
View on GitHubimport { spawnSync } from "node:child_process";
import { access, lstat, readFile, realpath } from "node:fs/promises";
import { constants } from "node:fs";
import { dirname, relative, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
import { capabilitiesFor, supportedEvidenceClasses } from "./capabilities.mjs";
import { DEFAULT_OUTPUT_LIMIT_BYTES } from "./constants.mjs";
import { declaredResourceIdentity, gitResourceIdentity, hashDirectory, hashFile, sha256, stableJson } from "./hash.mjs";
import { compileCaseFeasibility, renderFeasibilityRows } from "./feasibility.mjs";
import { createManifest } from "./materialize.mjs";
import { inspectFixtureOracle } from "./fixture-oracle.mjs";
import { assertNoSymlinkTree, isWithin } from "./path-policy.mjs";
import { resolveInside } from "./workspace.mjs";
const ADAPTER_VERSIONS = {
deterministic: "deterministic-v1",
json: "pi-outcome-v1",
"rpc-scripted": "pi-rpc-scripted-v1",
exec: "codex-exec-diagnostic-v1",
};
const LEGACY_MODEL_OPTION_KEYS = ["provider", "model", "thinking"];
const SUBJECT_MODEL_OPTION_KEYS = ["subject_provider", "subject_model", "subject_thinking"];
const GRADER_MODEL_OPTION_KEYS = ["grader_provider", "grader_model", "grader_thinking"];
const ALL_MODEL_OPTION_KEYS = [
...LEGACY_MODEL_OPTION_KEYS,
...SUBJECT_MODEL_OPTION_KEYS,
...GRADER_MODEL_OPTION_KEYS,
"max_turns_per_process",
"max_usd",
];
const scriptsRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const EVALUATOR_SOURCE_FILES = [
"skill-eval.mjs",
"pi-composition-runtime.mjs",
"pi-root-guard.mjs",
"lib/args.mjs",
"lib/capabilities.mjs",
"lib/compact-evidence.mjs",
"lib/coordinator.mjs",
"lib/codex-adapter.mjs",
"lib/decision.mjs",
"lib/evaluate.mjs",
"lib/feasibility.mjs",
"lib/fixture-oracle.mjs",
"lib/grade.mjs",
"lib/outcome.mjs",
"lib/hash.mjs",
"lib/integrity.mjs",
"lib/materialize.mjs",
"lib/path-policy.mjs",
"lib/pi-adapter.mjs",
"lib/plan.mjs",
"lib/process-outcome.mjs",
"lib/process.mjs",
"lib/rpc-client.mjs",
"lib/semantic-evidence.mjs",
"lib/publication.mjs",
"lib/workspace.mjs",
];
const SEMANTIC_SOURCE_FILES = [
"pi-root-guard.mjs",
"lib/compact-evidence.mjs",
"lib/outcome.mjs",
"lib/pi-adapter.mjs",
"lib/process-outcome.mjs",
"lib/process.mjs",
"lib/semantic-evidence.mjs",
"lib/semantic.mjs",
];
async function sourceIdentity(files) {
const entries = [];
for (const file of files) entries.push([file, await hashFile(resolve(scriptsRoot, file))]);
return sha256(stableJson(entries));
}
function requirePositiveInteger(value, label) {
if (!Number.isInteger(Number(value)) || Number(value) < 1) throw new Error(`${label} must be a positive integer`);
return Number(value);
}
function selectCase(workspace, caseId) {
if (typeof caseId !== "string" || caseId.length === 0) throw new Error("evaluate requires --case <id>");
const match = workspace.cases.find((item) => item.id === caseId);
if (!match) throw new Error(`Unknown case for ${workspace.suite.skill}: ${caseId}`);
return match;
}
async function validateWorkingTreeResources(repoRoot, variant) {
const subjectRoot = resolveInside(repoRoot, variant.path, `${variant.id}.path`);
const canonicalRoot = await realpath(subjectRoot);
for (const resource of variant.resources) {
const path = resolveInside(subjectRoot, resource, `${variant.id}.resource`);
await access(path, constants.R_OK).catch(() => {
throw new Error(`Missing subject resource: ${variant.path}/${resource}`);
});
const info = await lstat(path);
if (info.isSymbolicLink()) throw new Error(`Subject resource cannot be a symlink: ${variant.path}/${resource}`);
if (!isWithin(canonicalRoot, await realpath(path)))
throw new Error(`Subject resource escapes through symlink: ${variant.path}/${resource}`);
await assertNoSymlinkTree(path, `Subject resource ${variant.path}/${resource}`);
}
}
function validateGitResources(repoRoot, variant) {
const root = variant.path.replace(/\/$/, "") === "." ? "" : variant.path.replace(/\/$/, "");
const variantPrefix = root ? `${root}/` : "";
for (const resource of variant.resources) {
const fullResource = root ? `${root}/${resource}` : resource;
const result = spawnSync("git", ["ls-tree", "-r", "-z", "--full-tree", variant.revision, "--", fullResource], {
cwd: repoRoot,
encoding: "utf8",
});
if (result.status !== 0)
throw new Error(
`Unable to inspect git subject resource ${variant.revision}:${fullResource}: ${result.stderr.trim()}`,
);
const records = result.stdout.split("\0").filter(Boolean);
if (records.length === 0) throw new Error(`Missing git subject resource: ${variant.revision}:${fullResource}`);
for (const record of records) {
const [header, fullPath] = record.split("\t");
const [mode, type] = header.split(" ");
if (type !== "blob" || mode === "120000")
throw new Error(`Unsafe git subject resource: ${mode} ${type} ${fullPath}`);
if (
(variantPrefix && !fullPath.startsWith(variantPrefix)) ||
(fullPath !== fullResource && !fullPath.startsWith(`${fullResource}/`))
) {
throw new Error(`Git subject resource escapes declaration: ${fullPath}`);
}
}
}
}
async function resolveDeclaredSource(repoRoot, source, label, resources = source.resources) {
const descriptor = { ...source, id: source.id ?? source.name ?? label, resources };
const absolutePath = resolveInside(repoRoot, descriptor.path, `${label}.path`);
if (descriptor.kind === "working-tree") {
await validateWorkingTreeResources(repoRoot, descriptor);
const identity = await declaredResourceIdentity(absolutePath, resources);
return {
...descriptor,
absolute_path: absolutePath,
resource_identity: identity,
snapshot_hash: identity.aggregate_sha256,
};
}
if (descriptor.kind === "git") {
validateGitResources(repoRoot, descriptor);
const identity = gitResourceIdentity(repoRoot, descriptor.revision, descriptor.path, resources);
return {
...descriptor,
absolute_path: absolutePath,
resource_identity: identity,
snapshot_hash: identity.aggregate_sha256,
};
}
throw new Error(`Unsupported declared source kind: ${descriptor.kind}`);
}
async function resolveVariant(repoRoot, variant) {
return resolveDeclaredSource(repoRoot, variant, variant.id);
}
async function declaredSkillName(repoRoot, source) {
let text;
if (source.kind === "working-tree") text = await readFile(resolve(source.absolute_path, "SKILL.md"), "utf8");
else {
const root = source.path.replace(/\/$/, "") === "." ? "" : source.path.replace(/\/$/, "");
const path = root ? `${root}/SKILL.md` : "SKILL.md";
const result = spawnSync("git", ["show", `${source.revision}:${path}`], {
cwd: repoRoot,
encoding: "utf8",
maxBuffer: 1024 * 1024,
});
if (result.status !== 0) throw new Error(`Unable to read composition skill name: ${source.revision}:${path}`);
text = result.stdout;
}
const match = text.match(/^---\n[\s\S]*?^name:\s*([^\n]+)$/m);
if (!match) throw new Error(`Composition skill is missing frontmatter name: ${source.path}`);
return match[1].trim().replace(/^['"]|['"]$/g, "");
}
function fileHash(identity, path, label) {
const match = identity.entries.find((entry) => entry[0] === path && entry[1] === "file");
if (!match) throw new Error(`Missing ${label} identity: ${path}`);
return match[2];
}
async function resolveComposition(repoRoot, evalCase, variants) {
if (evalCase.composition === undefined) return null;
const baseStack = [];
for (const component of evalCase.composition.base_stack) {
const resolved = await resolveDeclaredSource(repoRoot, component, `composition.${component.name}`);
const actualName = await declaredSkillName(repoRoot, resolved);
if (actualName !== component.name)
throw new Error(`Composition component name mismatch: declared ${component.name}, found ${actualName}`);
baseStack.push(resolved);
}
let runtime = null;
if (evalCase.composition.runtime) {
const source = evalCase.composition.runtime;
const resources = [source.interaction_contract, source.workflow];
const resolved = await resolveDeclaredSource(repoRoot, source, "composition.runtime", resources);
const extensionPath = resolve(scriptsRoot, "pi-composition-runtime.mjs");
const evaluatorRoot = resolve(scriptsRoot, "..", "..", "..");
const productionHelperPath = resolve(evaluatorRoot, "pi-extension", "dist", "runtime-context.js");
runtime = {
...resolved,
profile: source.profile,
extension_path: extensionPath,
interaction_contract_identity: {
path: source.interaction_contract,
sha256: fileHash(resolved.resource_identity, source.interaction_contract, "Interaction Contract"),
},
workflow_identity: {
path: source.workflow,
sha256: fileHash(resolved.resource_identity, source.workflow, "runtime workflow"),
},
implementation_identity: {
evaluator_extension: {
path: relative(evaluatorRoot, extensionPath).split(sep).join("/"),
absolute_path: extensionPath,
sha256: await hashFile(extensionPath),
},
production_helper: {
path: relative(evaluatorRoot, productionHelperPath).split(sep).join("/"),
absolute_path: productionHelperPath,
sha256: await hashFile(productionHelperPath),
},
},
};
}
for (const variant of variants) {
const actualName = await declaredSkillName(repoRoot, variant);
if (actualName !== evalCase.composition.target_name)
throw new Error(
`Composition target name mismatch for ${variant.role}: declared ${evalCase.composition.target_name}, found ${actualName}`,
);
}
return {
target_name: evalCase.composition.target_name,
base_stack: baseStack,
runtime,
target_variants: Object.fromEntries(variants.map((variant) => [variant.role, variant])),
};
}
function compositionIdentity(composition) {
if (!composition) return null;
const source = (component) => ({
name: component.name ?? null,
kind: component.kind,
path: component.path,
revision: component.revision ?? null,
resources: component.resources,
identity: component.resource_identity,
});
return {
target_name: composition.target_name,
base_stack: composition.base_stack.map(source),
runtime: composition.runtime
? {
profile: composition.runtime.profile,
kind: composition.runtime.kind,
path: composition.runtime.path,
revision: composition.runtime.revision ?? null,
identity: composition.runtime.resource_identity,
interaction_contract: composition.runtime.interaction_contract_identity,
workflow: composition.runtime.workflow_identity,
implementation: Object.fromEntries(
Object.entries(composition.runtime.implementation_identity).map(([name, identity]) => [
name,
{ path: identity.path, sha256: identity.sha256 },
]),
),
}
: null,
targets: Object.fromEntries(
Object.entries(composition.target_variants).map(([role, variant]) => [
role,
source({ ...variant, name: composition.target_name }),
]),
),
};
}
function evidenceResolution(evalCase, selectedHost, effectiveMode) {
const supported = supportedEvidenceClasses(selectedHost, effectiveMode);
const state = (values = []) =>
Object.fromEntries(values.map((name) => [name, supported.has(name) ? "supported" : "unsupported"]));
const required = state(evalCase.evidence_classes);
const requested = state(evalCase.requested_evidence_classes);
return {
required,
requested,
unsupported_required: Object.entries(required)
.filter(([, value]) => value === "unsupported")
.map(([name]) => name),
unsupported_requested: Object.entries(requested)
.filter(([, value]) => value === "unsupported")
.map(([name]) => name),
};
}
function rerunCommand(summary) {
const args = [
"node skills/evaluate-skill/scripts/skill-eval.mjs evaluate",
`--skill ${summary.skill}`,
`--case ${summary.case}`,
`--timeout-ms ${summary.limits.timeout_ms}`,
`--output-limit-bytes ${summary.limits.output_limit_bytes}`,
];
if (summary.model) {
args.push(
`--provider ${summary.model.provider}`,
`--model ${summary.model.model}`,
`--thinking ${summary.model.thinking}`,
);
} else if (summary.subject_model) {
args.push(`--host ${summary.subject_host}`);
args.push(
`--subject-provider ${summary.subject_model.provider}`,
`--subject-model ${summary.subject_model.model}`,
`--subject-thinking ${summary.subject_model.thinking}`,
);
if (summary.grader_model)
args.push(
`--grader-provider ${summary.grader_model.provider}`,
`--grader-model ${summary.grader_model.model}`,
`--grader-thinking ${summary.grader_model.thinking}`,
);
}
if (summary.model || summary.subject_model) {
args.push(`--max-turns-per-process ${summary.limits.max_turns_per_process}`);
if (summary.spend.max_usd !== null) args.push(`--max-usd ${summary.spend.max_usd}`);
args.push("--owner-approved", `--expect-plan ${summary.fingerprint}`);
}
return args.join(" ");
}
export async function buildEvaluationPlan(workspace, options, dependencies = {}) {
const evalCase = selectCase(workspace, options.case);
const timeoutMs = requirePositiveInteger(options.timeout_ms, "--timeout-ms");
const outputLimitBytes = requirePositiveInteger(options.output_limit_bytes, "--output-limit-bytes");
const portable = evalCase.execution.host === "portable";
if (portable) {
if (typeof options.host !== "string") throw new Error("Portable evaluation requires --host pi|codex");
if (!evalCase.execution.allowed_hosts.includes(options.host))
throw new Error(`Selected host is not allowed by case: ${options.host}`);
} else if (options.host !== undefined) {
throw new Error("Fixed-host cases reject --host");
}
const selectedHost = portable ? options.host : evalCase.execution.host;
const effectiveMode = portable ? (selectedHost === "pi" ? "json" : "exec") : evalCase.execution.mode;
const modelDriven = selectedHost !== "none";
const hasSemantic = evalCase.assertions.some((assertion) => assertion.type === "semantic");
if (!modelDriven && hasSemantic) throw new Error("Host-free cases cannot require semantic Pi grading");
const legacySupplied = LEGACY_MODEL_OPTION_KEYS.filter((key) => options[key] !== undefined);
const subjectSupplied = SUBJECT_MODEL_OPTION_KEYS.filter((key) => options[key] !== undefined);
const graderSupplied = GRADER_MODEL_OPTION_KEYS.filter((key) => options[key] !== undefined);
let subjectModel = null;
let graderModel = null;
if (!modelDriven) {
const supplied = ALL_MODEL_OPTION_KEYS.filter((key) => options[key] !== undefined);
if (supplied.length > 0)
throw new Error(
`Host-free case rejects model options: ${supplied.map((key) => `--${key.replaceAll("_", "-")}`).join(", ")}`,
);
} else if (!portable) {
if (subjectSupplied.length > 0 || graderSupplied.length > 0)
throw new Error("Fixed Pi cases reject role-qualified model options");
const missing = LEGACY_MODEL_OPTION_KEYS.filter((key) => options[key] === undefined);
if (missing.length > 0)
throw new Error(
`Model-driven evaluation requires ${missing.map((key) => `--${key.replaceAll("_", "-")}`).join(", ")}`,
);
subjectModel = { provider: options.provider, model: options.model, thinking: options.thinking };
graderModel = hasSemantic ? { ...subjectModel } : null;
} else {
if (legacySupplied.length > 0) throw new Error("Portable cases reject legacy or mixed model options");
const missingSubject = SUBJECT_MODEL_OPTION_KEYS.filter((key) => options[key] === undefined);
if (missingSubject.length > 0)
throw new Error(
`Portable evaluation requires ${missingSubject.map((key) => `--${key.replaceAll("_", "-")}`).join(", ")}`,
);
if (hasSemantic) {
const missingGrader = GRADER_MODEL_OPTION_KEYS.filter((key) => options[key] === undefined);
if (missingGrader.length > 0)
throw new Error(
`Semantic portable evaluation requires ${missingGrader.map((key) => `--${key.replaceAll("_", "-")}`).join(", ")}`,
);
} else if (graderSupplied.length > 0) {
throw new Error("Grader options require semantic assertions");
}
subjectModel = {
provider: options.subject_provider,
model: options.subject_model,
thinking: options.subject_thinking,
};
graderModel = hasSemantic
? { provider: options.grader_provider, model: options.grader_model, thinking: options.grader_thinking }
: null;
if (selectedHost === "codex" && subjectModel.provider !== "openai")
throw new Error("Codex --subject-provider must be openai");
}
if (modelDriven && options.max_turns_per_process === undefined)
throw new Error("Model-driven evaluation requires --max-turns-per-process");
const maxTurns = modelDriven ? requirePositiveInteger(options.max_turns_per_process, "--max-turns-per-process") : 0;
if (options.max_usd !== undefined && (!Number.isFinite(Number(options.max_usd)) || Number(options.max_usd) <= 0))
throw new Error("--max-usd must be a positive number");
const fixturePath =
evalCase.fixture === null ? null : resolveInside(workspace.skillRoot, evalCase.fixture, `${evalCase.id}.fixture`);
const fixtureManifest = fixturePath ? await createManifest(fixturePath) : { files: {} };
const fixtureFiles = Object.keys(fixtureManifest.files ?? {}).sort();
const fixtureBytes = Object.values(fixtureManifest.files ?? {}).reduce(
(sum, file) => sum + Number(file.size ?? 0),
0,
);
const semanticInputBytes = Buffer.byteLength(
JSON.stringify({
prompts: Object.hasOwn(evalCase, "prompt") ? [evalCase.prompt] : evalCase.turns.map((turn) => turn.prompt),
criteria: evalCase.assertions
.filter((assertion) => assertion.type === "semantic")
.map((assertion) => assertion.rubric),
}),
);
const estimatedCompactBytes = semanticInputBytes + Math.min(fixtureBytes, 30000) + 4096;
const transportLimitBytes = modelDriven ? Math.max(outputLimitBytes, DEFAULT_OUTPUT_LIMIT_BYTES) : 0;
const feasibility = await compileCaseFeasibility(evalCase, {
modelDriven,
maxTurns,
outputLimitBytes,
transportLimitBytes,
estimatedCompactBytes,
estimatedTransportBytes: estimatedCompactBytes * 2,
fixtureFiles,
readFixture: async (path) => {
if (!fixturePath) return null;
try {
return await readFile(resolveInside(fixturePath, path, "feasibility fixture source"), "utf8");
} catch (error) {
if (error.code === "ENOENT") return null;
throw error;
}
},
runOracle: async (declaration) => inspectFixtureOracle(fixturePath, declaration),
});
if (feasibility.blocking) {
const blockedReasons = feasibility.findings.map((item) => item.id);
return {
status: "blocked",
summary: {
skill: workspace.suite.skill,
case: evalCase.id,
feasibility: { ...feasibility, rows: renderFeasibilityRows(feasibility) },
blocked_reasons: blockedReasons,
},
fingerprint: null,
eval_case: evalCase,
variants: [],
composition: null,
fixture_path: fixturePath,
plan_inputs: null,
blocked_evidence: [],
missing_capabilities: [],
limit_blocks: blockedReasons,
};
}
const resolveCapabilities = dependencies.capabilitiesFor ?? capabilitiesFor;
const capabilityMode = evalCase.composition !== undefined && selectedHost === "pi" ? "rpc-scripted" : effectiveMode;
const host = await resolveCapabilities(selectedHost, capabilityMode);
const baseRequiredCapabilities =
selectedHost === "codex"
? [
"exec_jsonl",
"isolated_home",
"strict_config",
"ephemeral",
"ignore_rules",
"ambient_context_disabled",
"explicit_skill",
"strict_filesystem_isolation",
"network_disabled",
"process_limits",
"provider_request_bound",
"spend_bound",
]
: effectiveMode === "rpc-scripted"
? [
"rpc_jsonl",
"multi_turn",
"native_skill_loading",
"explicit_extensions",
"disable_extension_discovery",
"disable_context_files",
"tool_allowlist",
"strict_tool_isolation",
]
: [
"one_shot_json",
"native_skill_loading",
"explicit_extensions",
"disable_extension_discovery",
"disable_context_files",
"tool_allowlist",
"strict_tool_isolation",
];
const compositionCapabilities =
evalCase.composition === undefined
? []
: ["multi_skill_loading", "explicit_runtime_context", "composition_activation_evidence"];
const requiredCapabilities = [...new Set([...baseRequiredCapabilities, ...compositionCapabilities])];
const missingCapabilities = modelDriven ? requiredCapabilities.filter((name) => !host.capabilities?.[name]) : [];
if (modelDriven && !host.available) missingCapabilities.unshift(`${selectedHost}-available`);
const evidence = evidenceResolution(evalCase, selectedHost, effectiveMode);
const blockedEvidence = [
...evidence.unsupported_required,
...(evalCase.unsupported_evidence === "block" ? evidence.unsupported_requested : []),
];
const fixtureHash = fixturePath ? await hashDirectory(fixturePath) : sha256("empty-fixture");
const variants = [];
for (const variant of evalCase.variants) variants.push(await resolveVariant(workspace.repoRoot, variant));
const composition = await resolveComposition(workspace.repoRoot, evalCase, variants);
const limitBlocks =
effectiveMode === "rpc-scripted" && maxTurns < evalCase.turns.length ? ["max-turns-below-scripted-user-turns"] : [];
const semanticPerVariant = hasSemantic ? 1 : 0;
const subjectProcesses = modelDriven ? variants.length : 0;
const semanticProcesses = modelDriven ? variants.length * semanticPerVariant : 0;
const piSubjectProcesses = selectedHost === "pi" ? subjectProcesses : 0;
const codexSubjectProcesses = selectedHost === "codex" ? subjectProcesses : 0;
const totalProcesses = subjectProcesses + semanticProcesses;
const limitations = !modelDriven
? []
: selectedHost === "codex"
? [
"Codex support is diagnostic only; public execution is blocked before auth access and model startup.",
"Codex provider-request count and monetary cost are unavailable, not zero.",
"The recorded isolation profile is codex-diagnostic-macos-v1 and is not claimed equivalent to Pi's tool allowlist.",
]
: [
"Provider requests are observed and reported; bootstrap does not claim an independent global provider-request hard cap.",
"The output limit applies to retained canonical evidence after cumulative Pi JSON updates are compacted; raw transport has a separate internal safeguard.",
];
if (effectiveMode === "rpc-scripted") {
limitations.push(
"Turn, timeout, retained-output, and raw-transport limits apply across each complete RPC process; scripted user turns are not separate process budgets.",
);
}
if (evidence.unsupported_requested.length > 0 && evalCase.unsupported_evidence === "behavior-under-test") {
limitations.push(`Unsupported evidence is behavior under test: ${evidence.unsupported_requested.join(", ")}.`);
}
if (options.max_usd === undefined && modelDriven && selectedHost === "pi")
limitations.push("Cost is reported when available; no aggregate spend ceiling was supplied.");
if (options.max_usd !== undefined && modelDriven && selectedHost === "pi")
limitations.push(
effectiveMode === "rpc-scripted"
? "The spend ceiling is checked between Pi processes and before later scripted prompts when the host reports cost; unavailable cost remains unavailable."
: "The spend ceiling is checked between Pi processes only when the host reports cost; unavailable cost remains unavailable.",
);
const { source_path: _sourcePath, ...caseContent } = evalCase;
const caseSourcePath = relative(workspace.repoRoot, evalCase.source_path).split(sep).join("/");
const identities = {
case: sha256(stableJson(caseContent)),
case_source: { path: caseSourcePath, sha256: await hashFile(evalCase.source_path) },
fixture: fixtureHash,
subjects: Object.fromEntries(variants.map((variant) => [variant.role, variant.snapshot_hash])),
composition: compositionIdentity(composition),
evaluator: await sourceIdentity(EVALUATOR_SOURCE_FILES),
semantic: await sourceIdentity(SEMANTIC_SOURCE_FILES),
};
const planInputs = {
schema_version: 1,
adapter_version: ADAPTER_VERSIONS[effectiveMode] ?? `unsupported-${effectiveMode}`,
skill: workspace.suite.skill,
case: caseContent,
identities,
variants: variants.map((variant) => ({
id: variant.id,
role: variant.role,
kind: variant.kind,
revision: variant.revision ?? null,
path: variant.path,
resources: variant.resources,
snapshot_hash: variant.snapshot_hash,
resource_identity: variant.resource_identity,
})),
composition: identities.composition,
host,
subject_host: selectedHost,
effective_mode: effectiveMode,
model: !portable && modelDriven ? subjectModel : null,
subject_model: subjectModel,
grader_model: graderModel,
isolation_profile: selectedHost === "codex" ? "codex-diagnostic-macos-v1" : null,
limits: {
timeout_ms: timeoutMs,
output_limit_bytes: outputLimitBytes,
transport_limit_bytes: transportLimitBytes,
max_turns_per_process: maxTurns,
},
max_usd: options.max_usd === undefined ? null : Number(options.max_usd),
evidence,
feasibility,
};
const fingerprint = sha256(stableJson(planInputs));
const summary = {
skill: workspace.suite.skill,
case: evalCase.id,
evaluation_kind: evalCase.evaluation_kind,
variants: variants.map(({ id, role }) => ({ id, role })),
model: planInputs.model,
subject_model: planInputs.subject_model,
grader_model: planInputs.grader_model,
subject_host: selectedHost,
fidelity: selectedHost === "codex" ? "diagnostic" : "accepted",
pi_processes: {
subject: piSubjectProcesses,
semantic_max: semanticProcesses,
total_max: piSubjectProcesses + semanticProcesses,
},
codex_processes: { subject: codexSubjectProcesses, total_max: codexSubjectProcesses },
total_processes: totalProcesses,
scripted_user_turns: effectiveMode === "rpc-scripted" ? evalCase.turns.length : null,
composition: composition
? {
skills: [...composition.base_stack.map((component) => component.name), composition.target_name],
target_name: composition.target_name,
runtime_profile: composition.runtime?.profile ?? null,
}
: null,
limits: planInputs.limits,
worst_case_approved_turns: selectedHost === "codex" ? null : totalProcesses * maxTurns,
spend: { max_usd: planInputs.max_usd },
evidence: { required: evidence.required, requested: evidence.requested },
...(evalCase.feasibility ? { feasibility: { ...feasibility, rows: renderFeasibilityRows(feasibility) } } : {}),
identities,
capabilities: { host: host.id, version: host.version, missing: missingCapabilities },
blocked_reasons: [...new Set([...blockedEvidence, ...missingCapabilities, ...limitBlocks])],
limitations,
fingerprint,
};
summary.rerun_command = selectedHost === "codex" ? null : rerunCommand(summary);
let status = "ready";
if (blockedEvidence.length > 0 || missingCapabilities.length > 0 || limitBlocks.length > 0) status = "blocked";
else if (options.plan_only) status = "planned";
else if (modelDriven && (!options.owner_approved || (options.expect_plan && options.expect_plan !== fingerprint)))
status = "needs_approval";
return {
status,
summary,
fingerprint,
eval_case: evalCase,
variants,
composition,
fixture_path: fixturePath,
plan_inputs: planInputs,
blocked_evidence: blockedEvidence,
missing_capabilities: missingCapabilities,
limit_blocks: limitBlocks,
};
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |