hassan-mohiddin/freeflow / .deprecated/skill-tooling-pre-rewrite/skills/evaluate-skill
evaluate-skill
VerifiedUse when evaluating, comparing, or making readiness claims about agent-skill behavior.
scripts/lib/feasibility.mjs
View on GitHubimport { sha256 } from "./hash.mjs";
function finding(id, sourceSpan, evidence, blockingReason) {
return { id, severity: "blocking", source_span: sourceSpan, evidence, blocking_reason: blockingReason };
}
function subjectPrompts(evalCase) {
return Object.hasOwn(evalCase, "prompt") ? [evalCase.prompt] : (evalCase.turns ?? []).map((turn) => turn.prompt);
}
function assertedWrites(evalCase) {
const paths = new Set();
for (const assertion of evalCase.assertions ?? []) {
if (assertion.type === "changed_paths") for (const path of assertion.equals ?? []) paths.add(path);
}
return [...paths].sort();
}
export async function compileCaseFeasibility(evalCase, context) {
const findings = [];
const prompts = subjectPrompts(evalCase);
const promptText = prompts.join("\n");
const tools = new Set(evalCase.execution?.tools ?? []);
const declaration = evalCase.feasibility ?? {};
const missingEvidence = (declaration.required_evidence_paths ?? []).filter(
(path) => !(context.fixtureFiles ?? []).includes(path),
);
if (missingEvidence.length > 0)
findings.push(
finding(
"FEAS-EVIDENCE-MISSING",
"case.feasibility.required_evidence_paths",
{ missing_paths: missingEvidence, fixture_files: context.fixtureFiles ?? [] },
"Required rubric evidence is absent from the fixture.",
),
);
const unnamedEvidence = (declaration.required_evidence_paths ?? []).filter((path) => !promptText.includes(path));
if (unnamedEvidence.length > 0 && !tools.has("ls") && !tools.has("find")) {
findings.push(
finding(
"FEAS-EVIDENCE-DISCOVERY",
"case.feasibility.required_evidence_paths",
{ unnamed_paths: unnamedEvidence, tools: [...tools] },
"Required evidence is unnamed and the subject cannot list fixture contents.",
),
);
}
const requiredReads = [
...new Set([
...(declaration.required_evidence_paths ?? []),
...(declaration.literal_requirements ?? [])
.map((item) => (item.source?.startsWith("fixture:") ? item.source.slice("fixture:".length) : null))
.filter(Boolean),
]),
].sort();
if (context.modelDriven !== false && requiredReads.length > 0 && !tools.has("read")) {
findings.push(
finding(
"FEAS-EVIDENCE-READ",
"case.execution.tools",
{ required_paths: requiredReads, tools: [...tools] },
"Declared fixture evidence exists but the subject has no read tool.",
),
);
}
for (let index = 0; index < (declaration.literal_requirements ?? []).length; index += 1) {
const requirement = declaration.literal_requirements[index];
let source = null;
if (requirement.source === "prompt") source = promptText;
else if (typeof requirement.source === "string" && requirement.source.startsWith("fixture:"))
source = await context.readFixture(requirement.source.slice("fixture:".length));
if (
requirement.equivalence_class &&
!(declaration.accepted_equivalences ?? []).includes(requirement.equivalence_class)
) {
findings.push(
finding(
"FEAS-EQUIVALENCE",
`case.feasibility.literal_requirements[${index}].equivalence_class`,
{ equivalence_class: requirement.equivalence_class, accepted: declaration.accepted_equivalences ?? [] },
"A semantic/setup equivalence is required but not declared as accepted.",
),
);
}
if (typeof source !== "string" || !source.includes(requirement.value)) {
findings.push(
finding(
"FEAS-LITERAL-SOURCE",
`case.feasibility.literal_requirements[${index}]`,
{
value: requirement.value,
source: requirement.source,
source_sha256: typeof source === "string" ? sha256(source) : null,
source_bytes: typeof source === "string" ? Buffer.byteLength(source) : null,
matched: false,
},
"An exact literal requirement is not supported by its declared source.",
),
);
}
}
const changedByScope = new Map();
for (const assertion of (evalCase.assertions ?? []).filter((item) => item.type === "changed_paths")) {
const scope = assertion.turn_id ?? "final";
const expectation = JSON.stringify([...(assertion.equals ?? [])].sort());
const values = changedByScope.get(scope) ?? [];
values.push(expectation);
changedByScope.set(scope, values);
}
for (const [scope, values] of changedByScope) {
if (new Set(values).size > 1)
findings.push(
finding(
"FEAS-CHANGED-PATH-CONFLICT",
"case.assertions",
{ scope, changed_path_expectations: values.map((value) => JSON.parse(value)) },
"Changed-path assertions define conflicting output contracts for one evidence scope.",
),
);
}
const writes = assertedWrites(evalCase);
if (context.modelDriven !== false && writes.length > 0 && !tools.has("write")) {
findings.push(
finding(
"FEAS-OUTPUT-TOOL",
"case.execution.tools",
{ asserted_writes: writes, tools: [...tools] },
"Assertions require written output but the subject has no write tool.",
),
);
}
const active = new Set(declaration.active_context_components ?? []);
for (const assertion of evalCase.assertions ?? []) {
if (
assertion.turn_id &&
["component_read", "skill_read"].includes(assertion.type) &&
((assertion.type === "skill_read" && declaration.active_context_skill === true) ||
active.has(assertion.component))
) {
const turnIndex = (evalCase.turns ?? []).findIndex((turn) => turn.id === assertion.turn_id);
if (turnIndex > 0)
findings.push(
finding(
"FEAS-REDUNDANT-REREAD",
`case.assertions.${assertion.id}`,
{ assertion, active_context_components: [...active] },
"A later-turn reread is required even though active-context presence is the intended evidence.",
),
);
}
}
for (const assertion of (evalCase.assertions ?? []).filter((item) => item.type === "semantic")) {
const leakedTurn = prompts.findIndex((prompt) => assertion.rubric && prompt.includes(assertion.rubric));
if (leakedTurn >= 0)
findings.push(
finding(
"FEAS-RUBRIC-LEAK",
Object.hasOwn(evalCase, "prompt") ? "case.prompt" : `case.turns[${leakedTurn}].prompt`,
{ assertion_id: assertion.id },
"The fixed grader rubric is visible to the subject.",
),
);
}
const scriptedTurns = Object.hasOwn(evalCase, "prompt") ? 1 : (evalCase.turns ?? []).length;
const requiredTurns = scriptedTurns + Number(declaration.expected_tool_round_trips ?? 0);
if (
context.modelDriven !== false &&
Number(declaration.expected_tool_round_trips ?? 0) > 0 &&
context.maxTurns < requiredTurns
)
findings.push(
finding(
"FEAS-TURN-BUDGET",
"limits.max_turns_per_process",
{ supplied: context.maxTurns, required: requiredTurns, scripted_turns: scriptedTurns },
"The hard turn budget cannot cover scripted turns plus declared tool round trips.",
),
);
if (context.modelDriven !== false && context.estimatedCompactBytes > context.outputLimitBytes)
findings.push(
finding(
"FEAS-COMPACT-LIMIT",
"limits.output_limit_bytes",
{ estimated: context.estimatedCompactBytes, limit: context.outputLimitBytes },
"The compact model packet exceeds its retained-output limit.",
),
);
if (context.modelDriven !== false && context.estimatedTransportBytes > context.transportLimitBytes)
findings.push(
finding(
"FEAS-TRANSPORT-LIMIT",
"limits.transport_limit_bytes",
{ estimated: context.estimatedTransportBytes, limit: context.transportLimitBytes },
"Raw transport exceeds its separate safeguard.",
),
);
if (declaration.fixture_oracle) {
const outcome = context.runOracle ? await context.runOracle(declaration.fixture_oracle) : null;
if (!outcome?.passed) {
findings.push(
finding(
"FEAS-FIXTURE-ORACLE",
"case.feasibility.fixture_oracle",
{ checks: declaration.fixture_oracle.checks, observed: outcome ?? null },
"The declarative fixture oracle does not establish the declared pressure.",
),
);
}
}
return { schema_version: 1, case_id: evalCase.id, blocking: findings.length > 0, provider_requests: 0, findings };
}
export function renderFeasibilityRows(result) {
return (
result.findings
.map((item) => `BLOCK|${item.id}|${item.source_span}|${item.blocking_reason.replaceAll("|", "\\|")}`)
.join("\n") + (result.findings.length ? "\n" : "")
);
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |