Skip to content

hassan-mohiddin/freeflow / .deprecated/skill-tooling-pre-rewrite/skills/evaluate-skill

evaluate-skill

Verified

Use when evaluating, comparing, or making readiness claims about agent-skill behavior.

scripts/lib/capabilities.mjs

View on GitHub
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { spawnSync } from "node:child_process";
import { startRpcClient } from "./rpc-client.mjs";

const here = dirname(fileURLToPath(import.meta.url));
const compositionRuntimeExtension = resolve(here, "..", "pi-composition-runtime.mjs");
const compositionInteractionContract = resolve(here, "..", "..", "..", "..", "runtime", "interaction-contract.md");
const compositionWorkflow = resolve(here, "..", "..", "..", "workflow", "SKILL.md");

function run(command, args, options = {}) {
  const result = spawnSync(command, args, { encoding: "utf8", ...options });
  return {
    command,
    args,
    status: result.status,
    stdout: result.stdout ?? "",
    stderr: result.stderr ?? "",
    error: result.error?.message ?? null,
  };
}

export function probePi() {
  const version = run("pi", ["--version"]);
  const help = run("pi", ["--help"]);
  const available = version.status === 0 && help.status === 0;
  const text = help.stdout;

  return {
    id: "pi",
    available,
    version: available ? version.stdout.trim() : null,
    error: available ? null : (version.error ?? version.stderr.trim() ?? help.stderr.trim()),
    rpc_error: null,
    capabilities: {
      one_shot_json: available && text.includes("--mode <mode>") && text.includes("json"),
      rpc_jsonl: false,
      native_skill_loading: available && text.includes("--skill <path>"),
      multi_skill_loading: available && text.includes("--skill <path>"),
      explicit_extensions: available && text.includes("--extension"),
      explicit_runtime_context: available && text.includes("--extension"),
      disable_extension_discovery: available && text.includes("--no-extensions"),
      disable_context_files: available && text.includes("--no-context-files"),
      tool_allowlist: available && text.includes("--tools"),
      usage_events: available,
      multi_turn: false,
      strict_tool_isolation: available && text.includes("--extension") && text.includes("--tools"),
      composition_activation_evidence: available && text.includes("--tools"),
    },
  };
}

export async function probePiRpc(base = probePi(), dependencies = {}) {
  if (!base.available) return base;
  const startClient = dependencies.startRpcClient ?? startRpcClient;
  const root = await mkdtemp(resolve(tmpdir(), "freeflow-pi-rpc-probe-"));
  const configDir = resolve(root, "pi-config");
  let client = null;
  try {
    await mkdir(configDir, { recursive: true });
    const alphaSkill = resolve(root, "eval-alpha");
    const betaSkill = resolve(root, "eval-beta");
    await Promise.all([mkdir(alphaSkill, { recursive: true }), mkdir(betaSkill, { recursive: true })]);
    await Promise.all([
      writeFile(
        resolve(alphaSkill, "SKILL.md"),
        "---\nname: eval-alpha\ndescription: Alpha evaluator probe.\n---\n\n# Alpha\n",
      ),
      writeFile(
        resolve(betaSkill, "SKILL.md"),
        "---\nname: eval-beta\ndescription: Beta evaluator probe.\n---\n\n# Beta\n",
      ),
    ]);
    await writeFile(
      resolve(configDir, "settings.json"),
      `${JSON.stringify(
        {
          defaultProjectTrust: "never",
          enableInstallTelemetry: false,
          enableUpdateCheck: false,
          quietStartup: true,
          retry: { enabled: false, maxRetries: 0, provider: { maxRetries: 0 } },
        },
        null,
        2,
      )}\n`,
    );
    client = await startClient(
      "pi",
      [
        "--mode",
        "rpc",
        "--no-session",
        "--no-extensions",
        "--no-skills",
        "--no-prompt-templates",
        "--no-themes",
        "--no-context-files",
        "--no-approve",
        "--offline",
        "--no-tools",
        "--extension",
        compositionRuntimeExtension,
        "--skill",
        alphaSkill,
        "--skill",
        betaSkill,
      ],
      {
        env: {
          ...process.env,
          PI_CODING_AGENT_DIR: configDir,
          PI_TELEMETRY: "0",
          FREEFLOW_EVAL_RUNTIME_INTERACTION_CONTRACT: compositionInteractionContract,
          FREEFLOW_EVAL_RUNTIME_WORKFLOW: compositionWorkflow,
          FREEFLOW_EVAL_RUNTIME_EVIDENCE: resolve(root, "composition-runtime-evidence.jsonl"),
        },
        timeoutMs: 10000,
        outputLimitBytes: 1024 * 1024,
        transportLimitBytes: 2 * 1024 * 1024,
      },
    );
    const retry = await client.request("set_auto_retry", { enabled: false });
    const compaction = await client.request("set_auto_compaction", { enabled: false });
    const state = await client.request("get_state");
    const commands = await client.request("get_commands");
    const processResult = await client.dispose();
    const commandNames = new Set(commands.data?.commands?.map((command) => command.name) ?? []);
    const multiSkillLoading = commandNames.has("skill:eval-alpha") && commandNames.has("skill:eval-beta");
    const supported =
      retry.success &&
      compaction.success &&
      state.success &&
      commands.success &&
      multiSkillLoading &&
      state.data?.isStreaming === false &&
      state.data?.isCompacting === false &&
      state.data?.messageCount === 0 &&
      processResult.code === 0 &&
      !processResult.timed_out &&
      !processResult.output_limit_exceeded &&
      !processResult.transport_limit_exceeded &&
      !processResult.protocol_failed;
    return {
      ...base,
      rpc_error: supported ? null : "Pi RPC handshake returned an unsupported state",
      capabilities: {
        ...base.capabilities,
        rpc_jsonl: supported,
        multi_turn: supported,
        multi_skill_loading: multiSkillLoading,
        explicit_runtime_context: supported,
        composition_activation_evidence: supported && base.capabilities.composition_activation_evidence,
      },
    };
  } catch (error) {
    return {
      ...base,
      rpc_error: error instanceof Error ? error.message : String(error),
      capabilities: {
        ...base.capabilities,
        rpc_jsonl: false,
        multi_turn: false,
        multi_skill_loading: false,
        explicit_runtime_context: false,
        composition_activation_evidence: false,
      },
    };
  } finally {
    if (client) await client.dispose().catch(() => {});
    await rm(root, { recursive: true, force: true });
  }
}

export function probeCodex(dependencies = {}) {
  const runVersion = dependencies.run ?? run;
  const version = runVersion("codex", ["--version"]);
  const available = version.status === 0;
  const value = available ? version.stdout.trim() : null;
  const provenVersion = value === "codex-cli 0.144.1";
  return {
    id: "codex",
    available,
    version: value,
    error: available ? null : (version.error ?? version.stderr.trim()),
    fidelity: "diagnostic",
    isolation_profile: "codex-diagnostic-macos-v1",
    capabilities: {
      exec_jsonl: provenVersion,
      isolated_home: provenVersion,
      strict_config: provenVersion,
      ephemeral: provenVersion,
      ignore_rules: provenVersion,
      ambient_context_disabled: provenVersion,
      explicit_skill: provenVersion,
      strict_filesystem_isolation: provenVersion,
      network_disabled: provenVersion,
      process_limits: provenVersion,
      provider_request_bound: false,
      spend_bound: false,
    },
  };
}

export async function capabilitiesFor(host, mode) {
  if (host === "none") {
    return {
      id: "none",
      available: true,
      version: null,
      capabilities: { structure: true },
    };
  }
  if (host === "pi") {
    const base = probePi();
    return mode === "rpc-scripted" ? probePiRpc(base) : base;
  }
  if (host === "codex") return probeCodex();
  return { id: host, available: false, version: null, capabilities: {} };
}

export function supportedEvidenceClasses(host, mode) {
  if (host === "none" || mode === "deterministic") return new Set(["structure"]);
  if (host === "pi" && mode === "json") {
    return new Set(["structure", "explicit-instruction", "native-activation", "artifact-outcome"]);
  }
  if (host === "pi" && mode === "rpc-scripted") {
    return new Set(["structure", "explicit-instruction", "native-activation", "artifact-outcome", "multi-turn"]);
  }
  if (host === "codex" && mode === "exec")
    return new Set(["structure", "explicit-instruction", "native-activation", "artifact-outcome"]);
  return new Set();
}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall