Skip to content

hassan-mohiddin/freeflow / .deprecated/skill-tooling-pre-rewrite/skills/evaluate-skill

evaluate-skill

Verified

Use when evaluating, comparing, or making readiness claims about agent-skill behavior.

scripts/lib/pi-adapter.mjs

View on GitHub
import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
import { constants } from "node:fs";
import { accessSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { DEFAULT_OUTPUT_LIMIT_BYTES } from "./constants.mjs";
import { runProcess } from "./process.mjs";
import { startRpcClient } from "./rpc-client.mjs";
import { sha256 } from "./hash.mjs";

export const PI_ADAPTER_VERSION = "pi-bootstrap-v2";
export const PI_RPC_ADAPTER_VERSION = "pi-rpc-scripted-v1";
const here = dirname(fileURLToPath(import.meta.url));
export const PI_ROOT_GUARD_PATH = resolve(here, "..", "pi-root-guard.mjs");

function sourceAgentDir(env) {
  return resolve(env.PI_CODING_AGENT_DIR ?? resolve(homedir(), ".pi", "agent"));
}

export async function prepareIsolatedPiConfig(configDir, env = process.env) {
  await mkdir(configDir, { recursive: true });
  await writeFile(
    resolve(configDir, "settings.json"),
    `${JSON.stringify(
      {
        defaultProjectTrust: "never",
        enableInstallTelemetry: false,
        enableUpdateCheck: false,
        quietStartup: true,
        retry: {
          enabled: false,
          maxRetries: 0,
          provider: { maxRetries: 0 },
        },
      },
      null,
      2,
    )}\n`,
  );
  const authSource = resolve(sourceAgentDir(env), "auth.json");
  try {
    accessSync(authSource, constants.R_OK);
    await copyFile(authSource, resolve(configDir, "auth.json"));
  } catch {}
}

function normalizedSkillSnapshots(skillSnapshot, skillSnapshots) {
  if (Array.isArray(skillSnapshots) && skillSnapshots.length > 0) return skillSnapshots;
  return skillSnapshot ? [{ name: null, path: skillSnapshot }] : [];
}

function appendDeclaredResources(args, { skillSnapshot, skillSnapshots, runtimeExtension }) {
  if (runtimeExtension) args.push("--extension", runtimeExtension);
  for (const snapshot of normalizedSkillSnapshots(skillSnapshot, skillSnapshots)) args.push("--skill", snapshot.path);
}

export function buildPiInvocation({
  prompt,
  provider,
  model,
  thinking,
  tools,
  skillSnapshot,
  skillSnapshots,
  runtimeExtension,
}) {
  if (!provider || !model || !thinking) throw new Error("Pi runs require provider, model, and thinking");
  const args = [
    "--mode",
    "json",
    "--print",
    "--no-session",
    "--no-extensions",
    "--no-skills",
    "--no-prompt-templates",
    "--no-themes",
    "--no-context-files",
    "--no-approve",
    "--offline",
    "--extension",
    PI_ROOT_GUARD_PATH,
    "--provider",
    provider,
    "--model",
    model,
    "--thinking",
    thinking,
  ];
  if (tools.length === 0) args.push("--no-tools");
  else args.push("--tools", tools.join(","));
  appendDeclaredResources(args, { skillSnapshot, skillSnapshots, runtimeExtension });
  args.push(prompt.startsWith("-") ? ` ${prompt}` : prompt);
  return { command: "pi", args };
}

export function buildPiRpcInvocation({
  provider,
  model,
  thinking,
  tools,
  skillSnapshot,
  skillSnapshots,
  runtimeExtension,
}) {
  if (!provider || !model || !thinking) throw new Error("Pi RPC runs require provider, model, and thinking");
  const args = [
    "--mode",
    "rpc",
    "--no-session",
    "--no-extensions",
    "--no-skills",
    "--no-prompt-templates",
    "--no-themes",
    "--no-context-files",
    "--no-approve",
    "--offline",
    "--extension",
    PI_ROOT_GUARD_PATH,
    "--provider",
    provider,
    "--model",
    model,
    "--thinking",
    thinking,
  ];
  if (tools.length === 0) args.push("--no-tools");
  else args.push("--tools", tools.join(","));
  appendDeclaredResources(args, { skillSnapshot, skillSnapshots, runtimeExtension });
  return { command: "pi", args };
}

export function compactPiRpcRecord(event) {
  if (event?.type === "message_update") {
    const {
      partial: _partial,
      delta: _delta,
      thinking: _thinking,
      ...assistantMessageEvent
    } = event.assistantMessageEvent ?? {};
    if (!String(assistantMessageEvent.type ?? "").startsWith("thinking_")) {
      if (event.assistantMessageEvent?.delta !== undefined)
        assistantMessageEvent.delta = event.assistantMessageEvent.delta;
    }
    return { type: event.type, assistantMessageEvent };
  }
  if (event?.type === "message_start")
    return { type: event.type, message: { id: event.message?.id, role: event.message?.role } };
  if (event?.type === "message_end") {
    const message = { ...event.message };
    if (Array.isArray(message.content)) message.content = message.content.filter((part) => part?.type !== "thinking");
    return { type: event.type, message };
  }
  if (event?.type === "turn_end" || event?.type === "agent_end")
    return { type: event.type, willRetry: event.willRetry ?? false };
  if (event?.type === "tool_execution_update")
    return { type: event.type, toolCallId: event.toolCallId, toolName: event.toolName };
  return event;
}

export function compactPiJsonLine(line) {
  let event;
  try {
    event = JSON.parse(line);
  } catch {
    return line;
  }
  if (event.type === "message_update") {
    const { partial: _partial, ...assistantMessageEvent } = event.assistantMessageEvent ?? {};
    return JSON.stringify({ type: event.type, assistantMessageEvent });
  }
  if (event.type === "message_start") {
    return JSON.stringify({ type: event.type, message: { id: event.message?.id, role: event.message?.role } });
  }
  if (event.type === "turn_end" || event.type === "agent_end") return JSON.stringify({ type: event.type });
  if (event.type === "tool_execution_update") {
    return JSON.stringify({ type: event.type, toolCallId: event.toolCallId, toolName: event.toolName });
  }
  return line;
}

function textFromContent(content) {
  if (typeof content === "string") return content;
  if (!Array.isArray(content)) return "";
  return content
    .filter((part) => part?.type === "text" && typeof part.text === "string")
    .map((part) => part.text)
    .join("\n");
}

export function parsePiJsonEvents(raw, { skillSnapshot, skillSnapshots } = {}) {
  const events = [];
  const parseErrors = [];
  for (const [index, line] of raw.split("\n").entries()) {
    if (!line.trim()) continue;
    try {
      events.push(JSON.parse(line));
    } catch (error) {
      parseErrors.push({ line: index + 1, error: error.message });
    }
  }

  const assistantMessages = events
    .filter((event) => event.type === "message_end" && event.message?.role === "assistant")
    .map((event) => event.message);
  const finalMessage = assistantMessages.at(-1) ?? null;
  const seen = new Set();
  const usage = { input: 0, output: 0, cache_read: 0, cache_write: 0, total_tokens: 0, cost: null };
  let hasUsage = false;
  let costTotal = 0;
  let hasCost = false;
  for (const message of assistantMessages) {
    const key = message.id ?? JSON.stringify(message.usage ?? {});
    if (seen.has(key)) continue;
    seen.add(key);
    const item = message.usage;
    if (!item) continue;
    hasUsage = true;
    usage.input += item.input ?? 0;
    usage.output += item.output ?? 0;
    usage.cache_read += item.cacheRead ?? 0;
    usage.cache_write += item.cacheWrite ?? 0;
    usage.total_tokens += item.totalTokens ?? (item.input ?? 0) + (item.output ?? 0);
    const cost = item.cost?.total;
    if (typeof cost === "number") {
      hasCost = true;
      costTotal += cost;
    }
  }
  if (hasCost) usage.cost = { total_usd: costTotal };

  const toolEvents = events
    .filter((event) => event.type === "tool_execution_start")
    .map((event) => ({
      tool_call_id: event.toolCallId,
      tool_name: event.toolName,
      args: event.args,
    }));
  const declaredSkills = normalizedSkillSnapshots(skillSnapshot, skillSnapshots);
  const skillReads = Object.fromEntries(
    declaredSkills.map((snapshot, index) => {
      const name = snapshot.name ?? `subject-${index + 1}`;
      const skillFile = resolve(snapshot.path, "SKILL.md");
      const observed = toolEvents.some((event) => {
        if (event.tool_name !== "read" || typeof event.args?.path !== "string") return false;
        return resolve(event.args.path.replace(/^@/, "")) === skillFile;
      });
      return [name, observed];
    }),
  );
  const skillRead = Object.values(skillReads).some(Boolean);

  return {
    events,
    parse_errors: parseErrors,
    final_text: finalMessage ? textFromContent(finalMessage.content) : "",
    usage: hasUsage ? usage : null,
    tool_events: toolEvents,
    skill_read: skillRead,
    skill_reads: skillReads,
  };
}

export async function runPiSubject({
  prompt,
  provider,
  model,
  thinking,
  tools,
  skillSnapshot,
  skillSnapshots,
  runtimeExtension,
  runtimeEnvironment = {},
  workspace,
  configDir,
  readRoots,
  writeRoots,
  timeoutMs,
  outputLimitBytes,
  transportLimitBytes,
  maxTurns,
  signal,
}) {
  await prepareIsolatedPiConfig(configDir);
  const invocation = buildPiInvocation({
    prompt,
    provider,
    model,
    thinking,
    tools,
    skillSnapshot,
    skillSnapshots,
    runtimeExtension,
  });
  const counterPath = resolve(configDir, "runtime-counters.json");
  const env = {
    ...process.env,
    PI_CODING_AGENT_DIR: configDir,
    PI_TELEMETRY: "0",
    FREEFLOW_EVAL_ROOT_POLICY: JSON.stringify({ read_roots: readRoots, write_roots: writeRoots }),
    FREEFLOW_EVAL_COUNTER_PATH: counterPath,
    FREEFLOW_EVAL_MAX_TURNS: String(maxTurns ?? 0),
    ...runtimeEnvironment,
  };
  const processResult = await runProcess(invocation.command, invocation.args, {
    cwd: workspace,
    env,
    timeoutMs,
    outputLimitBytes,
    transportLimitBytes: transportLimitBytes ?? Math.max(outputLimitBytes, DEFAULT_OUTPUT_LIMIT_BYTES),
    stdoutLineTransform: compactPiJsonLine,
    signal,
  });
  const parsed = parsePiJsonEvents(processResult.stdout, { skillSnapshot, skillSnapshots });
  let runtimeCounters = {
    provider_requests: parsed.events.filter((event) => event.type === "turn_start").length,
    turns_started: parsed.events.filter((event) => event.type === "turn_start").length,
    tool_calls: parsed.tool_events.length,
    hard_turn_limit_reached: false,
  };
  try {
    runtimeCounters = JSON.parse(await readFile(counterPath, "utf8"));
  } catch {}
  return { invocation, process: processResult, parsed, runtime_counters: runtimeCounters };
}

function canonicalEntry(entry) {
  if (entry?.type !== "message" || !entry.message) return entry;
  const message = { ...entry.message };
  if (Array.isArray(message.content)) message.content = message.content.filter((part) => part?.type !== "thinking");
  return { ...entry, message };
}

function normalizeSessionUsage(stats) {
  const tokens = stats?.tokens;
  if (!tokens) return null;
  const usage = {
    input: tokens.input ?? 0,
    output: tokens.output ?? 0,
    cache_read: tokens.cacheRead ?? 0,
    cache_write: tokens.cacheWrite ?? 0,
    total_tokens:
      tokens.total ?? (tokens.input ?? 0) + (tokens.output ?? 0) + (tokens.cacheRead ?? 0) + (tokens.cacheWrite ?? 0),
    cost: null,
  };
  if (typeof stats.cost === "number") usage.cost = { total_usd: stats.cost };
  return usage;
}

function usageDelta(current, previous) {
  if (!current) return null;
  const prior = previous ?? {
    input: 0,
    output: 0,
    cache_read: 0,
    cache_write: 0,
    total_tokens: 0,
    cost: { total_usd: 0 },
  };
  const delta = {
    input: current.input - (prior.input ?? 0),
    output: current.output - (prior.output ?? 0),
    cache_read: current.cache_read - (prior.cache_read ?? 0),
    cache_write: current.cache_write - (prior.cache_write ?? 0),
    total_tokens: current.total_tokens - (prior.total_tokens ?? 0),
    cost: null,
  };
  if (current.cost && (previous === null || prior.cost))
    delta.cost = { total_usd: current.cost.total_usd - (prior.cost?.total_usd ?? 0) };
  return delta;
}

function counterDelta(current, previous) {
  return {
    provider_requests: (current.provider_requests ?? 0) - (previous.provider_requests ?? 0),
    turns_started: (current.turns_started ?? 0) - (previous.turns_started ?? 0),
    tool_calls: (current.tool_calls ?? 0) - (previous.tool_calls ?? 0),
    hard_turn_limit_reached: Boolean(current.hard_turn_limit_reached),
  };
}

function textFromLastAssistant(response) {
  return typeof response?.data?.text === "string" ? response.data.text : "";
}

function toolEvents(events) {
  return events
    .filter((event) => event.type === "tool_execution_start")
    .map((event) => ({
      tool_call_id: event.toolCallId,
      tool_name: event.toolName,
      args: event.args,
    }));
}

async function readRuntimeCounters(path, fallback) {
  try {
    return JSON.parse(await readFile(path, "utf8"));
  } catch {
    return fallback;
  }
}

function requireRpcSuccess(response, command) {
  if (!response?.success) throw new Error(`Pi RPC ${command} failed: ${response?.error ?? "unknown error"}`);
  return response.data;
}

export async function runPiRpcSubject({
  turns,
  provider,
  model,
  thinking,
  tools,
  skillSnapshot,
  skillSnapshots,
  runtimeExtension,
  runtimeEnvironment = {},
  workspace,
  configDir,
  readRoots,
  writeRoots,
  timeoutMs,
  outputLimitBytes,
  transportLimitBytes,
  maxTurns,
  maxUsd,
  signal,
  onTurnSettled = async () => null,
  startClient = startRpcClient,
}) {
  await prepareIsolatedPiConfig(configDir);
  const invocation = buildPiRpcInvocation({
    provider,
    model,
    thinking,
    tools,
    skillSnapshot,
    skillSnapshots,
    runtimeExtension,
  });
  const counterPath = resolve(configDir, "runtime-counters.json");
  const env = {
    ...process.env,
    PI_CODING_AGENT_DIR: configDir,
    PI_TELEMETRY: "0",
    FREEFLOW_EVAL_ROOT_POLICY: JSON.stringify({ read_roots: readRoots, write_roots: writeRoots }),
    FREEFLOW_EVAL_COUNTER_PATH: counterPath,
    FREEFLOW_EVAL_MAX_TURNS: String(maxTurns ?? 0),
    ...runtimeEnvironment,
  };

  let client = null;
  let processResult = null;
  let operationError = null;
  let canonicalOutputLimitExceeded = false;
  let canonicalRetainedBytes = 0;
  let previousLeaf = null;
  let previousUsage = null;
  let previousCounters = { provider_requests: 0, turns_started: 0, tool_calls: 0, hard_turn_limit_reached: false };
  const settledTurns = [];
  const declaredSkills = normalizedSkillSnapshots(skillSnapshot, skillSnapshots);
  const skillReadsForEvents = (events) =>
    Object.fromEntries(
      declaredSkills.map((snapshot, index) => {
        const name = snapshot.name ?? `subject-${index + 1}`;
        const skillFile = resolve(snapshot.path, "SKILL.md");
        const observed = events.some(
          (event) =>
            event.tool_name === "read" &&
            typeof event.args?.path === "string" &&
            resolve(event.args.path.replace(/^@/, "")) === skillFile,
        );
        return [name, observed];
      }),
    );
  try {
    client = await startClient(invocation.command, invocation.args, {
      cwd: workspace,
      env,
      timeoutMs,
      outputLimitBytes,
      transportLimitBytes: transportLimitBytes ?? Math.max(outputLimitBytes, DEFAULT_OUTPUT_LIMIT_BYTES),
      signal,
      recordTransform: compactPiRpcRecord,
    });
    requireRpcSuccess(await client.request("set_auto_retry", { enabled: false }), "set_auto_retry");
    requireRpcSuccess(await client.request("set_auto_compaction", { enabled: false }), "set_auto_compaction");
    const initialState = requireRpcSuccess(await client.request("get_state"), "get_state");
    if (
      initialState.isStreaming ||
      initialState.isCompacting ||
      initialState.messageCount !== 0 ||
      initialState.pendingMessageCount !== 0
    ) {
      throw new Error("Pi RPC session did not start empty and settled");
    }

    for (const [index, turn] of turns.entries()) {
      const startedAt = new Date();
      const settled = await client.promptAndSettle({ turnId: turn.id, message: turn.prompt });
      const entriesData = requireRpcSuccess(
        await client.request("get_entries", previousLeaf ? { since: previousLeaf } : {}),
        "get_entries",
      );
      const finalData = requireRpcSuccess(await client.request("get_last_assistant_text"), "get_last_assistant_text");
      const stats = requireRpcSuccess(await client.request("get_session_stats"), "get_session_stats");
      const state = requireRpcSuccess(await client.request("get_state"), "get_state");
      if (state.isStreaming || state.isCompacting || state.pendingMessageCount !== 0)
        throw new Error(`Pi RPC ${turn.id} did not remain settled`);
      const currentUsage = normalizeSessionUsage(stats);
      const currentCounters = await readRuntimeCounters(counterPath, {
        provider_requests: client.records.filter((event) => event.type === "turn_start").length,
        turns_started: client.records.filter((event) => event.type === "turn_start").length,
        tool_calls: client.records.filter((event) => event.type === "tool_execution_start").length,
        hard_turn_limit_reached: false,
      });
      const events = settled.events;
      const toolsForTurn = toolEvents(events);
      const endedAt = new Date();
      const turnEvidence = {
        id: turn.id,
        index,
        prompt_sha256: sha256(turn.prompt),
        response: settled.response,
        entries: (entriesData.entries ?? []).map(canonicalEntry),
        leaf_id: entriesData.leafId ?? null,
        final_text: textFromLastAssistant({ data: finalData }),
        events,
        tool_events: toolsForTurn,
        usage: currentUsage,
        usage_delta: usageDelta(currentUsage, previousUsage),
        runtime_counters: currentCounters,
        runtime_counter_delta: counterDelta(currentCounters, previousCounters),
        started_at: startedAt.toISOString(),
        settled_at: endedAt.toISOString(),
        duration_ms: endedAt.getTime() - startedAt.getTime(),
        workspace: null,
        skill_reads: skillReadsForEvents(toolsForTurn),
      };
      turnEvidence.skill_read = Object.values(turnEvidence.skill_reads).some(Boolean);
      turnEvidence.workspace = await onTurnSettled(turnEvidence);
      settledTurns.push(turnEvidence);
      previousLeaf = entriesData.leafId ?? previousLeaf;
      previousUsage = currentUsage;
      previousCounters = currentCounters;
      canonicalRetainedBytes = Buffer.byteLength(
        `${JSON.stringify({ schema_version: 1, turns: settledTurns }, null, 2)}\n`,
      );
      if (canonicalRetainedBytes > outputLimitBytes) {
        canonicalOutputLimitExceeded = true;
        throw new Error(`Pi RPC canonical retained evidence exceeded ${outputLimitBytes} bytes`);
      }
      if (index < turns.length - 1) {
        if (maxTurns > 0 && currentCounters.provider_requests >= maxTurns)
          throw new Error(`Pi RPC provider-turn limit reached before ${turns[index + 1].id}`);
        if (maxUsd !== null && maxUsd !== undefined && currentUsage?.cost && currentUsage.cost.total_usd >= maxUsd) {
          throw new Error(`Pi RPC observed spend ceiling reached before ${turns[index + 1].id}`);
        }
      }
    }
  } catch (error) {
    operationError = error instanceof Error ? error.message : String(error);
  } finally {
    if (client) processResult = await client.dispose();
  }

  if (!processResult) throw new Error(operationError ?? "Pi RPC process did not start");
  const runtimeCounters = await readRuntimeCounters(counterPath, previousCounters);
  const allToolEvents = settledTurns.flatMap((turn) => turn.tool_events);
  const skillReads = skillReadsForEvents(allToolEvents);
  const skillRead = Object.values(skillReads).some(Boolean);
  canonicalRetainedBytes = Buffer.byteLength(
    `${JSON.stringify({ schema_version: 1, turns: settledTurns }, null, 2)}\n`,
  );
  if (canonicalRetainedBytes > outputLimitBytes) {
    canonicalOutputLimitExceeded = true;
    operationError ??= `Pi RPC canonical retained evidence exceeded ${outputLimitBytes} bytes`;
  }
  processResult = {
    ...processResult,
    output_limit_exceeded: Boolean(processResult.output_limit_exceeded || canonicalOutputLimitExceeded),
    retained_output_bytes: Math.max(Number(processResult.retained_output_bytes ?? 0), canonicalRetainedBytes),
  };
  const parseErrors =
    operationError || processResult.protocol_failed
      ? [{ line: null, error: operationError ?? processResult.failure ?? "Pi RPC protocol failure" }]
      : [];
  return {
    invocation,
    process: processResult,
    parsed: {
      events: client.records,
      parse_errors: parseErrors,
      final_text: settledTurns.at(-1)?.final_text ?? "",
      usage: previousUsage,
      tool_events: allToolEvents,
      skill_read: skillRead,
      skill_reads: skillReads,
      turns: settledTurns,
    },
    runtime_counters: runtimeCounters,
  };
}

export function redactedInvocation(invocation) {
  const args = [...invocation.args];
  const modeIndex = args.indexOf("--mode");
  if (args[modeIndex + 1] !== "rpc" && args.length > 0) args[args.length - 1] = "<natural-prompt>";
  for (const flag of ["--extension", "--skill"]) {
    for (let index = 0; index < args.length; index += 1) {
      if (args[index] === flag && index + 1 < args.length) args[index + 1] = `<${flag.slice(2)}>`;
    }
  }
  return { command: invocation.command, args };
}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall