Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

scripts/check-skillopt-readiness.mjs

View on GitHub
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";

const root = process.cwd();
const scriptPath = fileURLToPath(import.meta.url);
const skillRoot = path.resolve(path.dirname(scriptPath), "..");
const allowedModes = new Set(["native-provider", "hybrid-codex-target", "codex-cli-all"]);
const runProfiles = new Set(["official-parity", "exploratory"]);
const pythonManagers = new Set(["auto", "uv", "local"]);
const EXPLORATORY_MIN = { positive: 10, val: 3, test: 3 };
const OFFICIAL_RECOMMENDED = { positive: 20, val: 5, test: 5 };
const SAFE_MODEL_ENV_PLACEHOLDERS = [
  "SKILLOPT_OPTIMIZER_MODEL",
  "SKILLOPT_TARGET_MODEL",
  "SKILLOPT_JUDGE_MODEL",
  "SKILLOPT_REFLECTION_MODEL",
];
const OPENAI_COMPATIBLE_AUTH_MODES = new Set(["openai_compatible", "compat", "openai"]);
const TOKENLESS_AZURE_AUTH_MODES = new Set(["azure_cli", "managed_identity"]);
const REQUIRED_SPLIT_ITEM_FIELDS = new Set([
  "id",
  "skill_name",
  "case_path",
  "prompt",
  "expected_behavior",
  "rubric_path",
  "fixtures",
  "split_family",
  "split_group",
  "expected_artifacts",
  "deterministic_assertions",
  "visual_assertions",
  "tags",
  "should_trigger",
  "workspace_policy",
  "source_hash",
]);
const SUPPORTED_WORKSPACE_POLICIES = new Set(["text-only", "isolated-artifact-write"]);
const PYTHON_PATCH_AST_PROBE = String.raw`
import ast
import json
import sys

safe_names = set(json.loads(sys.argv[1]))
files = {
    "registry_train": sys.argv[2],
    "registry_eval": sys.argv[3],
    "safe_env": sys.argv[4],
    "trainer_steps": sys.argv[5],
}


def parse_file(filename):
    with open(filename, "r", encoding="utf-8") as handle:
        return ast.parse(handle.read(), filename="<live-skillopt-patch>")


def string_value(node):
    return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None


def subscript_key(node):
    return string_value(node.slice) if isinstance(node, ast.Subscript) else None


def is_name(node, name):
    return isinstance(node, ast.Name) and node.id == name


def is_os_environ(node):
    return (
        isinstance(node, ast.Attribute)
        and node.attr == "environ"
        and is_name(node.value, "os")
    )


def registry_assignment(tree):
    register = next(
        (
            node
            for node in tree.body
            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
            and node.name == "_register_builtins"
        ),
        None,
    )
    if register is None:
        return False
    for candidate in ast.walk(register):
        if not isinstance(candidate, ast.Try):
            continue
        imported = any(
            isinstance(node, ast.ImportFrom)
            and node.module == "skillopt.envs.agent_skills.adapter"
            and any(alias.name == "AgentSkillsAdapter" for alias in node.names)
            for statement in candidate.body
            for node in ast.walk(statement)
        )
        assigned = any(
            isinstance(node, (ast.Assign, ast.AnnAssign))
            and any(
                isinstance(target, ast.Subscript)
                and is_name(target.value, "_ENV_REGISTRY")
                and subscript_key(target) == "agent_skills"
                for target in (
                    node.targets
                    if isinstance(node, ast.Assign)
                    else [node.target]
                )
            )
            and is_name(node.value, "AgentSkillsAdapter")
            for statement in candidate.body
            for node in ast.walk(statement)
        )
        if imported and assigned:
            return True
    return False


def literal_string_set(node):
    if isinstance(node, ast.Call) and is_name(node.func, "frozenset") and len(node.args) == 1:
        node = node.args[0]
    if not isinstance(node, (ast.Set, ast.List, ast.Tuple)):
        return None
    values = [string_value(item) for item in node.elts]
    return set(values) if all(value is not None for value in values) else None


def safe_env_expansion(tree):
    allowlist = None
    for node in tree.body:
        if not isinstance(node, (ast.Assign, ast.AnnAssign)):
            continue
        targets = node.targets if isinstance(node, ast.Assign) else [node.target]
        if any(is_name(target, "_ENV_PLACEHOLDER_ALLOWLIST") for target in targets):
            allowlist = literal_string_set(node.value)
    helper = next(
        (
            node
            for node in tree.body
            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
            and node.name == "_expand_safe_env_placeholders"
        ),
        None,
    )
    if allowlist != safe_names or helper is None:
        return False

    guarded_read = False
    for node in ast.walk(helper):
        if not isinstance(node, ast.If):
            continue
        membership = any(
            isinstance(part, ast.Compare)
            and is_name(part.left, "env_name")
            and len(part.ops) == 1
            and isinstance(part.ops[0], ast.In)
            and len(part.comparators) == 1
            and is_name(part.comparators[0], "_ENV_PLACEHOLDER_ALLOWLIST")
            for part in ast.walk(node.test)
        )
        presence_check = any(
            isinstance(part, ast.Call)
            and isinstance(part.func, ast.Attribute)
            and part.func.attr == "get"
            and is_os_environ(part.func.value)
            and len(part.args) == 1
            and is_name(part.args[0], "env_name")
            for part in ast.walk(node.test)
        )
        guarded_return = any(
            isinstance(part, ast.Return)
            and isinstance(part.value, ast.Subscript)
            and is_os_environ(part.value.value)
            and is_name(part.value.slice, "env_name")
            for statement in node.body
            for part in ast.walk(statement)
        )
        if membership and presence_check and guarded_return:
            guarded_read = True
            break

    applied_to_cfg = any(
        isinstance(node, ast.Assign)
        and any(is_name(target, "cfg") for target in node.targets)
        and isinstance(node.value, ast.Call)
        and is_name(node.value.func, "_expand_safe_env_placeholders")
        and len(node.value.args) == 1
        and is_name(node.value.args[0], "cfg")
        for node in ast.walk(tree)
    )
    return guarded_read and applied_to_cfg


def expression_dump(source):
    return ast.dump(ast.parse(source, mode="eval").body, include_attributes=False)


def trainer_step_logic(tree):
    assignments = {}
    for node in ast.walk(tree):
        if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
            assignments.setdefault(node.targets[0].id, []).append(
                ast.dump(node.value, include_attributes=False)
            )
    expected = {
        "requested_steps_per_epoch": expression_dump('int(cfg.get("steps_per_epoch", 0) or 0)'),
        "auto_steps_per_epoch": expression_dump("math.ceil(train_size / (batch_size * accumulation))"),
        "steps_per_epoch": expression_dump(
            "requested_steps_per_epoch if requested_steps_per_epoch > 0 else auto_steps_per_epoch"
        ),
        "batches_per_epoch": expression_dump("steps_per_epoch * accumulation"),
        "total_steps": expression_dump("num_epochs * steps_per_epoch"),
        "steps_source": expression_dump(
            '"configured" if requested_steps_per_epoch > 0 else "auto"'
        ),
    }
    return all(value in assignments.get(name, []) for name, value in expected.items())


checks = {}
errors = {}
for key, filename in files.items():
    try:
        tree = parse_file(filename)
        if key.startswith("registry_"):
            checks[key] = registry_assignment(tree)
        elif key == "safe_env":
            checks[key] = safe_env_expansion(tree)
        else:
            checks[key] = trainer_step_logic(tree)
    except (OSError, SyntaxError, UnicodeError):
        checks[key] = False
        errors[key] = "unparseable"

print(json.dumps({"checks": checks, "errors": errors}, sort_keys=True))
`;

function defaultRunProfile(mode) {
  return mode === "codex-cli-all" ? "exploratory" : "official-parity";
}

function parseArgs(argv) {
  const args = {
    mode: "hybrid-codex-target",
    json: false,
    setupOnly: false,
    codexProbe: false,
    strictTrainingReady: false,
    python: "python3",
    pythonManager: "auto",
    runProfile: null,
  };
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === "--help" || arg === "-h") {
      printHelp();
      process.exit(0);
    } else if (arg === "--json") args.json = true;
    else if (arg === "--setup-only") args.setupOnly = true;
    else if (arg === "--strict-training-ready") args.strictTrainingReady = true;
    else if (arg === "--codex-probe") args.codexProbe = true;
    else if (arg === "--no-codex-probe") args.codexProbe = false;
    else if (arg === "--skill") args.skill = argv[++i];
    else if (arg === "--mode") args.mode = argv[++i];
    else if (arg === "--python") args.python = argv[++i];
    else if (arg === "--python-manager") args.pythonManager = argv[++i];
    else if (arg === "--run-profile") args.runProfile = argv[++i];
    else fail(`Unknown argument: ${arg}`);
  }
  if (!allowedModes.has(args.mode)) fail(`Unsupported mode: ${args.mode}`);
  if (!pythonManagers.has(args.pythonManager))
    fail(`Unsupported python manager: ${args.pythonManager}`);
  args.runProfile ||= defaultRunProfile(args.mode);
  if (!runProfiles.has(args.runProfile)) fail(`Unsupported run profile: ${args.runProfile}`);
  return args;
}

function printHelp() {
  console.log(`Usage: node check-skillopt-readiness.mjs --skill <skill> [options]

Options:
  --mode <native-provider|hybrid-codex-target|codex-cli-all>
  --run-profile <official-parity|exploratory>
  --python-manager <auto|uv|local>
  --python <command>
  --setup-only
  --strict-training-ready
  --codex-probe
  --no-codex-probe
  --json
  --help`);
}

function fail(message) {
  console.error(message);
  process.exit(2);
}

function commandResult(command, args = [], options = {}) {
  const result = spawnSync(command, args, {
    cwd: options.cwd || root,
    encoding: "utf8",
    timeout: options.timeout || 30000,
    env: options.env || process.env,
  });
  return {
    ok: result.status === 0,
    status: result.status,
    stdout: (result.stdout || "").trim(),
    stderr: redact((result.stderr || "").trim()),
  };
}

function commandExists(command) {
  if (process.platform === "win32") {
    return commandResult("where", [command]).ok;
  }
  return commandResult("sh", ["-lc", `command -v ${command}`]).ok;
}

function commandPath(command) {
  const result =
    process.platform === "win32"
      ? commandResult("where", [command])
      : commandResult("sh", ["-lc", `command -v ${command}`]);
  return result.ok ? result.stdout.split(/\r?\n/)[0] || null : null;
}

function detectDrawioCli() {
  for (const command of ["drawio", "diagrams.net"]) {
    const found = commandPath(command);
    if (found) return { installed: true, command, path: redact(found) };
  }
  return { installed: false, command: null, path: null };
}

function resolveUvCommand() {
  if (commandExists("uv")) return "uv";
  const home = process.env.HOME || process.env.USERPROFILE;
  const candidates = home
    ? [
        path.join(home, ".local", "bin", process.platform === "win32" ? "uv.exe" : "uv"),
        path.join(home, ".cargo", "bin", process.platform === "win32" ? "uv.exe" : "uv"),
      ]
    : [];
  return candidates.find((candidate) => fs.existsSync(candidate)) || null;
}

function redact(text) {
  return String(text)
    .replaceAll(path.resolve(process.env.HOME || root), "~")
    .replace(/sk-[A-Za-z0-9_-]{20,}/g, "[redacted-token]")
    .replace(
      /[A-Za-z0-9_./-]*(auth|token|credential)[A-Za-z0-9_./-]*/gi,
      "[redacted-auth-reference]",
    );
}

function parseFrontmatter(text) {
  const match = text.match(/^---\n([\s\S]*?)\n---\n?/);
  if (!match) return null;
  const fields = {};
  for (const line of match[1].split("\n")) {
    const i = line.indexOf(":");
    if (i === -1) continue;
    fields[line.slice(0, i).trim()] = line
      .slice(i + 1)
      .trim()
      .replace(/^["']|["']$/g, "");
  }
  return fields;
}

function walk(dir, predicate) {
  if (!fs.existsSync(dir)) return [];
  const found = [];
  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
    const full = path.join(dir, entry.name);
    if (entry.isDirectory()) found.push(...walk(full, predicate));
    else if (predicate(full)) found.push(full);
  }
  return found;
}

function resolveSkill(skill) {
  if (!skill) return null;
  const direct = path.resolve(root, skill);
  if (fs.existsSync(direct) && path.basename(direct) === "SKILL.md") return direct;
  if (fs.existsSync(path.join(direct, "SKILL.md"))) return path.join(direct, "SKILL.md");

  const matches = walk(root, (file) => path.basename(file) === "SKILL.md").filter((file) => {
    const rel = path.relative(root, file).replaceAll("\\", "/");
    if (!rel.startsWith("skills/") && !rel.startsWith("incubator/skills/")) return false;
    return path.basename(path.dirname(file)) === skill;
  });
  return matches[0] || null;
}

function providerPresence() {
  const names = [
    "AZURE_OPENAI_ENDPOINT",
    "AZURE_OPENAI_API_KEY",
    "AZURE_OPENAI_AUTH_MODE",
    "AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID",
    "OPTIMIZER_AZURE_OPENAI_ENDPOINT",
    "OPTIMIZER_AZURE_OPENAI_API_KEY",
    "OPTIMIZER_AZURE_OPENAI_AUTH_MODE",
    "OPTIMIZER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID",
    "AZURE_OPENAI_OPTIMIZER_ENDPOINT",
    "AZURE_OPENAI_OPTIMIZER_API_KEY",
    "AZURE_OPENAI_OPTIMIZER_AUTH_MODE",
    "AZURE_OPENAI_OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID",
    "TARGET_AZURE_OPENAI_ENDPOINT",
    "TARGET_AZURE_OPENAI_API_KEY",
    "TARGET_AZURE_OPENAI_AUTH_MODE",
    "TARGET_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID",
    "AZURE_OPENAI_TARGET_ENDPOINT",
    "AZURE_OPENAI_TARGET_API_KEY",
    "AZURE_OPENAI_TARGET_AUTH_MODE",
    "AZURE_OPENAI_TARGET_MANAGED_IDENTITY_CLIENT_ID",
    "OPENAI_API_KEY",
    "QWEN_CHAT_BASE_URL",
    "QWEN_CHAT_MODEL",
    "QWEN_CHAT_API_KEY",
    "OPTIMIZER_QWEN_CHAT_BASE_URL",
    "OPTIMIZER_QWEN_CHAT_MODEL",
    "OPTIMIZER_QWEN_CHAT_API_KEY",
    "TARGET_QWEN_CHAT_BASE_URL",
    "TARGET_QWEN_CHAT_MODEL",
    "TARGET_QWEN_CHAT_API_KEY",
    "MINIMAX_API_KEY",
    "MINIMAX_BASE_URL",
    "MINIMAX_MODEL",
    "SKILLOPT_OPTIMIZER_MODEL",
    "SKILLOPT_TARGET_MODEL",
    "SKILLOPT_JUDGE_MODEL",
  ];
  return Object.fromEntries(names.map((name) => [name, Boolean(process.env[name])]));
}

function readJsonArray(file) {
  return readJsonArrayInfo(file).items;
}

function readJsonArrayInfo(file) {
  if (!fs.existsSync(file)) return { exists: false, valid: false, items: [] };
  try {
    const data = JSON.parse(fs.readFileSync(file, "utf8"));
    return {
      exists: true,
      valid: Array.isArray(data),
      items: Array.isArray(data) ? data : [],
    };
  } catch {
    return { exists: true, valid: false, items: [] };
  }
}

function section(text, heading) {
  const lines = text.split(/\r?\n/);
  const start = lines.findIndex(
    (line) => line.trim().toLowerCase() === `## ${heading}`.toLowerCase(),
  );
  if (start === -1) return "";
  const collected = [];
  for (const line of lines.slice(start + 1)) {
    if (/^##\s+/.test(line)) break;
    collected.push(line);
  }
  return collected.join("\n").trim();
}

function bullets(text) {
  const items = [];
  let current = null;
  for (const line of text.split(/\r?\n/)) {
    const bullet = line.match(/^\s*[-*]\s+(.+)$/);
    if (bullet) {
      if (current) items.push(current);
      current = bullet[1].trim();
      continue;
    }
    if (current && /^\s+\S/.test(line)) {
      current = `${current} ${line.trim()}`;
      continue;
    }
    if (current) {
      items.push(current);
      current = null;
    }
  }
  if (current) items.push(current);
  return items;
}

function isNoneBullet(value) {
  return /^none\.?$/i.test(String(value || "").trim());
}

function visualAssertionBullets(text) {
  return bullets(text).filter((bullet) => !isNoneBullet(bullet));
}

function hasBullets(text, heading) {
  return bullets(section(text, heading)).length > 0;
}

function hasVisualAssertionBullets(text) {
  return visualAssertionBullets(section(text, "Visual Assertions")).length > 0;
}

function shouldTrigger(text) {
  const value = section(text, "Should Trigger")
    .trim()
    .split(/\s+/)[0]
    ?.replace(/[.]/g, "")
    .toLowerCase();
  return value !== "no" && value !== "false";
}

function canonicalFixturePath(value) {
  if (
    typeof value !== "string" ||
    !value ||
    value !== value.trim() ||
    value.includes("\\") ||
    value.includes("|") ||
    /[\0-\x1f\x7f]/.test(value) ||
    /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value) ||
    path.posix.isAbsolute(value)
  ) {
    return null;
  }
  const normalized = path.posix.normalize(value);
  if (normalized === "." || normalized === ".." || normalized.startsWith("../")) return null;
  return normalized;
}

function canonicalFixturePaths(text) {
  const fixtures = [...bullets(section(text, "Fixture")), ...bullets(section(text, "Fixtures"))];
  const normalized = fixtures.map(canonicalFixturePath);
  return normalized.every(Boolean) ? [...new Set(normalized)].sort() : null;
}

function expectedSplitFamily(text, caseFile, fixtures) {
  const explicit = section(text, "Split Family").trim();
  if (explicit) return explicit;
  return fixtures.length
    ? `fixture:${fixtures.join("|")}`
    : `case:${path.basename(caseFile, ".md")}`;
}

function expectedArtifactPaths(text, skillName) {
  const explicit = [
    ...bullets(section(text, "Expected Artifact")),
    ...bullets(section(text, "Expected Artifacts")),
  ];
  if (explicit.length) return explicit;
  return walk(path.join(root, "skill-evals", skillName, "expected"), (file) =>
    fs.statSync(file).isFile(),
  )
    .map((file) => path.relative(root, file).replaceAll("\\", "/"))
    .sort();
}

function validSplitFamily(value) {
  if (typeof value !== "string" || value !== value.trim()) return false;
  if (/^[a-z0-9][a-z0-9-]{0,79}$/.test(value)) return true;
  if (/^case:[a-z0-9][a-z0-9-]{0,127}$/.test(value)) return true;
  if (!value.startsWith("fixture:")) return false;
  const fixtures = value.slice("fixture:".length).split("|");
  const canonical = fixtures.map(canonicalFixturePath);
  return (
    fixtures.length > 0 &&
    canonical.every(Boolean) &&
    fixtures.join("|") === [...new Set(canonical)].sort().join("|")
  );
}

function evalCaseCounts(evalDir) {
  const counts = {
    positive: 0,
    negative: 0,
    total: 0,
    positive_with_deterministic_assertions: 0,
    positive_with_visual_assertions: 0,
    positive_with_fixtures: 0,
    positive_with_expected_artifacts: 0,
  };
  if (!evalDir || !fs.existsSync(evalDir)) return counts;
  for (const file of walk(path.join(evalDir, "cases"), (caseFile) => caseFile.endsWith(".md"))) {
    const text = fs.readFileSync(file, "utf8");
    counts.total += 1;
    if (!shouldTrigger(text)) {
      counts.negative += 1;
    } else {
      counts.positive += 1;
      if (hasBullets(text, "Deterministic Assertions")) {
        counts.positive_with_deterministic_assertions += 1;
      }
      if (hasVisualAssertionBullets(text)) {
        counts.positive_with_visual_assertions += 1;
      }
      if (hasBullets(text, "Fixture") || hasBullets(text, "Fixtures")) {
        counts.positive_with_fixtures += 1;
      }
      if (hasBullets(text, "Expected Artifact") || hasBullets(text, "Expected Artifacts")) {
        counts.positive_with_expected_artifacts += 1;
      }
    }
  }
  return counts;
}

function splitCountsInDir(splitDir) {
  const splitInfo = Object.fromEntries(
    ["train", "val", "test"].map((name) => [
      name,
      readJsonArrayInfo(path.join(splitDir, name, "items.json")),
    ]),
  );
  const train = splitInfo.train.items;
  const val = splitInfo.val.items;
  const test = splitInfo.test.items;
  const items = [...train, ...val, ...test];
  const splitEntries = Object.entries(splitInfo).flatMap(([splitName, info]) =>
    info.items.map((item) => ({ item, splitName })),
  );
  const crossingValues = (field) => {
    const memberships = new Map();
    for (const { item, splitName } of splitEntries) {
      const value = item && typeof item === "object" ? item[field] : null;
      if (typeof value !== "string" || !value.trim()) continue;
      if (!memberships.has(value)) memberships.set(value, new Set());
      memberships.get(value).add(splitName);
    }
    return new Set(
      [...memberships.entries()]
        .filter(([, splitNames]) => splitNames.size > 1)
        .map(([value]) => value),
    );
  };
  const crossingGroups = crossingValues("split_group");
  const crossingFamilies = crossingValues("split_family");
  const fixtureMemberships = new Map();
  for (const { item, splitName } of splitEntries) {
    if (!Array.isArray(item?.fixtures)) continue;
    for (const fixture of item.fixtures) {
      if (typeof fixture !== "string" || !fixture.trim()) continue;
      const normalized = canonicalFixturePath(fixture);
      if (!normalized) continue;
      if (!fixtureMemberships.has(normalized)) fixtureMemberships.set(normalized, new Set());
      fixtureMemberships.get(normalized).add(splitName);
    }
  }
  const crossingFixtures = new Set(
    [...fixtureMemberships.entries()]
      .filter(([, splitNames]) => splitNames.size > 1)
      .map(([fixture]) => fixture),
  );
  const validFixtures = (item) =>
    Array.isArray(item.fixtures) &&
    item.fixtures.every((fixture) => {
      const normalized = canonicalFixturePath(fixture);
      return Boolean(normalized) && !crossingFixtures.has(normalized);
    });
  const invalidItems = items.filter(
    (item) =>
      !item ||
      typeof item !== "object" ||
      [...REQUIRED_SPLIT_ITEM_FIELDS].some((field) => !(field in item)) ||
      typeof item.skill_name !== "string" ||
      !/^[a-z0-9][a-z0-9-]{0,127}$/.test(item.skill_name) ||
      typeof item.id !== "string" ||
      !new RegExp(
        `^${item.skill_name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/[a-z0-9][a-z0-9-]{0,127}$`,
      ).test(item.id) ||
      !validSplitFamily(item.split_family) ||
      !/^sha256:[0-9a-f]{64}$/.test(item.split_group || "") ||
      crossingGroups.has(item.split_group) ||
      crossingFamilies.has(item.split_family) ||
      !validFixtures(item) ||
      !SUPPORTED_WORKSPACE_POLICIES.has(item.workspace_policy) ||
      (item.workspace_policy === "isolated-artifact-write") !==
        (item.visual_assertions || []).some((assertion) => String(assertion || "").trim()),
  );
  return {
    train: train.length,
    val: val.length,
    test: test.length,
    positive: items.length,
    positive_with_visual_assertions: items.filter((item) =>
      (item.visual_assertions || []).some((assertion) => String(assertion || "").trim()),
    ).length,
    path: path.relative(root, splitDir).replaceAll("\\", "/"),
    exists: fs.existsSync(splitDir),
    missing_splits: Object.entries(splitInfo)
      .filter(([, info]) => !info.exists)
      .map(([name]) => name),
    invalid_splits: Object.entries(splitInfo)
      .filter(([, info]) => info.exists && !info.valid)
      .map(([name]) => name),
    invalid_item_count: invalidItems.length,
    cross_split_group_count: crossingGroups.size,
    cross_split_family_count: crossingFamilies.size,
    cross_split_fixture_count: crossingFixtures.size,
  };
}

function generatedSplitCounts(skillName, dirName = "data") {
  if (!skillName) {
    return {
      train: 0,
      val: 0,
      test: 0,
      positive: 0,
      positive_with_visual_assertions: 0,
      activation_negative: 0,
      path: null,
      exists: false,
      configured: false,
      configured_path: null,
    };
  }
  const workDir = path.join(root, ".agents/skillopt-work", skillName);
  const split = splitCountsInDir(path.join(workDir, dirName));
  return {
    ...split,
    configured: false,
    configured_path: null,
    activation_negative: readJsonArray(path.join(workDir, "activation/negative-cases.json")).length,
  };
}

function generatedSplitCountsFromConfig(skillName, configInfo) {
  const configured = String(configInfo?.values?.split_dir || "").trim();
  if (!configured || configured.includes("<")) return generatedSplitCounts(skillName);
  const absolute = path.isAbsolute(configured) ? configured : path.resolve(root, configured);
  const split = splitCountsInDir(absolute);
  return {
    ...split,
    configured: true,
    configured_path: configured,
    missing_configured_split: !split.exists,
    activation_negative: skillName
      ? readJsonArray(
          path.join(root, ".agents/skillopt-work", skillName, "activation/negative-cases.json"),
        ).length
      : 0,
  };
}

function activeSplitDir(skillName, configInfo) {
  const configured = String(configInfo?.values?.split_dir || "").trim();
  if (configured && !configured.includes("<")) {
    return path.isAbsolute(configured) ? configured : path.resolve(root, configured);
  }
  return path.join(root, ".agents/skillopt-work", skillName, "data");
}

function skillBody(text) {
  const match = text.match(/^---\n[\s\S]*?\n---\n?/);
  return (match ? text.slice(match[0].length) : text).trimStart();
}

function generatedDataFreshness(skillPath, skillName, configInfo) {
  if (!skillPath || !skillName) {
    return {
      status: "blocked",
      variant: null,
      blockers: ["target skill is unresolved"],
    };
  }
  const splitDir = activeSplitDir(skillName, configInfo);
  const splitInfo = splitCountsInDir(splitDir);
  const expectedTextOnly =
    path.resolve(splitDir) ===
      path.resolve(root, ".agents/skillopt-work", skillName, "data-text-only") ||
    configInfo.values.visual_eval_policy === "text-only";
  const blockers = [];
  if (!splitInfo.exists) blockers.push(`active split directory is missing: ${splitInfo.path}`);
  if (splitInfo.missing_splits.length) {
    blockers.push(`active split is missing items.json for: ${splitInfo.missing_splits.join(", ")}`);
  }
  if (splitInfo.invalid_splits.length) {
    blockers.push(
      `active split has invalid JSON arrays for: ${splitInfo.invalid_splits.join(", ")}`,
    );
  }

  const casesDir = path.join(root, "skill-evals", skillName, "cases");
  const expected = [];
  const expectedActivation = [];
  for (const caseFile of walk(casesDir, (file) => file.endsWith(".md")).sort()) {
    const text = fs.readFileSync(caseFile, "utf8");
    const fixtures = canonicalFixturePaths(text);
    const visualAssertions = visualAssertionBullets(section(text, "Visual Assertions"));
    const expectedItem = {
      id: `${skillName}/${path.basename(caseFile, ".md")}`,
      skill_name: skillName,
      case_path: path.relative(root, caseFile).replaceAll("\\", "/"),
      source_hash: `sha256:${sha256Text(text)}`,
      fixtures,
      split_family: fixtures ? expectedSplitFamily(text, caseFile, fixtures) : null,
      prompt: section(text, "Prompt"),
      expected_behavior: bullets(section(text, "Expected Behavior")),
      deterministic_assertions: bullets(section(text, "Deterministic Assertions")),
      visual_assertions: visualAssertions,
      expected_artifacts: expectedArtifactPaths(text, skillName),
      rubric_path: fs.existsSync(path.join(root, "skill-evals", skillName, "rubric.md"))
        ? `skill-evals/${skillName}/rubric.md`
        : null,
      tags: shouldTrigger(text) ? ["positive"] : ["negative", "activation"],
      should_trigger: shouldTrigger(text),
      workspace_policy: visualAssertions.length ? "isolated-artifact-write" : "text-only",
    };
    if (!shouldTrigger(text)) {
      expectedActivation.push(expectedItem);
      continue;
    }
    if (expectedTextOnly && hasVisualAssertionBullets(text)) continue;
    expected.push(expectedItem);
  }

  const duplicateExpectedIds = (items) => {
    const counts = new Map();
    for (const item of items) counts.set(item.id, (counts.get(item.id) || 0) + 1);
    return [...counts.entries()].filter(([, count]) => count > 1).map(([id]) => id);
  };
  const duplicateSourceIds = duplicateExpectedIds([...expected, ...expectedActivation]);
  if (duplicateSourceIds.length) {
    blockers.push(
      `eval sources contain duplicate case IDs: ${duplicateSourceIds.slice(0, 5).join(", ")}`,
    );
  }

  const generated = ["train", "val", "test"].flatMap((name) =>
    readJsonArray(path.join(splitDir, name, "items.json")),
  );
  const generatedById = new Map();
  for (const item of generated) {
    const id = String(item?.id || "");
    const current = generatedById.get(id) || [];
    current.push(item);
    generatedById.set(id, current);
  }
  const duplicateIds = [...generatedById.entries()]
    .filter(([, items]) => items.length > 1)
    .map(([id]) => id);
  if (duplicateIds.length) {
    blockers.push(
      `active split contains duplicate case IDs: ${duplicateIds.slice(0, 5).join(", ")}`,
    );
  }
  const expectedById = new Map(expected.map((item) => [item.id, item]));
  const missingIds = expected.filter((item) => !generatedById.has(item.id)).map((item) => item.id);
  const unexpectedIds = [...generatedById.keys()].filter((id) => !expectedById.has(id));
  if (missingIds.length) {
    blockers.push(
      `active split is missing current positive cases: ${missingIds.slice(0, 5).join(", ")}`,
    );
  }
  if (unexpectedIds.length) {
    blockers.push(
      `active split contains stale or unexpected cases: ${unexpectedIds.slice(0, 5).join(", ")}`,
    );
  }
  for (const expectedItem of expected) {
    const generatedItem = generatedById.get(expectedItem.id)?.[0];
    if (!generatedItem) continue;
    if (generatedItem.case_path !== expectedItem.case_path) {
      blockers.push(`active split case path is stale for ${expectedItem.id}`);
    }
    if (generatedItem.source_hash !== expectedItem.source_hash) {
      blockers.push(`active split source hash is stale for ${expectedItem.id}`);
    }
    if (
      !expectedItem.fixtures ||
      JSON.stringify(generatedItem.fixtures) !== JSON.stringify(expectedItem.fixtures)
    ) {
      blockers.push(`active split fixtures are stale or invalid for ${expectedItem.id}`);
    }
    if (generatedItem.split_family !== expectedItem.split_family) {
      blockers.push(`active split family is stale for ${expectedItem.id}`);
    }
    for (const field of [
      "prompt",
      "skill_name",
      "expected_behavior",
      "deterministic_assertions",
      "visual_assertions",
      "expected_artifacts",
      "rubric_path",
      "tags",
      "should_trigger",
      "workspace_policy",
    ]) {
      if (JSON.stringify(generatedItem[field]) !== JSON.stringify(expectedItem[field])) {
        blockers.push(`active split ${field} is stale for ${expectedItem.id}`);
      }
    }
  }

  const activationPath = path.join(
    root,
    ".agents/skillopt-work",
    skillName,
    "activation/negative-cases.json",
  );
  const activationInfo = readJsonArrayInfo(activationPath);
  if (!activationInfo.exists) blockers.push("activation negative-cases.json is missing");
  else if (!activationInfo.valid)
    blockers.push("activation negative-cases.json is not a JSON array");
  const generatedActivationById = new Map();
  for (const item of activationInfo.items) {
    const id = String(item?.id || "");
    const current = generatedActivationById.get(id) || [];
    current.push(item);
    generatedActivationById.set(id, current);
  }
  const duplicateActivationIds = [...generatedActivationById.entries()]
    .filter(([, items]) => items.length > 1)
    .map(([id]) => id);
  if (duplicateActivationIds.length) {
    blockers.push(
      `activation split contains duplicate case IDs: ${duplicateActivationIds.slice(0, 5).join(", ")}`,
    );
  }
  const expectedActivationById = new Map(expectedActivation.map((item) => [item.id, item]));
  const missingActivationIds = expectedActivation
    .filter((item) => !generatedActivationById.has(item.id))
    .map((item) => item.id);
  const unexpectedActivationIds = [...generatedActivationById.keys()].filter(
    (id) => !expectedActivationById.has(id),
  );
  if (missingActivationIds.length) {
    blockers.push(
      `activation split is missing current negative cases: ${missingActivationIds.slice(0, 5).join(", ")}`,
    );
  }
  if (unexpectedActivationIds.length) {
    blockers.push(
      `activation split contains stale or unexpected cases: ${unexpectedActivationIds.slice(0, 5).join(", ")}`,
    );
  }
  for (const expectedItem of expectedActivation) {
    const generatedItem = generatedActivationById.get(expectedItem.id)?.[0];
    if (!generatedItem) continue;
    if (generatedItem.case_path !== expectedItem.case_path) {
      blockers.push(`activation case path is stale for ${expectedItem.id}`);
    }
    if (generatedItem.source_hash !== expectedItem.source_hash) {
      blockers.push(`activation source hash is stale for ${expectedItem.id}`);
    }
    for (const field of [
      "prompt",
      "skill_name",
      "expected_behavior",
      "deterministic_assertions",
      "visual_assertions",
      "expected_artifacts",
      "rubric_path",
      "tags",
      "should_trigger",
      "workspace_policy",
      "fixtures",
      "split_family",
    ]) {
      if (JSON.stringify(generatedItem[field]) !== JSON.stringify(expectedItem[field])) {
        blockers.push(`activation ${field} is stale for ${expectedItem.id}`);
      }
    }
  }

  const workDir = path.join(root, ".agents/skillopt-work", skillName);
  const currentSkill = fs.readFileSync(skillPath, "utf8");
  const expectedSkillHash = `sha256:${sha256Text(currentSkill)}`;
  const recordedHashPath = path.join(workDir, "initial/initial-skill.sha256");
  const initialBodyPath = path.join(workDir, "initial/skill-body.md");
  const recordedHash = fs.existsSync(recordedHashPath)
    ? fs.readFileSync(recordedHashPath, "utf8").trim()
    : "";
  if (recordedHash !== expectedSkillHash)
    blockers.push("initial skill checksum is stale or missing");
  const recordedBody = fs.existsSync(initialBodyPath)
    ? fs.readFileSync(initialBodyPath, "utf8")
    : null;
  if (recordedBody !== skillBody(currentSkill))
    blockers.push("initial skill body is stale or missing");

  return {
    status: blockers.length ? "refresh_required" : "matched",
    variant: expectedTextOnly ? "text-only" : "full",
    path: splitInfo.path,
    expected_positive_cases: expected.length,
    generated_positive_cases: generated.length,
    expected_activation_negative_cases: expectedActivation.length,
    generated_activation_negative_cases: activationInfo.items.length,
    blockers: [...new Set(blockers)],
  };
}

function estimateSplitCounts(positive) {
  if (positive <= 0) return { train: 0, val: 0, test: 0 };
  if (positive === 1) return { train: 1, val: 0, test: 0 };
  if (positive === 2) return { train: 1, val: 1, test: 0 };
  let train = Math.round(positive * 0.6);
  let val = Math.round(positive * 0.2);
  let test = positive - train - val;
  const heldoutFloor =
    positive >= OFFICIAL_RECOMMENDED.positive
      ? OFFICIAL_RECOMMENDED.val
      : positive >= EXPLORATORY_MIN.positive
        ? EXPLORATORY_MIN.val
        : 1;
  val = Math.max(heldoutFloor, val);
  test = Math.max(heldoutFloor, test);
  if (val + test > positive - 1) {
    const availableHeldout = positive - 1;
    val = Math.floor(availableHeldout / 2);
    test = availableHeldout - val;
  }
  train = positive - val - test;
  return { train, val, test };
}

function readText(file) {
  return fs.existsSync(file) ? fs.readFileSync(file, "utf8") : "";
}

function extractYamlValue(text, key) {
  const match = text.match(new RegExp(`^[ \\t]*${key}:[ \\t]*(.*?)[ \\t]*$`, "m"));
  return match?.[1]?.trim().replace(/^["']|["']$/g, "") || "";
}

function extractYamlLeafValue(text, key) {
  const matches = [...text.matchAll(new RegExp(`^[ \\t]+${key}:[ \\t]*(.*?)[ \\t]*$`, "gm"))];
  const value = matches
    .at(-1)?.[1]
    ?.trim()
    .replace(/^["']|["']$/g, "");
  return value || "";
}

function configPathForMode(skillName, mode) {
  const workConfig = skillName
    ? path.join(root, ".agents/skillopt-work", skillName, "configs", `agent-skills.${mode}.yaml`)
    : null;
  if (workConfig && fs.existsSync(workConfig)) return workConfig;
  return path.join(skillRoot, "assets/agent-skills-benchmark", `config.${mode}.yaml`);
}

function pinState(value) {
  const clean = String(value || "").trim();
  const envMatch = clean.match(/^\$\{([A-Z0-9_]+)\}$/);
  if (!clean) return { value: "", status: "inherited_default" };
  if (envMatch) {
    return {
      value: clean,
      status: process.env[envMatch[1]] ? "env_present" : "env_missing",
      env: envMatch[1],
    };
  }
  return { value: clean, status: "pinned" };
}

function configProfile(skillName, mode) {
  const configPath = configPathForMode(skillName, mode);
  const text = readText(configPath);
  const values = {
    run_profile: extractYamlValue(text, "run_profile") || defaultRunProfile(mode),
    env_out_root: extractYamlValue(text, "out_root"),
    split_dir: extractYamlValue(text, "split_dir"),
    visual_eval_policy: extractYamlValue(text, "visual_eval_policy"),
    train_num_epochs: extractYamlValue(text, "num_epochs"),
    train_batch_size: extractYamlValue(text, "batch_size"),
    workers: extractYamlValue(text, "workers"),
    tool_rollout_for_visual_assertions: extractYamlValue(
      text,
      "tool_rollout_for_visual_assertions",
    ),
    require_drawio_cli_for_visual_rollouts: extractYamlValue(
      text,
      "require_drawio_cli_for_visual_rollouts",
    ),
    visual_exec_timeout: extractYamlValue(text, "visual_exec_timeout"),
    gradient_minibatch_size: extractYamlValue(text, "minibatch_size"),
    gradient_analyst_workers: extractYamlValue(text, "analyst_workers"),
    optimizer_learning_rate: extractYamlValue(text, "learning_rate"),
    optimizer_min_learning_rate: extractYamlValue(text, "min_learning_rate"),
    optimizer_lr_scheduler: extractYamlValue(text, "lr_scheduler"),
    optimizer_use_slow_update: extractYamlValue(text, "use_slow_update"),
    optimizer_slow_update_samples: extractYamlValue(text, "slow_update_samples"),
    optimizer_slow_update_gate_with_selection: extractYamlValue(
      text,
      "slow_update_gate_with_selection",
    ),
    optimizer_use_meta_skill: extractYamlValue(text, "use_meta_skill"),
    optimizer_use_skill_aware_reflection: extractYamlValue(text, "use_skill_aware_reflection"),
    optimizer_skill_aware_appendix_source: extractYamlValue(text, "skill_aware_appendix_source"),
    optimizer_skill_aware_consolidate_threshold: extractYamlValue(
      text,
      "skill_aware_consolidate_threshold",
    ),
    evaluation_use_gate: extractYamlValue(text, "use_gate"),
    evaluation_eval_test: extractYamlValue(text, "eval_test"),
    target_backend: extractYamlValue(text, "target_backend"),
    optimizer_backend: extractYamlValue(text, "optimizer_backend"),
    judge_backend: extractYamlValue(text, "judge_backend"),
    reflection_backend: extractYamlValue(text, "reflection_backend"),
    codex_exec_path: extractYamlValue(text, "codex_exec_path") || "codex",
    scale_down_reason:
      mode === "hybrid-codex-target"
        ? "Hybrid Codex CLI target rollouts use smaller batch/workers than the upstream provider-backed default."
        : "",
  };
  if (values.run_profile === "<run-profile>") values.run_profile = defaultRunProfile(mode);
  const modelPins = {
    optimizer: pinState(extractYamlLeafValue(text, "optimizer")),
    target: pinState(extractYamlLeafValue(text, "target")),
    codex_cli_judge_model: pinState(extractYamlLeafValue(text, "codex_cli_judge_model")),
    codex_cli_reflection_model: pinState(extractYamlLeafValue(text, "codex_cli_reflection_model")),
  };
  return {
    path: path.relative(root, configPath).replaceAll("\\", "/"),
    values,
    modelPins,
  };
}

function parseYamlKeys(text) {
  const keys = [];
  const stack = [];
  for (const line of text.split(/\r?\n/)) {
    const match = line.match(/^(\s*)([A-Za-z0-9_<>-]+):/);
    if (!match) continue;
    const indent = match[1].length;
    const key = match[2];
    while (stack.length && stack.at(-1).indent >= indent) stack.pop();
    const parent = stack.at(-1)?.path;
    const current = parent ? `${parent}.${key}` : key;
    keys.push({ key, path: current, indent });
    stack.push({ indent, path: current });
  }
  return keys;
}

function parseCliOptions(helpText) {
  return new Set([...String(helpText || "").matchAll(/--([a-z0-9_]+)/g)].map((match) => match[1]));
}

function localPythonCommand(skillOptPath) {
  const venv =
    process.platform === "win32"
      ? path.join(skillOptPath, ".venv", "Scripts", "python.exe")
      : path.join(skillOptPath, ".venv", "bin", "python");
  if (fs.existsSync(venv)) return { command: venv, argsPrefix: [] };
  const uv = resolveUvCommand();
  if (uv) return { command: uv, argsPrefix: ["run", "python"] };
  return null;
}

function localHelpOptions(skillOptPath, script) {
  const python = localPythonCommand(skillOptPath);
  if (!python)
    return {
      ok: false,
      options: new Set(),
      error: "uv or SkillOpt virtualenv unavailable",
    };
  const result = commandResult(python.command, [...python.argsPrefix, script, "--help"], {
    cwd: skillOptPath,
    timeout: 60000,
  });
  if (!result.ok) {
    return {
      ok: false,
      options: new Set(),
      error: result.stderr || result.stdout || "help failed",
    };
  }
  return {
    ok: true,
    options: parseCliOptions(result.stdout || result.stderr),
    error: null,
  };
}

function adapterConfigKeys(skillOptPath) {
  const candidates = [
    path.join(skillOptPath, "skillopt/envs/agent_skills/adapter.py"),
    path.join(skillRoot, "assets/agent-skills-benchmark/adapter.py.template"),
  ];
  const keys = new Set([
    "_base_",
    "env",
    "train",
    "gradient",
    "optimizer",
    "evaluation",
    "model",
    "name",
    "run_profile",
  ]);
  for (const file of candidates) {
    const text = readText(file);
    if (!text) continue;
    for (const match of text.matchAll(/^\s{8}([a-zA-Z_][a-zA-Z0-9_]*)[=,]/gm)) {
      keys.add(match[1]);
    }
    for (const match of text.matchAll(/"([a-zA-Z_][a-zA-Z0-9_]*)"/g)) {
      keys.add(match[1]);
    }
  }
  return keys;
}

function structuredConfigPaths(skillOptPath) {
  const configPy = path.join(skillOptPath, "skillopt/config.py");
  const text = readText(configPy);
  return new Set([...text.matchAll(/"([a-z]+\.[a-zA-Z0-9_]+)"\s*:/g)].map((match) => match[1]));
}

function configSchemaCheck(skillName, mode) {
  const skillOptPath = path.join(root, ".agents/tools/SkillOpt");
  const configPath = configPathForMode(skillName, mode);
  const text = readText(configPath);
  if (!text) {
    return {
      status: "missing_config",
      configPath: path.relative(root, configPath).replaceAll("\\", "/"),
      unsupportedKeys: [],
      checkedAgainst: [],
    };
  }
  if (!fs.existsSync(skillOptPath)) {
    return {
      status: "unavailable",
      configPath: path.relative(root, configPath).replaceAll("\\", "/"),
      unsupportedKeys: [],
      checkedAgainst: ["tracked adapter template"],
      reason: "local SkillOpt clone is missing",
    };
  }

  const trainHelp = localHelpOptions(skillOptPath, "scripts/train.py");
  const evalHelp = localHelpOptions(skillOptPath, "scripts/eval_only.py");
  const supported = adapterConfigKeys(skillOptPath);
  const supportedPaths = structuredConfigPaths(skillOptPath);
  for (const option of trainHelp.options) supported.add(option);
  for (const option of evalHelp.options) supported.add(option);

  const leafKeys = parseYamlKeys(text).filter((entry) => {
    const prefix = `${entry.path}.`;
    return !parseYamlKeys(text).some((other) => other.path.startsWith(prefix));
  });
  const unsupportedKeys = leafKeys
    .map((entry) => ({ key: entry.key, path: entry.path }))
    .filter((entry) => !supported.has(entry.key) && !supportedPaths.has(entry.path));
  return {
    status: unsupportedKeys.length ? "review_required" : "passed",
    configPath: path.relative(root, configPath).replaceAll("\\", "/"),
    unsupportedKeys,
    checkedAgainst: [
      trainHelp.ok ? "local scripts/train.py --help" : `train help unavailable: ${trainHelp.error}`,
      evalHelp.ok
        ? "local scripts/eval_only.py --help"
        : `eval help unavailable: ${evalHelp.error}`,
      "agent_skills adapter constructor/template keys",
    ],
  };
}

function splitCountsFor(datasetCounts) {
  const active = datasetCounts.generated_active;
  if (active?.configured) return active;
  if (active?.exists && active.train + active.val + active.test) return active;
  const generated = datasetCounts.generated;
  return generated.train + generated.val + generated.test
    ? generated
    : datasetCounts.estimated_split;
}

function benchmarkQuality(datasetCounts) {
  const split = splitCountsFor(datasetCounts);
  const positiveCases = Number.isFinite(Number(split.positive))
    ? Number(split.positive)
    : datasetCounts.eval_positive;
  const officialFloorMet =
    positiveCases >= OFFICIAL_RECOMMENDED.positive &&
    split.val >= OFFICIAL_RECOMMENDED.val &&
    split.test >= OFFICIAL_RECOMMENDED.test;
  const exploratoryFloorMet =
    positiveCases >= EXPLORATORY_MIN.positive &&
    split.val >= EXPLORATORY_MIN.val &&
    split.test >= EXPLORATORY_MIN.test;
  const blockers = [];
  if (positiveCases < OFFICIAL_RECOMMENDED.positive) {
    blockers.push(
      `needs ${OFFICIAL_RECOMMENDED.positive}+ active positive cases; found ${positiveCases}`,
    );
  }
  if (split.val < OFFICIAL_RECOMMENDED.val) {
    blockers.push(`needs ${OFFICIAL_RECOMMENDED.val}+ validation cases; found ${split.val}`);
  }
  if (split.test < OFFICIAL_RECOMMENDED.test) {
    blockers.push(`needs ${OFFICIAL_RECOMMENDED.test}+ test cases; found ${split.test}`);
  }
  return {
    classification: officialFloorMet
      ? "official-parity-candidate"
      : exploratoryFloorMet
        ? "exploratory-candidate"
        : "insufficient",
    officialFloorMet,
    exploratoryFloorMet,
    splitCounts: split,
    activePositiveCases: positiveCases,
    sourcePositiveCases: datasetCounts.eval_positive,
    activeSplitCounts: datasetCounts.generated_active || null,
    fullSplitCounts: datasetCounts.generated || null,
    thresholds: {
      exploratoryMinimum: EXPLORATORY_MIN,
      officialRecommended: OFFICIAL_RECOMMENDED,
    },
    positiveWithDeterministicAssertions: datasetCounts.positive_with_deterministic_assertions || 0,
    positiveWithVisualAssertions: datasetCounts.positive_with_visual_assertions || 0,
    positiveWithFixtures: datasetCounts.positive_with_fixtures || 0,
    positiveWithExpectedArtifacts: datasetCounts.positive_with_expected_artifacts || 0,
    blockers,
  };
}

function runnableSplitBlockers(datasetCounts, configInfo) {
  const split = datasetCounts.generated_active || datasetCounts.generated;
  const blockers = [];
  if (!split?.exists)
    blockers.push(`active split directory is missing: ${split?.path || "unknown"}`);
  for (const name of split?.missing_splits || []) {
    blockers.push(`active split is missing ${name}/items.json`);
  }
  for (const name of split?.invalid_splits || []) {
    blockers.push(`active split ${name}/items.json is not a JSON array`);
  }
  if (split?.invalid_item_count) {
    blockers.push(
      `active split contains ${split.invalid_item_count} item(s) with missing required fields`,
    );
  }
  if (split?.cross_split_group_count) {
    blockers.push(
      `active split leaks ${split.cross_split_group_count} split_group value(s) across train/validation/test`,
    );
  }
  if (split?.cross_split_family_count) {
    blockers.push(
      `active split leaks ${split.cross_split_family_count} split_family value(s) across train/validation/test`,
    );
  }
  if (split?.cross_split_fixture_count) {
    blockers.push(
      `active split leaks ${split.cross_split_fixture_count} normalized fixture path(s) across train/validation/test`,
    );
  }
  if (!split || split.train < 1) blockers.push("active split needs at least one training case");
  if (configInfo.values.evaluation_use_gate !== "false" && (!split || split.val < 1)) {
    blockers.push(
      "active split needs at least one validation case while the evaluation gate is enabled",
    );
  }
  if (configInfo.values.evaluation_eval_test !== "false" && (!split || split.test < 1)) {
    blockers.push("active split needs at least one test case while test evaluation is enabled");
  }
  return [...new Set(blockers)];
}

function artifactExpectations(skillName, configInfo) {
  const outRoot =
    configInfo.values.env_out_root ||
    (skillName ? `.agents/skillopt-work/${skillName}/outputs/run-unknown` : "");
  return [
    "config.json",
    "history.json",
    "runtime_state.json",
    "best_skill.md",
    "steps/",
    "skills/",
    "training.log",
  ].map((name) => `${outRoot.replace(/\/$/, "")}/${name}`);
}

function skillOptCommit(skillOptPath, commitPath) {
  if (fs.existsSync(commitPath)) {
    const recorded = fs.readFileSync(commitPath, "utf8").trim();
    if (/^[a-f0-9]{40}$/i.test(recorded)) return recorded;
  }
  const head = path.join(skillOptPath, ".git/HEAD");
  if (!fs.existsSync(head)) return null;
  const value = fs.readFileSync(head, "utf8").trim();
  if (!value.startsWith("ref: ")) return value;
  const refFile = path.join(skillOptPath, ".git", value.slice(5));
  return fs.existsSync(refFile) ? fs.readFileSync(refFile, "utf8").trim() : value;
}

function upstreamBehaviorBypassed(mode) {
  if (mode !== "codex-cli-all") return [];
  return [
    "provider-backed target rollout",
    "provider-backed semantic judging",
    "provider-backed reflection",
    "provider-backed patch aggregation",
    "provider-backed patch ranking",
    "provider-backed slow update (keep optimizer.use_slow_update disabled in codex-cli-all)",
    "provider-backed meta skill (keep optimizer.use_meta_skill disabled in codex-cli-all)",
  ];
}

function readAdapterManifest(skillName) {
  const targetPath = skillName
    ? path.join(root, ".agents/skillopt-work", skillName, "adapter-manifest.json")
    : null;
  const legacyPath = path.join(root, ".agents/skillopt-work/adapter-manifest.json");
  const manifestPath = targetPath && fs.existsSync(targetPath) ? targetPath : legacyPath;
  if (!fs.existsSync(manifestPath)) return { path: null, data: null, target_specific: false };
  try {
    return {
      path: path.relative(root, manifestPath).replaceAll("\\", "/"),
      data: JSON.parse(fs.readFileSync(manifestPath, "utf8")),
      target_specific: targetPath === manifestPath,
    };
  } catch {
    return {
      path: path.relative(root, manifestPath).replaceAll("\\", "/"),
      data: null,
      target_specific: targetPath === manifestPath,
    };
  }
}

function sha256File(file) {
  return createHash("sha256").update(fs.readFileSync(file)).digest("hex");
}

function sha256Text(text) {
  return createHash("sha256").update(text).digest("hex");
}

function templateHashWarnings(manifest) {
  const sources = manifest.template_sources || manifest.templateSources;
  if (!sources || typeof sources !== "object" || Array.isArray(sources)) {
    return [
      "Adapter manifest predates current-template freshness tracking; production setup must refresh it before training.",
    ];
  }
  const warnings = [];
  for (const [rel, sourceSpec] of Object.entries(sources)) {
    const file = path.join(root, rel);
    if (!fs.existsSync(file)) {
      warnings.push(`Adapter installed file is missing: ${rel}`);
      continue;
    }
    const sourceRel = typeof sourceSpec === "string" ? sourceSpec : sourceSpec?.source;
    if (!sourceRel || typeof sourceRel !== "string") {
      warnings.push(`Adapter manifest has no tracked source for installed file: ${rel}`);
      continue;
    }
    const source = path.join(root, sourceRel);
    if (!fs.existsSync(source)) {
      warnings.push(`Adapter tracked source template is missing: ${sourceRel}`);
      continue;
    }
    let expectedText = fs.readFileSync(source, "utf8");
    const replacements =
      typeof sourceSpec === "object" && sourceSpec && !Array.isArray(sourceSpec)
        ? sourceSpec.replacements || {}
        : {};
    for (const [from, to] of Object.entries(replacements)) {
      expectedText = expectedText.replaceAll(from, String(to));
    }
    const actualHash = sha256File(file);
    if (actualHash !== sha256Text(expectedText)) {
      warnings.push(`Adapter installed file differs from the current tracked template: ${rel}`);
    }
  }
  return warnings;
}

function actualSkillOptCommit(skillOptPath) {
  if (!fs.existsSync(skillOptPath)) return null;
  const result = commandResult("git", ["rev-parse", "HEAD"], {
    cwd: skillOptPath,
  });
  return result.ok && /^[a-f0-9]{40}$/i.test(result.stdout) ? result.stdout : null;
}

function pythonAstPatchProof(skillOptPath, pythonCommand) {
  const relativeFiles = {
    registry_train: "scripts/train.py",
    registry_eval: "scripts/eval_only.py",
    safe_env: "skillopt/config.py",
    trainer_steps: "skillopt/engine/trainer.py",
  };
  if (Object.values(relativeFiles).some((file) => !fs.existsSync(path.join(skillOptPath, file)))) {
    return {
      status: "failed",
      diagnostic: "required_file_missing",
      checks: {},
    };
  }
  const env = Object.fromEntries(
    ["PATH", "HOME", "USER", "LOGNAME", "LANG", "LC_ALL", "LC_CTYPE"].flatMap((name) =>
      process.env[name] ? [[name, process.env[name]]] : [],
    ),
  );
  const result = spawnSync(
    pythonCommand,
    [
      "-c",
      PYTHON_PATCH_AST_PROBE,
      JSON.stringify(SAFE_MODEL_ENV_PLACEHOLDERS),
      ...Object.values(relativeFiles).map((file) => path.join(skillOptPath, file)),
    ],
    {
      cwd: skillOptPath,
      encoding: "utf8",
      timeout: 5000,
      env,
      input: "",
    },
  );
  if (result.status !== 0) {
    const diagnostic =
      result.error?.code === "ENOENT"
        ? "python_unavailable"
        : result.error?.code === "ETIMEDOUT"
          ? "probe_timed_out"
          : "probe_failed";
    return { status: "failed", diagnostic, checks: {} };
  }
  try {
    const parsed = JSON.parse(result.stdout);
    if (!parsed || typeof parsed.checks !== "object" || Array.isArray(parsed.checks)) {
      return {
        status: "failed",
        diagnostic: "invalid_probe_output",
        checks: {},
      };
    }
    return {
      status: "completed",
      diagnostic: Object.keys(parsed.errors || {}).length ? "unparseable_python" : null,
      checks: parsed.checks,
    };
  } catch {
    return { status: "failed", diagnostic: "invalid_probe_output", checks: {} };
  }
}

function liveAdapterPatchCheck(skillOptPath, manifest, pythonCommand) {
  const blockers = [];
  const files = [];
  const proof = pythonAstPatchProof(skillOptPath, pythonCommand);
  function check(relativeFile, proofKey, missingMessage) {
    const file = path.join(skillOptPath, relativeFile);
    if (!fs.existsSync(file)) {
      blockers.push(`${relativeFile} is missing from the live SkillOpt clone`);
      files.push({ file: relativeFile, status: "missing" });
      return;
    }
    const passed = proof.status === "completed" && proof.checks[proofKey] === true;
    files.push({
      file: relativeFile,
      status: passed ? "matched" : "refresh_required",
    });
    if (!passed) blockers.push(missingMessage);
  }

  check(
    "scripts/train.py",
    "registry_train",
    "scripts/train.py is missing the structural live agent_skills registry patch",
  );
  check(
    "scripts/eval_only.py",
    "registry_eval",
    "scripts/eval_only.py is missing the structural live agent_skills registry patch",
  );
  check(
    "skillopt/config.py",
    "safe_env",
    "skillopt/config.py is missing the structural safe model environment expansion patch",
  );
  check(
    "skillopt/engine/trainer.py",
    "trainer_steps",
    "skillopt/engine/trainer.py is missing the structural configured steps-per-epoch patch",
  );
  if (proof.status !== "completed") {
    blockers.push(`live SkillOpt structural patch proof failed: ${proof.diagnostic}`);
  }

  const manifestCommit = manifest?.skillopt_commit || manifest?.skilloptCommit || null;
  const liveCommit = actualSkillOptCommit(skillOptPath);
  if (!manifestCommit) blockers.push("adapter manifest is missing the SkillOpt commit identity");
  if (!liveCommit) blockers.push("live SkillOpt commit could not be resolved");
  if (manifestCommit && liveCommit && manifestCommit !== liveCommit) {
    blockers.push("adapter manifest SkillOpt commit does not match the live clone");
  }
  return {
    status: blockers.length ? "refresh_required" : "matched",
    manifest_commit: manifestCommit,
    live_commit: liveCommit,
    proof_method: "python_ast",
    proof_status: proof.status,
    proof_diagnostic: proof.diagnostic,
    files,
    blockers: [...new Set(blockers)],
  };
}

function requiredModelPinNames(mode, configInfo) {
  if (mode === "native-provider") return ["optimizer", "target"];
  if (mode === "hybrid-codex-target") return ["optimizer", "target", "codex_cli_judge_model"];
  if (configInfo.values.judge_backend === "codex_cli") {
    return ["target", "codex_cli_judge_model", "codex_cli_reflection_model"];
  }
  return [];
}

function semanticJudgeReadiness(args, configInfo) {
  const runProfile = args.mode === "codex-cli-all" ? "exploratory" : args.runProfile;
  const backend = String(configInfo.values.judge_backend || "heuristic")
    .trim()
    .toLowerCase();
  const officialParityRequired = runProfile === "official-parity";
  const supported = ["provider", "codex_cli"].includes(backend);
  const blockers =
    officialParityRequired && !supported
      ? [
          `official-parity runs require semantic judge_backend=provider or codex_cli; ${backend || "heuristic"} does not provide semantic judging`,
        ]
      : [];
  return {
    status: blockers.length ? "blocked" : supported ? "ready" : "exploratory_only",
    backend,
    officialParityRequired,
    blockers,
  };
}

function officialParityReport(
  args,
  providerOk,
  datasetCounts,
  configInfo,
  quality,
  schemaCheck,
  semanticJudge,
) {
  const gaps = [];
  const proofBlockers = [];
  const bypassed = upstreamBehaviorBypassed(args.mode);
  const effectiveProfile = args.mode === "codex-cli-all" ? "exploratory" : args.runProfile;
  const split = quality.splitCounts;
  const activePositiveCases = quality.activePositiveCases ?? datasetCounts.eval_positive;

  if (args.mode === "codex-cli-all") {
    gaps.push(
      "codex-cli-all is provider-free exploratory mode, not upstream-native optimizer parity",
    );
  }
  if (effectiveProfile !== "official-parity") gaps.push("run profile is exploratory");
  if (effectiveProfile === "official-parity" && !providerOk) {
    proofBlockers.push(
      "provider-backed optimizer credentials or endpoint preflight are incomplete",
    );
  }
  proofBlockers.push(...semanticJudge.blockers);
  if (semanticJudge.status === "exploratory_only") {
    gaps.push(
      `judge backend ${semanticJudge.backend} is non-semantic and is allowed only for exploratory runs`,
    );
  }
  if (activePositiveCases < EXPLORATORY_MIN.positive) {
    gaps.push(
      `only ${activePositiveCases} active positive eval case(s); ${EXPLORATORY_MIN.positive}+ recommended before non-exploratory use`,
    );
  }
  if (split.val < EXPLORATORY_MIN.val || split.test < EXPLORATORY_MIN.test) {
    gaps.push(
      `split has val=${split.val}, test=${split.test}; ${EXPLORATORY_MIN.val}+ val and ${EXPLORATORY_MIN.test}+ test cases recommended before non-exploratory use`,
    );
  }
  if (activePositiveCases < OFFICIAL_RECOMMENDED.positive) {
    gaps.push(
      `below official-parity recommendation of ${OFFICIAL_RECOMMENDED.positive}+ positive cases`,
    );
  }
  if (split.val < OFFICIAL_RECOMMENDED.val || split.test < OFFICIAL_RECOMMENDED.test) {
    gaps.push(
      `below official-parity split recommendation of ${OFFICIAL_RECOMMENDED.val}+ validation and ${OFFICIAL_RECOMMENDED.test}+ test cases`,
    );
  }
  if (effectiveProfile === "official-parity" && !quality.officialFloorMet) {
    proofBlockers.push(...quality.blockers.map((blocker) => `dataset floor: ${blocker}`));
  }
  const requiredPins =
    effectiveProfile === "official-parity" ? requiredModelPinNames(args.mode, configInfo) : [];
  const gapPins = new Set([
    ...requiredPins,
    ...(args.mode === "codex-cli-all"
      ? ["target", "codex_cli_judge_model", "codex_cli_reflection_model"]
      : []),
  ]);
  for (const [key, state] of Object.entries(configInfo.modelPins)) {
    if (!gapPins.has(key) && !state.value) continue;
    if (state.status === "inherited_default") {
      gaps.push(`${key} model is blank and will inherit runtime defaults`);
      if (requiredPins.includes(key)) proofBlockers.push(`${key} model pin is required`);
    } else if (state.status === "env_missing") {
      gaps.push(`${key} model uses ${state.value}, but ${state.env} is not present`);
      if (requiredPins.includes(key))
        proofBlockers.push(`${key} model env ${state.env} is missing`);
    }
  }
  if (schemaCheck.status === "review_required") {
    proofBlockers.push(
      `generated config has unsupported keys: ${schemaCheck.unsupportedKeys.map((item) => item.path).join(", ")}`,
    );
  }
  if (schemaCheck.status === "missing_config") {
    proofBlockers.push("generated config is missing");
  }
  if (
    configInfo.values.optimizer_lr_scheduler &&
    configInfo.values.optimizer_lr_scheduler !== "cosine"
  ) {
    gaps.push(`optimizer.lr_scheduler is ${configInfo.values.optimizer_lr_scheduler}, not cosine`);
  }
  if (configInfo.values.optimizer_use_slow_update === "false") {
    gaps.push("optimizer.use_slow_update is disabled");
  }
  if (configInfo.values.optimizer_use_meta_skill === "false") {
    gaps.push("optimizer.use_meta_skill is disabled");
  }

  let status = "ready";
  if (args.mode === "codex-cli-all" || effectiveProfile === "exploratory") status = "exploratory";
  else if (proofBlockers.length) {
    status = "blocked";
  } else if (gaps.length) {
    status = "ready_with_gaps";
  }

  return {
    requestedRunProfile: args.runProfile,
    runProfile: effectiveProfile,
    officialParityStatus: status,
    proofStatus: status,
    officialParityGaps: gaps,
    proofBlockers: [...new Set(proofBlockers)],
    upstreamBehaviorBypassed: bypassed,
  };
}

function cleanEnv(name) {
  return String(process.env[name] || "").trim();
}

function roleAzureConfig(role) {
  const upper = role.toUpperCase();
  const alternate = role === "optimizer" ? "OPTIMIZER" : "TARGET";
  const endpoint =
    cleanEnv(`${upper}_AZURE_OPENAI_ENDPOINT`) ||
    cleanEnv(`AZURE_OPENAI_${alternate}_ENDPOINT`) ||
    cleanEnv("AZURE_OPENAI_ENDPOINT");
  const apiKey =
    cleanEnv(`${upper}_AZURE_OPENAI_API_KEY`) ||
    cleanEnv(`AZURE_OPENAI_${alternate}_API_KEY`) ||
    cleanEnv("AZURE_OPENAI_API_KEY");
  const authMode = (
    cleanEnv(`${upper}_AZURE_OPENAI_AUTH_MODE`) ||
    cleanEnv(`AZURE_OPENAI_${alternate}_AUTH_MODE`) ||
    cleanEnv("AZURE_OPENAI_AUTH_MODE") ||
    "azure_cli"
  ).toLowerCase();
  return { endpoint, apiKey, authMode };
}

function resolvedModelPin(role, configInfo) {
  const state = configInfo.modelPins[role];
  if (!state) return "";
  if (state.env) return cleanEnv(state.env);
  return state.status === "pinned" ? state.value : "";
}

function providerRoleConfig(role, backend, configInfo) {
  const normalizedBackend = String(backend || "openai_chat")
    .trim()
    .toLowerCase();
  const blockers = [];
  let endpointProbe = null;
  const publicStatus = {
    role,
    backend: normalizedBackend,
    configured: false,
    endpoint_present: false,
    credential_present: false,
    auth_mode: null,
    blockers,
  };

  if (normalizedBackend === "codex_exec") {
    if (role === "target") publicStatus.configured = true;
    else blockers.push(`${role} codex_exec is unsupported for provider optimizer work`);
    return { publicStatus, endpointProbe };
  }

  if (normalizedBackend === "claude_code_exec") {
    blockers.push(`${role} claude_code_exec is unsupported by the Agent Skills rollout adapter`);
    publicStatus.auth_mode = "unsupported_local_cli";
    return { publicStatus, endpointProbe };
  }

  if (normalizedBackend === "openai_chat") {
    const azure = roleAzureConfig(role);
    const supportedAuthMode =
      ["api_key", "key"].includes(azure.authMode) ||
      TOKENLESS_AZURE_AUTH_MODES.has(azure.authMode) ||
      OPENAI_COMPATIBLE_AUTH_MODES.has(azure.authMode);
    publicStatus.endpoint_present = Boolean(azure.endpoint);
    publicStatus.credential_present = Boolean(azure.apiKey);
    publicStatus.auth_mode = supportedAuthMode ? azure.authMode : "unsupported";
    if (!azure.endpoint) blockers.push(`${role} openai_chat endpoint is missing`);
    if (["api_key", "key"].includes(azure.authMode) && !azure.apiKey) {
      blockers.push(`${role} openai_chat API key is missing`);
    } else if (azure.authMode === "azure_cli" && !commandExists("az")) {
      blockers.push(`${role} openai_chat requires Azure CLI for azure_cli auth`);
    } else if (!supportedAuthMode) {
      blockers.push(`${role} openai_chat auth mode is unsupported`);
    }
    if (azure.endpoint && OPENAI_COMPATIBLE_AUTH_MODES.has(azure.authMode)) {
      endpointProbe = {
        role,
        endpoint: azure.endpoint,
        apiKey: azure.apiKey,
        model: resolvedModelPin(role, configInfo),
      };
    }
  } else if (normalizedBackend === "claude_chat") {
    publicStatus.auth_mode = "unsupported_local_cli";
    blockers.push(
      `${role} claude_chat is unsupported because installed SkillOpt routes it through a local Claude CLI without a verified read-isolated execution boundary`,
    );
  } else if (normalizedBackend === "qwen_chat") {
    const upper = role.toUpperCase();
    const baseUrl = cleanEnv(`${upper}_QWEN_CHAT_BASE_URL`) || cleanEnv("QWEN_CHAT_BASE_URL");
    const model = cleanEnv(`${upper}_QWEN_CHAT_MODEL`) || cleanEnv("QWEN_CHAT_MODEL");
    const apiKey = cleanEnv(`${upper}_QWEN_CHAT_API_KEY`) || cleanEnv("QWEN_CHAT_API_KEY");
    publicStatus.endpoint_present = Boolean(baseUrl);
    publicStatus.credential_present = Boolean(apiKey);
    if (!baseUrl) blockers.push(`${role} qwen_chat base URL is missing`);
    if (!model) blockers.push(`${role} qwen_chat model is missing`);
    if (!apiKey) blockers.push(`${role} qwen_chat API key is missing`);
  } else if (normalizedBackend === "minimax_chat") {
    publicStatus.endpoint_present = Boolean(cleanEnv("MINIMAX_BASE_URL"));
    publicStatus.credential_present = Boolean(cleanEnv("MINIMAX_API_KEY"));
    if (role === "optimizer") {
      blockers.push(
        "optimizer minimax_chat is unsupported because installed SkillOpt implements MiniMax for target chat only",
      );
    }
    if (!publicStatus.credential_present) blockers.push(`${role} minimax_chat API key is missing`);
  } else {
    blockers.push(`${role} provider backend ${normalizedBackend || "<missing>"} is unsupported`);
  }

  publicStatus.configured = blockers.length === 0;
  return { publicStatus, endpointProbe };
}

function runOpenAiCompatibleEndpointProbe(spec) {
  if (!spec.model) {
    return {
      ok: false,
      status: "blocked",
      error: `${spec.role} model pin is missing`,
    };
  }
  const script = path.join(skillRoot, "scripts/probe-openai-compatible-endpoint.mjs");
  const result = commandResult(process.execPath, [script, "--json"], {
    timeout: 70000,
    env: {
      ...process.env,
      SKILLOPT_OPENAI_BASE_URL: spec.endpoint,
      SKILLOPT_OPENAI_API_KEY: spec.apiKey,
      SKILLOPT_TARGET_MODEL: spec.model,
      OPENAI_BASE_URL: spec.endpoint,
      OPENAI_API_KEY: spec.apiKey,
    },
  });
  if (!result.ok) {
    return {
      ok: false,
      status: "failed",
      error: `${spec.role} chat-completion preflight failed`,
    };
  }
  try {
    const parsed = JSON.parse(result.stdout);
    return {
      ok: parsed.ok === true,
      status: parsed.ok === true ? "passed" : "failed",
      error: parsed.ok === true ? null : `${spec.role} chat-completion preflight failed`,
    };
  } catch {
    return {
      ok: false,
      status: "failed",
      error: `${spec.role} endpoint probe returned invalid JSON`,
    };
  }
}

function providerReadiness(args, configInfo) {
  if (args.mode === "codex-cli-all") {
    return { configured: true, roles: [], blockers: [], endpoint_probes: [] };
  }
  const requiredRoles = ["optimizer"];
  if (args.mode === "native-provider") requiredRoles.push("target");
  const roles = [];
  const endpointProbes = [];
  const blockers = [];
  for (const role of requiredRoles) {
    const backend =
      role === "optimizer" ? configInfo.values.optimizer_backend : configInfo.values.target_backend;
    const check = providerRoleConfig(role, backend, configInfo);
    roles.push(check.publicStatus);
    blockers.push(...check.publicStatus.blockers);
    if (check.endpointProbe) {
      const probe = args.strictTrainingReady
        ? runOpenAiCompatibleEndpointProbe(check.endpointProbe)
        : { ok: false, status: "not_run", error: null };
      endpointProbes.push({
        role,
        required: args.strictTrainingReady,
        ...probe,
      });
      if (args.strictTrainingReady && !probe.ok) blockers.push(probe.error);
    }
  }
  return {
    configured: blockers.length === 0,
    roles,
    blockers: [...new Set(blockers.filter(Boolean))],
    endpoint_probes: endpointProbes,
  };
}

function configBool(value) {
  return (
    String(value || "")
      .trim()
      .toLowerCase() === "true"
  );
}

function configBoolDefault(value, fallback) {
  const clean = String(value ?? "").trim();
  if (!clean) return fallback;
  return configBool(clean);
}

function codexPermissionCapability(configInfo) {
  const command = configInfo.values.codex_exec_path || "codex";
  let probeRoot = null;
  try {
    probeRoot = fs.mkdtempSync(path.join(os.tmpdir(), "skillopt-codex-capability-"));
    const codexHome = path.join(probeRoot, "codex-home");
    const workspace = path.join(probeRoot, "workspace");
    fs.mkdirSync(codexHome, { mode: 0o700 });
    fs.mkdirSync(workspace, { mode: 0o700 });
    const missingSchemaName = "probe-output-schema-never-created.json";
    const missingSchema = path.join(workspace, missingSchemaName);
    const env = Object.fromEntries(
      ["PATH", "HOME", "USER", "LOGNAME", "LANG", "LC_ALL", "LC_CTYPE"].flatMap((name) =>
        process.env[name] ? [[name, process.env[name]]] : [],
      ),
    );
    env.CODEX_HOME = codexHome;
    env.NO_COLOR = "1";
    env.TERM = "dumb";
    const baseArgs = [
      "exec",
      "--ignore-user-config",
      "--ignore-rules",
      "--strict-config",
      "--skip-git-repo-check",
      "--ephemeral",
      "--color",
      "never",
      "--cd",
      workspace,
      "--output-schema",
      missingSchema,
    ];
    const invoke = (permissionOverrides) =>
      spawnSync(
        command,
        [
          ...baseArgs,
          "-c",
          'default_permissions="skillopt_capability_probe"',
          ...permissionOverrides.flatMap((value) => ["-c", value]),
          "bounded local configuration capability probe",
        ],
        {
          cwd: workspace,
          encoding: "utf8",
          timeout: 5000,
          env,
          input: "",
        },
      );
    const validResult = invoke([
      'permissions.skillopt_capability_probe.filesystem={\":minimal\"=\"read\",\":workspace_roots\"={\".\"=\"write\"}}',
      "permissions.skillopt_capability_probe.network.enabled=false",
    ]);
    const invalidControl = invoke([
      "permissions.skillopt_capability_probe.filesystem=7",
      "permissions.skillopt_capability_probe.network.enabled=false",
    ]);
    const validOutput = `${validResult.stdout || ""}\n${validResult.stderr || ""}`;
    const invalidOutput = `${invalidControl.stdout || ""}\n${invalidControl.stderr || ""}`;
    const deliberatePreflightFailure =
      validResult.status !== 0 &&
      validOutput.includes("Failed to read output schema file") &&
      validOutput.includes(missingSchemaName);
    const invalidControlRejected =
      invalidControl.status !== 0 &&
      /Error loading config\.toml:[\s\S]*invalid type/i.test(invalidOutput) &&
      /permissions/i.test(invalidOutput) &&
      !invalidOutput.includes("Failed to read output schema file");
    const output = `${validOutput}\n${invalidOutput}`;
    const sessionStarted = /(?:session id:|responses_websocket|Reconnecting\.\.\.)/i.test(output);
    const supported = deliberatePreflightFailure && invalidControlRejected && !sessionStarted;
    let diagnostic = null;
    if (!supported) {
      if (validResult.error?.code === "ENOENT" || invalidControl.error?.code === "ENOENT") {
        diagnostic = "command_unavailable";
      } else if (
        validResult.error?.code === "ETIMEDOUT" ||
        invalidControl.error?.code === "ETIMEDOUT"
      ) {
        diagnostic = "probe_timed_out";
      } else if (/unknown configuration field|strict-config.*not supported/i.test(output)) {
        diagnostic = "strict_config_rejected";
      } else if (sessionStarted) diagnostic = "probe_crossed_model_boundary";
      else if (!invalidControlRejected) diagnostic = "strict_config_negative_control_failed";
      else diagnostic = "unexpected_probe_result";
    }
    return {
      status: supported ? "supported" : "unsupported",
      command: path.basename(command),
      strict_config: supported,
      permission_config: supported,
      probe: "strict_config_parse_with_invalid_control_before_deliberate_missing_schema_failure",
      negative_control_rejected: invalidControlRejected,
      model_or_network_started: sessionStarted,
      diagnostic,
    };
  } catch {
    return {
      status: "unsupported",
      command: path.basename(command),
      strict_config: false,
      permission_config: false,
      probe: "strict_config_parse_with_invalid_control_before_deliberate_missing_schema_failure",
      negative_control_rejected: false,
      model_or_network_started: false,
      diagnostic: "probe_setup_failed",
    };
  } finally {
    if (probeRoot) fs.rmSync(probeRoot, { recursive: true, force: true });
  }
}

function codexExecutionIsolationReadiness(datasetCounts, configInfo) {
  const targetBackend = String(configInfo.values.target_backend || "")
    .trim()
    .toLowerCase();
  const judgeBackend = String(configInfo.values.judge_backend || "")
    .trim()
    .toLowerCase();
  const reflectionBackend = String(configInfo.values.reflection_backend || "")
    .trim()
    .toLowerCase();
  const activePositiveCases = datasetCounts.generated_active?.exists
    ? datasetCounts.generated_active.positive
    : datasetCounts.eval_positive || 0;
  const activeRoles = [
    ...(targetBackend === "codex_exec" ? ["target"] : []),
    ...(judgeBackend === "codex_cli" ? ["judge"] : []),
    ...(reflectionBackend === "codex_cli" ? ["reflection"] : []),
  ];
  const required = activeRoles.length > 0 && activePositiveCases > 0;
  const capability = required
    ? codexPermissionCapability(configInfo)
    : {
        status: "not_applicable",
        command: null,
        strict_config: false,
        permission_config: false,
        probe: null,
        model_or_network_started: false,
        diagnostic: null,
      };
  const blockers =
    required && capability.status !== "supported"
      ? [
          `active Codex ${activeRoles.join("/")} execution requires a verified strict read-isolated permission profile`,
        ]
      : [];
  return {
    status: required ? (blockers.length ? "blocked" : "ready") : "not_applicable",
    required,
    activeRoles,
    targetBackend,
    judgeBackend,
    reflectionBackend,
    activePositiveCases,
    capability,
    blockers,
  };
}

function visualArtifactReadiness(datasetCounts, configInfo, codexIsolation) {
  const visualCases = datasetCounts.positive_with_visual_assertions || 0;
  const activeVisualCases =
    datasetCounts.generated_active?.positive_with_visual_assertions ?? visualCases;
  const activePositiveCases = datasetCounts.generated_active?.positive || 0;
  const drawioCli = detectDrawioCli();
  const toolRolloutForVisualAssertions = configBoolDefault(
    configInfo.values.tool_rollout_for_visual_assertions,
    true,
  );
  const requireDrawioCliForVisualRollouts = configBoolDefault(
    configInfo.values.require_drawio_cli_for_visual_rollouts,
    true,
  );
  const visualExecTimeoutSeconds = Number(configInfo.values.visual_exec_timeout || 0) || null;
  const activeSplitDir = datasetCounts.generated_active?.path || configInfo.values.split_dir || "";
  const visualEvalPolicy = configInfo.values.visual_eval_policy || "";
  const targetBackend = String(configInfo.values.target_backend || "")
    .trim()
    .toLowerCase();
  const codexPermissionProfile = codexIsolation.capability;
  const blockers = [];
  const warnings = [];
  let status = "not_applicable";

  if (visualCases > 0) {
    if (activeVisualCases > 0 && targetBackend !== "codex_exec") {
      status = "unsupported_visual_target_backend";
      blockers.push(
        `active visual assertion cases require target_backend=codex_exec; ${targetBackend || "the configured provider target"} cannot create local artifacts`,
      );
      warnings.push(
        "Provider-backed target runs must select the generated text-only split when source evals include visual assertions.",
      );
    } else if (codexIsolation.required && codexPermissionProfile.status !== "supported") {
      status = "unsupported_codex_permission_profile";
      blockers.push(
        "active Codex target cases require a verified strict custom permission profile, including text-only rollouts",
      );
    } else if (activePositiveCases > 0 && activeVisualCases === 0) {
      status = "text_only_ready";
      const isolationNote = codexIsolation.required
        ? "Codex read isolation remains mandatory."
        : "The provider-backed target remains text-only.";
      warnings.push(
        `${visualCases} visual assertion eval case(s) are excluded from the active text-only split ${activeSplitDir}; text-only bypasses draw.io readiness only. ${isolationNote}`,
      );
    } else if (!toolRolloutForVisualAssertions) {
      status = "blocked";
      blockers.push("visual assertions exist but tool_rollout_for_visual_assertions is disabled");
    } else if (requireDrawioCliForVisualRollouts && !drawioCli.installed) {
      status = "missing_drawio_cli";
      blockers.push(
        `${activeVisualCases} active visual assertion eval case(s) require draw.io Desktop CLI on PATH`,
      );
      warnings.push(
        `${activeVisualCases} active visual assertion eval case(s) will fast-fail until drawio or diagrams.net is on PATH, or an explicitly text-only split is used.`,
      );
    } else {
      status = "ready";
    }
  }

  return {
    status,
    visualAssertionCases: visualCases,
    activeVisualAssertionCases: activeVisualCases,
    activeSplitDir,
    visualEvalPolicy,
    targetBackend,
    drawioCli,
    toolRolloutForVisualAssertions,
    requireDrawioCliForVisualRollouts,
    visualExecTimeoutSeconds,
    codexPermissionProfile,
    blockers,
    warnings,
  };
}

function codexCliAllUnsupportedProviderFeatures(args, configInfo) {
  if (args.mode !== "codex-cli-all") return [];
  const blockers = [];
  if (configBool(configInfo.values.optimizer_use_slow_update)) {
    blockers.push(
      "codex-cli-all cannot enable optimizer.use_slow_update; slow update uses provider-backed chat_optimizer",
    );
  }
  if (configBool(configInfo.values.optimizer_use_meta_skill)) {
    blockers.push(
      "codex-cli-all cannot enable optimizer.use_meta_skill; meta skill uses provider-backed chat_optimizer",
    );
  }
  return blockers;
}

function adapterManifestCompatibility(adapterManifest, skillName, args, effectiveRunProfile) {
  if (!adapterManifest.data) {
    return {
      status: "missing",
      targetMatches: false,
      modeMatches: false,
      runProfileMatches: false,
      warnings: [
        "Adapter manifest is missing; production setup must create target-specific config before training.",
      ],
    };
  }

  const manifest = adapterManifest.data;
  const manifestTarget =
    manifest.target_skill || manifest.proof_target || manifest.proofTarget || null;
  const manifestMode = manifest.mode || manifest.selected_mode || null;
  const manifestRunProfile = manifest.runProfile || manifest.run_profile || null;
  const warnings = [];
  let reviewRequired = false;

  if (!adapterManifest.target_specific) {
    warnings.push(
      "Adapter manifest is legacy/global rather than target-specific; production setup must refresh it before training.",
    );
  }
  if (!manifestTarget) {
    warnings.push(
      "Adapter manifest is missing target identity; production setup must refresh it before training.",
    );
  } else if (manifestTarget !== skillName) {
    warnings.push(
      `Adapter manifest target ${manifestTarget} does not match requested target ${skillName}; production setup must refresh it before training.`,
    );
  }
  if (!manifestMode) {
    warnings.push(
      "Adapter manifest is missing mode identity; production setup must refresh it before training.",
    );
  } else if (manifestMode !== args.mode) {
    warnings.push(
      `Adapter manifest mode ${manifestMode} does not match requested mode ${args.mode}; production setup must refresh it before training.`,
    );
  }
  if (!manifestRunProfile) {
    warnings.push(
      "Adapter manifest is missing run profile identity; production setup must refresh it before training.",
    );
  } else if (manifestRunProfile !== effectiveRunProfile) {
    warnings.push(
      `Adapter manifest run profile ${manifestRunProfile} does not match requested run profile ${effectiveRunProfile}; production setup must refresh it before training.`,
    );
  }
  const registryStatus = manifest.registry_patch?.status || manifest.registryPatch?.status || null;
  if (registryStatus !== "ready") {
    reviewRequired = true;
    warnings.push(
      `Local SkillOpt registry/config patch status is ${registryStatus || "unknown"}; rerun adapter preparation and inspect the manifest if it still fails.`,
    );
  }
  warnings.push(...templateHashWarnings(manifest));

  return {
    status: reviewRequired ? "review_required" : warnings.length ? "refresh_required" : "matched",
    target: manifestTarget,
    mode: manifestMode,
    runProfile: manifestRunProfile,
    targetMatches: manifestTarget === skillName,
    modeMatches: manifestMode === args.mode,
    runProfileMatches: manifestRunProfile === effectiveRunProfile,
    warnings,
  };
}

function pythonStatus(args) {
  const uvCommand = resolveUvCommand();
  const uvVersion = uvCommand ? commandResult(uvCommand, ["--version"]).stdout : null;
  const local = commandResult(args.python, ["--version"]);
  const localVersion = (local.stdout || local.stderr || "").trim();
  const version = localVersion.match(/Python\s+(\d+)\.(\d+)(?:\.(\d+))?/);
  const localCompatible =
    Boolean(version) &&
    (Number(version[1]) > 3 || (Number(version[1]) === 3 && Number(version[2]) >= 10));

  let preference_required = false;
  if (args.pythonManager === "auto" && !uvCommand && localCompatible) preference_required = true;

  return {
    preference: args.pythonManager,
    uv: {
      installed: Boolean(uvCommand),
      command: uvCommand,
      version: uvVersion?.trim() || null,
    },
    local: {
      command: args.python,
      available: local.ok,
      version: localVersion || null,
      compatible: localCompatible,
    },
    preference_required,
  };
}

function runCodexProbe() {
  const script = path.join(skillRoot, "scripts/probe-codex-cli.mjs");
  const result = spawnSync(process.execPath, [script, "--json"], {
    cwd: root,
    encoding: "utf8",
    timeout: 300000,
  });
  if (result.status !== 0) {
    return {
      ok: false,
      error: redact(result.stderr || result.stdout || "Codex probe failed"),
    };
  }
  try {
    return JSON.parse(result.stdout);
  } catch {
    return { ok: false, error: "Codex probe returned non-JSON output" };
  }
}

function readExistingCodexProbe() {
  const readinessDir = path.join(root, ".agents/skillopt-work/_readiness");
  const finalPath = path.join(readinessDir, "codex-probe-final.txt");
  const diagnosticPath = path.join(readinessDir, "codex-probe-output.txt");
  if (!fs.existsSync(finalPath) || !fs.existsSync(diagnosticPath)) return null;
  const final = fs.readFileSync(finalPath, "utf8").trim();
  const diagnostic = fs.readFileSync(diagnosticPath, "utf8");
  const ok = final === "CODEX_READY" && /^status:\s*0\s*$/m.test(diagnostic);
  return {
    ok,
    cached: true,
    final_path: path.relative(root, finalPath).replaceAll("\\", "/"),
    diagnostic_path: path.relative(root, diagnosticPath).replaceAll("\\", "/"),
    error: ok ? null : "Existing Codex probe diagnostics are missing CODEX_READY or status 0",
  };
}

const args = parseArgs(process.argv.slice(2));
const missing = [];
const warnings = [];
const repoRootOk =
  fs.existsSync(path.join(root, "package.json")) && fs.existsSync(path.join(root, "AGENTS.md"));
if (!repoRootOk) missing.push("repo root");

const gitignore = fs.existsSync(path.join(root, ".gitignore"))
  ? fs.readFileSync(path.join(root, ".gitignore"), "utf8")
  : "";
if (!gitignore.split(/\r?\n/).some((line) => line.trim() === ".agents/")) {
  missing.push(".agents ignore rule");
}

const skillPath = resolveSkill(args.skill);
let skillName = args.skill || null;
let skillMaturity = "unspecified";
let frontmatterOk = false;
if (!skillPath) {
  missing.push("target skill");
} else {
  const rel = path.relative(root, skillPath).replaceAll("\\", "/");
  skillMaturity = rel.startsWith("skills/") ? "promoted" : "incubator";
  skillName = path.basename(path.dirname(skillPath));
  const text = fs.readFileSync(skillPath, "utf8");
  frontmatterOk = Boolean(parseFrontmatter(text)?.name);
  if (!frontmatterOk) missing.push("parseable skill frontmatter");
}

const evalDir = skillName ? path.join(root, "skill-evals", skillName) : null;
let datasetCounts = {
  eval_positive: 0,
  eval_negative: 0,
  eval_total: 0,
  estimated_split: { train: 0, val: 0, test: 0 },
  generated: { train: 0, val: 0, test: 0, activation_negative: 0 },
};
if (!evalDir || !fs.existsSync(evalDir)) missing.push("skill eval proof");
else {
  const evalCounts = evalCaseCounts(evalDir);
  datasetCounts = {
    eval_positive: evalCounts.positive,
    eval_negative: evalCounts.negative,
    eval_total: evalCounts.total,
    positive_with_deterministic_assertions: evalCounts.positive_with_deterministic_assertions,
    positive_with_visual_assertions: evalCounts.positive_with_visual_assertions,
    positive_with_fixtures: evalCounts.positive_with_fixtures,
    positive_with_expected_artifacts: evalCounts.positive_with_expected_artifacts,
    estimated_split: estimateSplitCounts(evalCounts.positive),
    generated: generatedSplitCounts(skillName),
  };
  if (evalCounts.positive < EXPLORATORY_MIN.positive) {
    warnings.push(
      `Only ${evalCounts.positive} positive eval case(s) found; treat optimization as exploratory.`,
    );
  }
  const split =
    datasetCounts.generated.train + datasetCounts.generated.val + datasetCounts.generated.test
      ? datasetCounts.generated
      : datasetCounts.estimated_split;
  if (split.val < EXPLORATORY_MIN.val || split.test < EXPLORATORY_MIN.test) {
    warnings.push(
      `Small validation/test split; val=${split.val}, test=${split.test}. Treat optimization as exploratory.`,
    );
  }
}

const python = pythonStatus(args);
if (args.pythonManager === "uv" && !python.uv.installed) {
  missing.push("uv");
} else if (args.pythonManager === "local" && !python.local.compatible) {
  missing.push("Python 3.10+");
} else if (!python.uv.installed && !python.local.compatible) {
  missing.push("uv or Python 3.10+");
}
if (python.preference_required) {
  warnings.push("uv is not installed; ask whether to install uv or explicitly use local Python.");
}

if (!commandResult("git", ["--version"]).ok) missing.push("Git");
if (!commandResult("node", ["--version"]).ok) missing.push("Node.js");
if (!commandResult("pnpm", ["--version"]).ok && !commandResult("npm", ["--version"]).ok) {
  missing.push("npm or pnpm");
}

const skillOptPath = path.join(root, ".agents/tools/SkillOpt");
const skillOptClone = fs.existsSync(skillOptPath);
const skillOptVenv = fs.existsSync(path.join(skillOptPath, ".venv"));
const commitPath = path.join(root, ".agents/tools/SkillOpt.commit");
if (!skillOptClone) missing.push("SkillOpt clone");
if (skillOptClone && !skillOptVenv) missing.push("SkillOpt virtualenv");

const credentials = providerPresence();
const configInfo = configProfile(skillName, args.mode);
const configSchema = configSchemaCheck(skillName, args.mode);
const provider = providerReadiness(args, configInfo);
const providerOk = provider.configured;
if (provider.endpoint_probes.some((probe) => probe.status === "not_run")) {
  warnings.push(
    "OpenAI-compatible endpoint chat preflight was not run; use strict training readiness before handoff.",
  );
}
if (args.mode === "native-provider" && !providerOk) missing.push(...provider.blockers);
if (args.mode === "hybrid-codex-target" && !providerOk && !args.setupOnly) {
  missing.push(...provider.blockers);
}
datasetCounts.generated_active = generatedSplitCountsFromConfig(skillName, configInfo);
const quality = benchmarkQuality(datasetCounts);
const semanticJudge = semanticJudgeReadiness(args, configInfo);
const codexCliAllProviderFeatureBlockers = codexCliAllUnsupportedProviderFeatures(args, configInfo);
if (codexCliAllProviderFeatureBlockers.length) {
  missing.push(...codexCliAllProviderFeatureBlockers);
  warnings.push(
    "codex-cli-all is the provider-free exploratory path; keep slow update and meta skill disabled, or choose hybrid-codex-target/native-provider for provider-backed optimizer features.",
  );
}
const officialParity = officialParityReport(
  args,
  providerOk,
  datasetCounts,
  configInfo,
  quality,
  configSchema,
  semanticJudge,
);
if (officialParity.officialParityStatus === "exploratory") {
  warnings.push(
    "Run is classified as exploratory; detailed parity differences are listed in JSON output.",
  );
}

let codex = {
  required:
    args.mode.includes("codex") ||
    configInfo.values.target_backend === "codex_exec" ||
    configInfo.values.judge_backend === "codex_cli" ||
    configInfo.values.reflection_backend === "codex_cli",
  installed: false,
  probe: null,
};
if (codex.required) {
  codex.installed = commandExists("codex");
  if (!codex.installed) {
    missing.push("Codex CLI");
  } else if (args.codexProbe) {
    codex.probe = runCodexProbe();
    if (!codex.probe.ok) missing.push("Codex CLI login probe");
  } else {
    const existingProbe = readExistingCodexProbe();
    if (existingProbe?.ok) {
      codex.probe = existingProbe;
    } else {
      codex.probe = existingProbe;
      warnings.push(
        "Codex CLI login probe was not run; ask before running it because the probe writes ignored diagnostics under .agents/skillopt-work/_readiness.",
      );
    }
    if (args.strictTrainingReady && !codex.probe?.ok) {
      missing.push("Codex CLI login probe");
    }
  }
}

const adapterManifest = readAdapterManifest(skillName);
const adapterManifestCheck = adapterManifestCompatibility(
  adapterManifest,
  skillName,
  args,
  officialParity.runProfile,
);
warnings.push(...adapterManifestCheck.warnings);
const liveAdapterPatches = adapterManifest.data
  ? liveAdapterPatchCheck(skillOptPath, adapterManifest.data, args.python)
  : {
      status: "refresh_required",
      manifest_commit: null,
      live_commit: actualSkillOptCommit(skillOptPath),
      files: [],
      blockers: ["target-specific adapter manifest is missing or invalid"],
    };
const datasetFreshness = generatedDataFreshness(skillPath, skillName, configInfo);
const codexIsolation = codexExecutionIsolationReadiness(datasetCounts, configInfo);
const visualArtifact = visualArtifactReadiness(datasetCounts, configInfo, codexIsolation);
warnings.push(...visualArtifact.warnings);
const configuredSplitBlockers =
  datasetCounts.generated_active?.configured && !datasetCounts.generated_active.exists
    ? [`configured split_dir is missing: ${datasetCounts.generated_active.configured_path}`]
    : [];
const dataFloorTrainingBlockers =
  officialParity.runProfile === "official-parity" && !quality.officialFloorMet
    ? quality.blockers.map((blocker) => `active dataset floor: ${blocker}`)
    : [];
const runnableDataBlockers = runnableSplitBlockers(datasetCounts, configInfo);
if (args.mode === "codex-cli-all") {
  if (!adapterManifest.data) {
    missing.push("adapter manifest for codex-cli-all");
  } else {
    const manifest = adapterManifest.data;
    if (!manifest.installed_files?.some((file) => file.endsWith("codex_cli_reflector.py"))) {
      missing.push("codex_cli reflection adapter");
    }
    if (!manifest.installed_files?.some((file) => file.endsWith("evaluator.py"))) {
      missing.push("Agent Skills evaluator adapter");
    }
  }
  const reflectorPath = path.join(
    root,
    ".agents/tools/SkillOpt/skillopt/envs/agent_skills/codex_cli_reflector.py",
  );
  if (fs.existsSync(reflectorPath)) {
    const reflectorText = fs.readFileSync(reflectorPath, "utf8");
    if (!reflectorText.includes("codex-cli local truncation")) {
      missing.push("provider-free codex_cli reflection adapter");
    }
  }
  const codexAllConfig = skillName
    ? path.join(root, ".agents/skillopt-work", skillName, "configs/agent-skills.codex-cli-all.yaml")
    : null;
  if (!codexAllConfig || !fs.existsSync(codexAllConfig)) {
    missing.push("codex-cli-all work config");
  } else {
    const configText = fs.readFileSync(codexAllConfig, "utf8");
    if (!/^\s*judge_backend:\s*codex_cli\s*$/m.test(configText)) {
      missing.push("codex_cli LLM judge config");
    }
  }
}

const safeToSetup =
  repoRootOk &&
  Boolean(skillPath) &&
  Boolean(evalDir && fs.existsSync(evalDir)) &&
  !missing.includes(".agents ignore rule");
const setupBlockers = [
  ...(!repoRootOk ? ["repo root"] : []),
  ...(!skillPath ? ["target skill"] : []),
  ...(!evalDir || !fs.existsSync(evalDir) ? ["skill eval proof"] : []),
  ...(missing.includes(".agents ignore rule") ? [".agents ignore rule"] : []),
];
const trainingBlockers = [
  ...missing,
  ...officialParity.proofBlockers,
  ...configuredSplitBlockers,
  ...dataFloorTrainingBlockers,
  ...runnableDataBlockers,
  ...datasetFreshness.blockers,
  ...liveAdapterPatches.blockers,
  ...codexIsolation.blockers,
  ...visualArtifact.blockers,
  ...(adapterManifestCheck.status !== "matched"
    ? [
        adapterManifestCheck.status === "review_required"
          ? "adapter registry/config patch review required"
          : "adapter manifest/config refresh required",
      ]
    : []),
];
const uniqueTrainingBlockers = [...new Set(trainingBlockers)];
const finalProofBlockers =
  officialParity.runProfile === "official-parity"
    ? [...new Set([...officialParity.proofBlockers, ...uniqueTrainingBlockers])]
    : officialParity.proofBlockers;
const finalProofStatus =
  officialParity.runProfile === "official-parity" && finalProofBlockers.length
    ? "blocked"
    : officialParity.proofStatus;

const result = {
  ok: uniqueTrainingBlockers.length === 0,
  target_skill: skillName,
  mode: args.mode,
  requested_run_profile: officialParity.requestedRunProfile,
  run_profile: officialParity.runProfile,
  requestedRunProfile: officialParity.requestedRunProfile,
  runProfile: officialParity.runProfile,
  proofStatus: finalProofStatus,
  proofBlockers: finalProofBlockers,
  officialParityStatus:
    officialParity.runProfile === "official-parity" && finalProofStatus === "blocked"
      ? "blocked"
      : officialParity.officialParityStatus,
  officialParityGaps: officialParity.officialParityGaps,
  upstreamBehaviorBypassed: officialParity.upstreamBehaviorBypassed,
  maturity: skillMaturity,
  skill_path: skillPath ? path.relative(root, skillPath).replaceAll("\\", "/") : null,
  frontmatter_ok: frontmatterOk,
  skillopt: {
    clone: skillOptClone,
    virtualenv: skillOptVenv,
    commit: skillOptCommit(skillOptPath, commitPath),
  },
  python,
  datasetCounts,
  benchmarkQuality: quality,
  modelPins: configInfo.modelPins,
  configDefaults: configInfo.values,
  configSchemaCheck: configSchema,
  semanticJudgeReadiness: semanticJudge,
  datasetFreshness,
  liveAdapterPatchCheck: liveAdapterPatches,
  codexIsolationReadiness: codexIsolation,
  visualArtifactReadiness: visualArtifact,
  artifactExpectations: artifactExpectations(skillName, configInfo),
  officialParityChecklist: {
    providerCredentials: providerOk,
    datasetFloor: quality.officialFloorMet,
    datasetFreshness: datasetFreshness.status,
    liveAdapterPatches: liveAdapterPatches.status,
    codexIsolation: codexIsolation.status,
    semanticJudge: semanticJudge.status,
    configSchema: configSchema.status,
    validationGate: configInfo.values.evaluation_use_gate !== "false",
    testEvaluation: configInfo.values.evaluation_eval_test !== "false",
    slowUpdate: configInfo.values.optimizer_use_slow_update === "true",
    slowUpdateGateWithSelection:
      configInfo.values.optimizer_slow_update_gate_with_selection === "true",
    metaSkill: configInfo.values.optimizer_use_meta_skill === "true",
    skillAwareReflection: configInfo.values.optimizer_use_skill_aware_reflection === "true",
    cosineScheduler: configInfo.values.optimizer_lr_scheduler === "cosine",
    visualArtifactReadiness: visualArtifact.status,
    requiredModelPins: requiredModelPinNames(args.mode, configInfo).map((name) => ({
      name,
      status: configInfo.modelPins[name]?.status || "not_applicable",
    })),
  },
  configPath: configInfo.path,
  adapterManifest,
  adapterManifestCheck,
  providerReadiness: provider,
  credential_presence: credentials,
  codex,
  missing,
  warnings,
  setupReadiness: safeToSetup ? "ready" : "blocked",
  setupBlockers,
  trainingReadiness: uniqueTrainingBlockers.length === 0 ? "ready" : "blocked",
  trainingBlockers: uniqueTrainingBlockers,
  safe_to_setup: safeToSetup,
};

if (args.json) {
  console.log(JSON.stringify(result, null, 2));
} else {
  console.log(`SkillOpt readiness for ${result.target_skill || "<missing skill>"}`);
  console.log(`Mode: ${result.mode}`);
  console.log(`Run profile: ${result.run_profile}`);
  console.log(`Setup readiness: ${result.setupReadiness}`);
  console.log(`Training readiness: ${result.trainingReadiness}`);
  console.log(`Proof status: ${result.proofStatus}`);
  console.log(`Official-parity status: ${result.officialParityStatus}`);
  if (result.proofBlockers.length)
    console.log(`Proof blockers: ${result.proofBlockers.join("; ")}`);
  if (result.officialParityGaps.length) {
    if (result.mode === "codex-cli-all" || result.run_profile === "exploratory") {
      const bypassed = result.upstreamBehaviorBypassed.length
        ? ` Bypassed upstream provider-backed behavior: ${result.upstreamBehaviorBypassed.join(", ")}.`
        : "";
      console.log(
        `Expected exploratory differences: provider-free exploratory mode, not upstream-native official optimizer parity.${bypassed}`,
      );
    } else {
      console.log(`Official-parity gaps: ${result.officialParityGaps.join("; ")}`);
    }
  }
  console.log(`Config schema: ${result.configSchemaCheck.status}`);
  console.log(
    `Data floor: ${result.benchmarkQuality.officialFloorMet ? "official floor met" : "official floor not met"}; classification ${result.benchmarkQuality.classification}`,
  );
  const modelPinBlockers = Object.values(result.modelPins)
    .filter((pin) => pin.status === "env_missing")
    .map((pin) => pin.env)
    .filter(Boolean);
  if (modelPinBlockers.length) console.log(`Missing model pin env: ${modelPinBlockers.join(", ")}`);
  console.log(
    `Dataset: positive ${result.datasetCounts.eval_positive}, negative ${result.datasetCounts.eval_negative}, generated train ${result.datasetCounts.generated.train}, val ${result.datasetCounts.generated.val}, test ${result.datasetCounts.generated.test}`,
  );
  if (result.visualArtifactReadiness.visualAssertionCases > 0) {
    console.log(`Visual artifact readiness: ${result.visualArtifactReadiness.status}`);
  }
  if (result.adapterManifestCheck.status !== "matched")
    console.log(`Adapter manifest: ${result.adapterManifestCheck.status}`);
  if (missing.length) console.log(`Missing: ${missing.join(", ")}`);
  if (warnings.length) console.log(`Warnings: ${warnings.join("; ")}`);
  if (
    result.mode !== "codex-cli-all" &&
    result.proofBlockers.some((blocker) =>
      blocker.includes("provider-backed optimizer credentials"),
    )
  ) {
    console.log(
      "Provider-free alternative: choose codex-cli-all for exploratory setup through Codex CLI login.",
    );
  }
  console.log(`Safe to setup: ${result.safe_to_setup ? "yes" : "no"}`);
}

if (args.strictTrainingReady && result.trainingReadiness !== "ready") {
  process.exitCode = 1;
} else {
  process.exitCode = result.safe_to_setup ? 0 : 1;
}
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall