Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

scripts/probe-codex-cli.mjs

View on GitHub
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";

const root = process.cwd();
const readinessDir = path.join(root, ".agents/skillopt-work/_readiness");
const finalPath = path.join(readinessDir, "codex-probe-final.txt");
const outputPath = path.join(readinessDir, "codex-probe-output.txt");

function parseArgs(argv) {
  const args = { json: false, authMode: "codex-cli", timeout: 240000 };
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === "--help" || arg === "-h") {
      printUsage();
      process.exit(0);
    } else if (arg === "--json") args.json = true;
    else if (arg === "--auth-mode") args.authMode = argv[++i];
    else if (arg === "--timeout-ms") args.timeout = Number(argv[++i]);
    else fail(`Unknown argument: ${arg}`);
  }
  return args;
}

function printUsage() {
  console.log(`Usage: node probe-codex-cli.mjs [options]

Options:
  --auth-mode <codex-cli>
  --timeout-ms <number>
  --json
  --help`);
}

function fail(message) {
  console.error(message);
  process.exit(2);
}

function redact(text) {
  const home = process.env.HOME ? path.resolve(process.env.HOME) : "";
  return String(text || "")
    .replaceAll(home, "~")
    .replace(/sk-[A-Za-z0-9_-]{20,}/g, "[redacted-token]")
    .replace(/[A-Za-z0-9+/=._-]{40,}/g, (token) =>
      /[\\/]/.test(token) || /^[a-f0-9]{40,64}$/i.test(token)
        ? token
        : "[redacted-long-token]",
    )
    .replace(
      /[A-Za-z0-9_./-]*(auth|token|credential)[A-Za-z0-9_./-]*/gi,
      "[redacted-auth-reference]",
    );
}

function commandExists(command) {
  const result =
    process.platform === "win32"
      ? spawnSync("where", [command], { encoding: "utf8" })
      : spawnSync("sh", ["-lc", `command -v ${command}`], { encoding: "utf8" });
  return result.status === 0;
}

function writeDiagnostic(result) {
  fs.mkdirSync(readinessDir, { recursive: true });
  const diagnostic = [
    `status: ${result.status ?? "signal:" + result.signal}`,
    "stdout:",
    redact(result.stdout || ""),
    "stderr:",
    redact(result.stderr || ""),
  ].join("\n");
  fs.writeFileSync(outputPath, diagnostic.slice(0, 8000), "utf8");
}

function runCodexExec(timeout) {
  const prompt = "Return exactly CODEX_READY. Do not inspect files.";
  return spawnSync(
    "codex",
    [
      "exec",
      "--skip-git-repo-check",
      "--color",
      "never",
      "--sandbox",
      "read-only",
      "--output-last-message",
      finalPath,
      prompt,
    ],
    {
      cwd: root,
      encoding: "utf8",
      timeout,
    },
  );
}

const args = parseArgs(process.argv.slice(2));
fs.mkdirSync(readinessDir, { recursive: true });
fs.rmSync(finalPath, { force: true });

const warnings = [];
if (args.authMode === "codex-cli" && process.env.OPENAI_API_KEY) {
  warnings.push(
    "OPENAI_API_KEY is set while --auth-mode codex-cli was selected; verify the run uses the intended local auth path.",
  );
}

const result = {
  ok: false,
  codex_installed: false,
  codex_version: null,
  final_path: path.relative(root, finalPath).replaceAll("\\", "/"),
  diagnostic_path: path.relative(root, outputPath).replaceAll("\\", "/"),
  warnings,
  error: null,
};

if (!commandExists("codex")) {
  result.error = "codex executable not found";
} else {
  result.codex_installed = true;
  const version = spawnSync("codex", ["--version"], {
    cwd: root,
    encoding: "utf8",
    timeout: 30000,
  });
  result.codex_version = redact((version.stdout || version.stderr || "").trim());

  const probe = runCodexExec(args.timeout);
  writeDiagnostic(probe);

  const final = fs.existsSync(finalPath) ? fs.readFileSync(finalPath, "utf8").trim() : "";
  if (probe.status === 0 && final === "CODEX_READY") {
    result.ok = true;
  } else {
    result.error = `Codex probe did not return CODEX_READY; final response was ${JSON.stringify(redact(final))}.`;
  }
}

if (args.json) {
  console.log(JSON.stringify(result, null, 2));
} else {
  console.log(`Codex CLI installed: ${result.codex_installed ? "yes" : "no"}`);
  if (result.codex_version) console.log(`Codex CLI version: ${result.codex_version}`);
  console.log(`Probe: ${result.ok ? "pass" : "fail"}`);
  if (result.error) console.log(`Error: ${result.error}`);
  if (warnings.length) console.log(`Warnings: ${warnings.join("; ")}`);
}

process.exit(result.ok ? 0 : 1);
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall