Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

scripts/apply-skillopt-best.mjs

View on GitHub
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";

const root = process.cwd();
const FORBIDDEN_OPTIMIZER_PATTERN = /codex[-_\s]?opt/i;

function parseArgs(argv) {
  const args = { approved: false, dryRun: true, json: false, summary: false };
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === "--help" || arg === "-h") {
      printUsage();
      process.exit(0);
    } else if (arg === "--approved") {
      args.approved = true;
      args.dryRun = false;
    } else if (arg === "--dry-run") {
      args.dryRun = true;
    } else if (arg === "--json") args.json = true;
    else if (arg === "--summary" || arg === "--compact") args.summary = true;
    else if (arg === "--skill") args.skill = argv[++i];
    else if (arg === "--best") args.best = argv[++i];
    else if (arg === "--version") args.version = argv[++i];
    else fail(`Unknown argument: ${arg}`);
  }
  if (!args.skill) fail("--skill is required");
  if (!args.best) fail("--best is required");
  return args;
}

function printUsage() {
  console.log(`Usage: node apply-skillopt-best.mjs --skill <skill> --best <best_skill.md> [options]

Options:
  --dry-run
  --approved
  --version <x.y.z>
  --summary
  --json
  --help`);
}

function fail(message) {
  console.error(message);
  process.exit(2);
}

function walk(dir, predicate) {
  if (!fs.existsSync(dir)) return [];
  const files = [];
  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
    const full = path.join(dir, entry.name);
    if (entry.isDirectory()) files.push(...walk(full, predicate));
    else if (predicate(full)) files.push(full);
  }
  return files;
}

function resolveSkill(skill) {
  const direct = path.resolve(root, skill);
  if (fs.existsSync(direct) && path.basename(direct) === "SKILL.md") return direct;
  if (fs.existsSync(path.join(direct, "SKILL.md"))) return path.join(direct, "SKILL.md");
  return walk(root, (file) => path.basename(file) === "SKILL.md").find((file) => {
    const rel = path.relative(root, file).replaceAll("\\", "/");
    return (
      (rel.startsWith("skills/") || rel.startsWith("incubator/skills/")) &&
      path.basename(path.dirname(file)) === skill
    );
  });
}

function splitFrontmatter(text) {
  const match = text.match(/^---\n([\s\S]*?)\n---\n?/);
  if (!match) return { frontmatter: null, body: text };
  return { frontmatter: match[1], body: text.slice(match[0].length) };
}

function parseTopLevelFrontmatter(frontmatter) {
  const data = {};
  if (!frontmatter) return data;
  for (const line of frontmatter.split("\n")) {
    if (/^\s/.test(line)) continue;
    const i = line.indexOf(":");
    if (i === -1) continue;
    data[line.slice(0, i).trim()] = line
      .slice(i + 1)
      .trim()
      .replace(/^["']|["']$/g, "");
  }
  return data;
}

function updateVersion(frontmatter, version) {
  if (!version) return frontmatter;
  if (!/^\d+\.\d+\.\d+$/.test(version)) {
    throw new Error("--version must use x.y.z semver");
  }
  if (!/^\s+version:\s*["']?[^"'\n]+["']?\s*$/m.test(frontmatter)) {
    throw new Error("Cannot update promoted skill version because metadata.version was not found");
  }
  return frontmatter.replace(/^(\s+version:\s*)["']?[^"'\n]+["']?\s*$/m, `$1"${version}"`);
}

function simpleDiff(before, after) {
  const a = logicalLines(before);
  const b = logicalLines(after);
  const dp = Array.from({ length: a.length + 1 }, () => Array(b.length + 1).fill(0));
  for (let i = a.length - 1; i >= 0; i -= 1) {
    for (let j = b.length - 1; j >= 0; j -= 1) {
      dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
    }
  }
  const lines = [];
  let i = 0;
  let j = 0;
  while (i < a.length || j < b.length) {
    if (a[i] === b[j]) {
      lines.push(` ${a[i]}`);
      i += 1;
      j += 1;
    } else if (j < b.length && (i >= a.length || dp[i][j + 1] >= dp[i + 1][j])) {
      lines.push(`+${b[j]}`);
      j += 1;
    } else if (i < a.length) {
      lines.push(`-${a[i]}`);
      i += 1;
    }
  }
  return lines.join("\n");
}

function logicalLines(text) {
  const normalized = String(text || "").replace(/\r\n/g, "\n");
  const trimmed = normalized.endsWith("\n") ? normalized.slice(0, -1) : normalized;
  return trimmed ? trimmed.split("\n") : [];
}

function changeStats(before, after) {
  const beforeLines = logicalLines(before);
  const afterLines = logicalLines(after);
  const dp = Array.from({ length: beforeLines.length + 1 }, () =>
    Array(afterLines.length + 1).fill(0),
  );
  for (let i = beforeLines.length - 1; i >= 0; i -= 1) {
    for (let j = afterLines.length - 1; j >= 0; j -= 1) {
      dp[i][j] =
        beforeLines[i] === afterLines[j]
          ? dp[i + 1][j + 1] + 1
          : Math.max(dp[i + 1][j], dp[i][j + 1]);
    }
  }
  const unchanged = dp[0][0];
  return {
    added_lines: afterLines.length - unchanged,
    removed_lines: beforeLines.length - unchanged,
    net_lines: afterLines.length - beforeLines.length,
    before_lines: beforeLines.length,
    after_lines: afterLines.length,
    char_delta: after.length - before.length,
  };
}

function signed(value) {
  return value > 0 ? `+${value}` : String(value);
}

function formatMetric(value) {
  return value === null || value === undefined ? "not recorded" : String(value);
}

function formatChangeStats(stats) {
  return `+${stats.added_lines} / -${stats.removed_lines} lines (net ${signed(stats.net_lines)}), chars ${signed(stats.char_delta)}`;
}

function readJson(file, fallback = null) {
  if (!fs.existsSync(file)) return fallback;
  try {
    return JSON.parse(fs.readFileSync(file, "utf8"));
  } catch {
    return fallback;
  }
}

function finiteNumber(value) {
  const number = Number(value);
  return Number.isFinite(number) ? number : null;
}

function candidateRunSummary(bestPath) {
  const summary = readJson(path.join(path.dirname(bestPath), "summary.json"), null);
  if (!summary || typeof summary !== "object") {
    return {
      status: "not_recorded",
      reason: "summary.json not found next to best_skill.md",
    };
  }

  const baselineTestHard = finiteNumber(summary.baseline_test_hard);
  const testHard = finiteNumber(summary.test_hard);
  const testDeltaHard =
    finiteNumber(summary.test_delta_hard) ??
    (baselineTestHard === null || testHard === null ? null : testHard - baselineTestHard);
  const regressed =
    (testDeltaHard !== null && testDeltaHard < 0) ||
    (baselineTestHard !== null && testHard !== null && testHard < baselineTestHard);

  return {
    status: regressed ? "regressed" : "checked",
    baseline_test_hard: baselineTestHard,
    test_hard: testHard,
    test_delta_hard: testDeltaHard,
  };
}

function containsSecretLikeString(text) {
  if (/sk-[A-Za-z0-9_-]{20,}/.test(text)) return true;

  const knownSecretPrefix = /\b(?:ghp_|github_pat_|xox[baprs]-|AKIA|ASIA)[A-Za-z0-9_/-]{12,}\b/;
  if (knownSecretPrefix.test(text)) return true;

  const genericToken = /(?<![A-Za-z0-9+/=._-])[A-Za-z0-9+/=._-]{40,}(?![A-Za-z0-9+/=._-])/g;
  for (const match of text.matchAll(genericToken)) {
    const token = match[0];
    const mixedAlphaNumeric = /[a-z]/.test(token) && /[A-Z]/.test(token) && /\d/.test(token);
    const encodedPayload = token.length % 4 === 0 && /^[A-Za-z0-9+/]+={1,2}$/.test(token);
    if (mixedAlphaNumeric || encodedPayload) return true;
  }
  return false;
}

function rejectReasons(originalBody, candidateBody, candidateFrontmatter, originalFrontmatter) {
  const reasons = [];
  const topOriginal = parseTopLevelFrontmatter(originalFrontmatter);
  const topCandidate = parseTopLevelFrontmatter(candidateFrontmatter);

  if (candidateFrontmatter && candidateFrontmatter.trim() !== originalFrontmatter.trim()) {
    for (const key of ["name", "description", "license"]) {
      if (topCandidate[key] && topCandidate[key] !== topOriginal[key]) {
        reasons.push(`candidate changes frontmatter field ${key}`);
      }
    }
    if (!reasons.length)
      reasons.push(
        "candidate includes frontmatter; adoption preserves the original frontmatter instead",
      );
  }

  if (candidateBody.split(/\r?\n/).length > 500) reasons.push("candidate body exceeds 500 lines");
  if (containsSecretLikeString(candidateBody)) {
    reasons.push("candidate contains a secret-like string");
  }
  if (/\/home\/|\/Users\/|[A-Z]:\\/.test(candidateBody))
    reasons.push("candidate contains a private local path");
  if (/\.agents\//.test(candidateBody))
    reasons.push("candidate depends on a local optimizer workspace path");
  if (FORBIDDEN_OPTIMIZER_PATTERN.test(candidateBody)) {
    reasons.push("candidate references an unsupported optimizer source");
  }
  if (/^##\s+Rubric\b/im.test(candidateBody) || /^#\s+.*Eval Proof\b/im.test(candidateBody)) {
    reasons.push("candidate appears to copy eval or rubric material into SKILL.md");
  }
  if (/##\s+Safety rules/i.test(originalBody) && !/##\s+Safety rules/i.test(candidateBody)) {
    reasons.push("candidate removes the Safety rules section");
  }
  for (const phrase of ["approval", "secret"]) {
    if (
      new RegExp(`\\b${phrase}\\b`, "i").test(originalBody) &&
      !new RegExp(`\\b${phrase}\\b`, "i").test(candidateBody)
    ) {
      reasons.push(`candidate drops existing ${phrase} guidance`);
    }
  }
  return reasons;
}

const args = parseArgs(process.argv.slice(2));
const skillPath = resolveSkill(args.skill);
if (!skillPath) fail(`Could not find skill: ${args.skill}`);

const bestPath = path.resolve(root, args.best);
if (!fs.existsSync(bestPath))
  fail(`Missing best skill candidate: ${path.relative(root, bestPath)}`);

const original = fs.readFileSync(skillPath, "utf8");
const candidate = fs.readFileSync(bestPath, "utf8");
const originalParts = splitFrontmatter(original);
const candidateParts = splitFrontmatter(candidate);
if (!originalParts.frontmatter) fail("Original SKILL.md has no frontmatter");

const candidateBody = candidateParts.body.trimStart();
const rejections = rejectReasons(
  originalParts.body,
  candidateBody,
  candidateParts.frontmatter,
  originalParts.frontmatter,
);
const scoreGate = candidateRunSummary(bestPath);
if (scoreGate.status === "regressed") {
  rejections.push(
    `candidate regresses test hard score (${scoreGate.baseline_test_hard} -> ${scoreGate.test_hard})`,
  );
}
const promoted = path.relative(root, skillPath).replaceAll("\\", "/").startsWith("skills/");
if (promoted && args.approved && !args.version) {
  rejections.push("promoted public skills require --version for approved adoption");
}

let finalFrontmatter = originalParts.frontmatter;
try {
  finalFrontmatter = updateVersion(originalParts.frontmatter, args.version);
} catch (error) {
  rejections.push(error.message);
}

const normalizedBody = candidateBody.endsWith("\n") ? candidateBody : `${candidateBody}\n`;
const nextText = `---\n${finalFrontmatter.trim()}\n---\n\n${normalizedBody}`;
const diff = simpleDiff(original, nextText);
const changes = changeStats(original, nextText);
const report = {
  ok: rejections.length === 0,
  dry_run: args.dryRun || !args.approved,
  approved: args.approved,
  skill_path: path.relative(root, skillPath).replaceAll("\\", "/"),
  candidate_path: path.relative(root, bestPath).replaceAll("\\", "/"),
  promoted,
  preserved_frontmatter: true,
  score_gate: scoreGate,
  rejections,
  changed: original !== nextText,
  changes,
};

if (rejections.length === 0 && args.approved) {
  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
  const backupDir = path.join(
    root,
    ".agents/skillopt-work",
    path.basename(path.dirname(skillPath)),
    "backups",
  );
  const backupPath = path.join(backupDir, `${timestamp}-SKILL.md`);
  fs.mkdirSync(backupDir, { recursive: true });
  fs.writeFileSync(backupPath, original, "utf8");
  fs.writeFileSync(skillPath, nextText, "utf8");
  report.backup_path = path.relative(root, backupPath).replaceAll("\\", "/");
}

if (args.json) {
  console.log(JSON.stringify({ ...report, diff }, null, 2));
} else if (args.summary) {
  console.log(`SkillOpt adoption preview: ${args.skill}`);
  console.log(`Target: ${report.skill_path}`);
  console.log(`Candidate: ${report.candidate_path}`);
  console.log(`Changed: ${report.changed ? "yes" : "no"}`);
  console.log(`Change amount: ${formatChangeStats(report.changes)}`);
  if (report.score_gate.status !== "not_recorded") {
    console.log(
      `Test hard score: ${formatMetric(report.score_gate.baseline_test_hard)} -> ${formatMetric(report.score_gate.test_hard)} (delta ${formatMetric(report.score_gate.test_delta_hard)})`,
    );
  }
  console.log(`Safety checks: ${report.ok ? "pass" : "review blockers"}`);
  if (rejections.length) {
    console.log("Review blockers:");
    for (const reason of rejections) console.log(`- ${reason}`);
  }
  console.log("No files written.");
  if (report.changed) {
    console.log("For the full diff, rerun this command without --summary.");
  } else {
    console.log("No content differences after preserving frontmatter.");
  }
} else {
  console.log(`Skill: ${report.skill_path}`);
  console.log(`Candidate: ${report.candidate_path}`);
  console.log(`Dry run: ${report.dry_run ? "yes" : "no"}`);
  console.log(`Accepted by safety checks: ${report.ok ? "yes" : "no"}`);
  console.log(`Change amount: ${formatChangeStats(report.changes)}`);
  if (report.score_gate.status !== "not_recorded") {
    console.log(
      `Test hard score: ${formatMetric(report.score_gate.baseline_test_hard)} -> ${formatMetric(report.score_gate.test_hard)} (delta ${formatMetric(report.score_gate.test_delta_hard)})`,
    );
  }
  if (rejections.length) {
    console.log("Rejections:");
    for (const reason of rejections) console.log(`- ${reason}`);
  }
  console.log("");
  if (report.changed) {
    console.log(diff);
  } else {
    console.log("No content differences after preserving frontmatter.");
  }
  if (args.approved && report.ok) {
    console.log("");
    console.log("Wrote updated SKILL.md.");
    console.log(`Backup: ${report.backup_path}`);
    console.log("Next validation: npm run validate");
  } else if (!args.approved) {
    console.log("");
    console.log("No files written. Rerun with --approved after review.");
  }
}

process.exit(rejections.length ? 1 : 0);
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall