Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

scripts/probe-openai-compatible-endpoint.mjs

View on GitHub
#!/usr/bin/env node
const defaultTimeoutMs = 60000;

function printHelp() {
  console.log(`Usage: node probe-openai-compatible-endpoint.mjs [options]

Checks an OpenAI-compatible /v1 endpoint by listing models and running one chat completion.

Options:
  --base-url <url>    Base URL, default SKILLOPT_OPENAI_BASE_URL or OPENAI_BASE_URL
  --api-key <token>   Bearer token, default SKILLOPT_OPENAI_API_KEY or OPENAI_API_KEY
  --model <name>      Model to call, default SKILLOPT_TARGET_MODEL or SKILLOPT_OPTIMIZER_MODEL
  --timeout-ms <ms>   Request timeout, default 60000
  --json
  --help`);
}

function parseArgs(argv) {
  const args = {
    baseUrl:
      process.env.SKILLOPT_OPENAI_BASE_URL || process.env.OPENAI_BASE_URL || "",
    apiKey:
      process.env.SKILLOPT_OPENAI_API_KEY || process.env.OPENAI_API_KEY || "",
    model:
      process.env.SKILLOPT_TARGET_MODEL ||
      process.env.SKILLOPT_OPTIMIZER_MODEL ||
      "",
    timeoutMs: defaultTimeoutMs,
    json: false,
  };
  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    if (arg === "--help" || arg === "-h") {
      printHelp();
      process.exit(0);
    } else if (arg === "--base-url") args.baseUrl = argv[++index];
    else if (arg === "--api-key") args.apiKey = argv[++index];
    else if (arg === "--model") args.model = argv[++index];
    else if (arg === "--timeout-ms") args.timeoutMs = Number(argv[++index]);
    else if (arg === "--json") args.json = true;
    else fail(`Unknown argument: ${arg}`);
  }
  if (!args.baseUrl) fail("--base-url or SKILLOPT_OPENAI_BASE_URL is required");
  if (!args.model) fail("--model or SKILLOPT_TARGET_MODEL is required");
  if (!Number.isInteger(args.timeoutMs) || args.timeoutMs <= 0) {
    fail("--timeout-ms must be a positive integer");
  }
  args.baseUrl = args.baseUrl.replace(/\/+$/, "");
  return args;
}

function fail(message) {
  console.error(message);
  process.exit(2);
}

function isSensitiveQueryKey(key) {
  const normalized = String(key || "")
    .toLowerCase()
    .replace(/[^a-z0-9]/g, "");
  return (
    [
      "apikey",
      "accesskey",
      "accesstoken",
      "authorization",
      "auth",
      "credential",
      "password",
      "passwd",
      "secret",
      "signature",
      "sig",
      "token",
    ].includes(normalized) ||
    /(accesstoken|securitytoken|credential|password|secret|signature)$/.test(
      normalized,
    )
  );
}

function sanitizeDisplayUrl(value) {
  try {
    const url = new URL(String(value || ""));
    url.username = "";
    url.password = "";
    for (const key of [...url.searchParams.keys()]) {
      if (isSensitiveQueryKey(key)) url.searchParams.set(key, "[redacted]");
    }
    if (url.hash) url.hash = "#redacted";
    return url.toString();
  } catch {
    return String(value || "")
      .replace(/(https?:\/\/)[^/\s@]+@/gi, "$1[redacted]@")
      .replace(
        /([?&](?:api[_-]?key|access[_-]?token|auth|credential|password|passwd|secret|signature|sig|token)=)[^&#\s]*/gi,
        "$1[redacted]",
      );
  }
}

function sensitiveUrlValues(value) {
  try {
    const url = new URL(String(value || ""));
    const values = [url.username, url.password];
    for (const [key, queryValue] of url.searchParams.entries()) {
      if (isSensitiveQueryKey(key)) values.push(queryValue);
    }
    if (url.hash) values.push(url.hash.slice(1));
    return values.filter(Boolean);
  } catch {
    return [];
  }
}

function redact(text, sensitiveValues = []) {
  let output = String(text || "");
  for (const value of sensitiveValues.filter(Boolean)) {
    output = output.replaceAll(String(value), "[redacted]");
    output = output.replaceAll(
      encodeURIComponent(String(value)),
      "%5Bredacted%5D",
    );
  }
  return output
    .replace(/https?:\/\/[^\s"'<>]+/gi, (url) => sanitizeDisplayUrl(url))
    .replace(/(https?:\/\/)[^/\s@]+@/gi, "$1[redacted]@")
    .replace(
      /([?&](?:api[_-]?key|access[_-]?token|auth|credential|password|passwd|secret|signature|sig|token)=)[^&#\s]*/gi,
      "$1[redacted]",
    )
    .replace(/sk-[A-Za-z0-9_-]{8,}/g, "[redacted-token]")
    .replace(/[A-Za-z0-9+/=._-]{48,}/g, "[redacted-long-token]");
}

function sanitizePayload(payload, sensitiveValues) {
  if (!payload || typeof payload !== "object" || Array.isArray(payload))
    return null;
  if (payload.error && typeof payload.error === "object") {
    return {
      error: {
        type: redact(payload.error.type, sensitiveValues),
        code: redact(payload.error.code, sensitiveValues),
        message: redact(payload.error.message, sensitiveValues),
      },
    };
  }
  return {
    object: typeof payload.object === "string" ? payload.object : null,
    model_count: Array.isArray(payload.data) ? payload.data.length : null,
    choices_count: Array.isArray(payload.choices)
      ? payload.choices.length
      : null,
    assistant_content_present:
      typeof payload.choices?.[0]?.message?.content === "string",
  };
}

function assistantContent(payload) {
  return typeof payload?.choices?.[0]?.message?.content === "string"
    ? payload.choices[0].message.content
    : "";
}

async function requestJson(args, method, path, body = null) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), args.timeoutMs);
  const headers = { accept: "application/json" };
  if (args.apiKey) headers.authorization = `Bearer ${args.apiKey}`;
  if (body) headers["content-type"] = "application/json";
  try {
    const response = await fetch(`${args.baseUrl}${path}`, {
      method,
      headers,
      body: body ? JSON.stringify(body) : undefined,
      signal: controller.signal,
    });
    const text = await response.text();
    let payload = null;
    try {
      payload = text ? JSON.parse(text) : null;
    } catch {
      payload = {
        error: {
          message: redact(text, args.sensitiveValues).slice(0, 1000),
          type: "non_json_response",
        },
      };
    }
    return {
      ok: response.ok,
      status: response.status,
      payload: sanitizePayload(payload, args.sensitiveValues),
      assistantContent: assistantContent(payload),
    };
  } finally {
    clearTimeout(timer);
  }
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  args.sensitiveValues = [
    ...sensitiveUrlValues(args.baseUrl),
    args.apiKey,
  ].filter(Boolean);
  const result = {
    ok: false,
    base_url: sanitizeDisplayUrl(args.baseUrl),
    model: args.model,
    auth_present: Boolean(args.apiKey),
    models: null,
    chat: null,
  };

  try {
    result.models = await requestJson(args, "GET", "/models");
    const chatBody = {
      model: args.model,
      messages: [
        {
          role: "user",
          content: "Reply with exactly SKILLOPT_ENDPOINT_READY.",
        },
      ],
      temperature: 0,
      max_tokens: 16,
    };
    result.chat = await requestJson(
      args,
      "POST",
      "/chat/completions",
      chatBody,
    );
    const content = result.chat.assistantContent || "";
    delete result.models.assistantContent;
    delete result.chat.assistantContent;
    result.ok =
      result.models.ok &&
      result.chat.ok &&
      typeof content === "string" &&
      content.includes("SKILLOPT_ENDPOINT_READY");
  } catch (error) {
    result.error = redact(
      error instanceof Error ? error.message : String(error),
      args.sensitiveValues,
    );
  }

  if (args.json) {
    console.log(JSON.stringify(result, null, 2));
  } else {
    console.log(
      `OpenAI-compatible endpoint probe: ${result.ok ? "ready" : "blocked"}`,
    );
    console.log(`Base URL: ${result.base_url}`);
    console.log(`Model: ${result.model}`);
    console.log(`Auth present: ${result.auth_present ? "yes" : "no"}`);
    if (result.models)
      console.log(`Models request: HTTP ${result.models.status}`);
    if (result.chat) console.log(`Chat request: HTTP ${result.chat.status}`);
    if (result.error) console.log(`Error: ${result.error}`);
  }
  process.exit(result.ok ? 0 : 1);
}

main();
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall