Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

assets/agent-skills-benchmark/adapter.py.template

View on GitHub
import os
from pathlib import Path

from .dataloader import AgentSkillsDataLoader
from .evaluator import AgentSkillsEvaluator
from .rollout import AgentSkillsRollout
from .codex_cli_reflector import CodexCliReflector

from skillopt.datasets.base import BatchSpec
from skillopt.envs.base import EnvAdapter
from skillopt.gradient.reflect import run_minibatch_reflect


def _compact(values):
    return {key: value for key, value in values.items() if value is not None}


def _flatten_config(config):
    flat = {}
    if not isinstance(config, dict):
        return flat
    for value in config.values():
        if isinstance(value, dict):
            flat.update(_flatten_config(value))
    for key, value in config.items():
        if not isinstance(value, dict):
            flat[key] = value
    return flat


def _safe_id(value):
    return "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in str(value)).strip("_") or "case"


class AgentSkillsAdapter(EnvAdapter):
    def __init__(
        self,
        env="agent_skills",
        split_dir="",
        data_path="",
        split_mode="split_dir",
        split_ratio="2:1:7",
        split_seed=42,
        split_output_dir="",
        visual_eval_policy="auto",
        out_root="",
        max_turns=1,
        exec_timeout=240,
        tool_rollout_for_visual_assertions=True,
        require_drawio_cli_for_visual_rollouts=True,
        visual_exec_timeout=120,
        workers=1,
        analyst_workers=1,
        failure_only=False,
        minibatch_size=1,
        edit_budget=2,
        seed=42,
        limit=0,
        reflection_backend="native",
        codex_cli_reflection_model="",
        codex_cli_reflection_timeout=240,
        judge_backend="heuristic",
        codex_cli_judge_model="",
        codex_cli_judge_profile="",
        codex_cli_judge_timeout=240,
        codex_cli_judge_sandbox="read-only",
        codex_exec_path="codex",
        codex_exec_sandbox="workspace-write",
        codex_exec_profile="",
        codex_exec_full_auto=False,
        codex_exec_reasoning_effort="none",
        codex_exec_use_sdk="cli",
        codex_exec_network_access=False,
        codex_exec_web_search=False,
        codex_exec_approval_policy="never",
        target_backend="",
        target_model="",
        optimizer_backend="",
        optimizer_model="",
        run_profile="exploratory",
        official_parity_status="unspecified",
        config=None,
        **extra_config,
    ):
        if isinstance(env, dict) and config is None:
            config = env
            env = "agent_skills"
        startup_config = _flatten_config(config)
        startup_config.update(_flatten_config(extra_config))
        self.max_turns = int(max_turns or 1)
        self.exec_timeout = int(exec_timeout or 240)
        self.workers = int(workers or 1)
        self.analyst_workers = int(analyst_workers or 1)
        self.failure_only = bool(failure_only)
        self.minibatch_size = int(minibatch_size or 1)
        self.edit_budget = int(edit_budget or 2)
        self.env_config = _compact(
            {
                "name": env,
                "split_dir": split_dir,
                "data_path": data_path,
                "split_mode": split_mode,
                "split_ratio": split_ratio,
                "split_seed": split_seed,
                "split_output_dir": split_output_dir,
                "visual_eval_policy": visual_eval_policy,
                "out_root": out_root,
                "max_turns": max_turns,
                "exec_timeout": exec_timeout,
                "tool_rollout_for_visual_assertions": tool_rollout_for_visual_assertions,
                "require_drawio_cli_for_visual_rollouts": require_drawio_cli_for_visual_rollouts,
                "visual_exec_timeout": visual_exec_timeout,
                "workers": workers,
                "analyst_workers": analyst_workers,
                "failure_only": failure_only,
                "minibatch_size": minibatch_size,
                "edit_budget": edit_budget,
                "seed": seed,
                "limit": limit,
                "reflection_backend": reflection_backend,
                "codex_cli_reflection_model": codex_cli_reflection_model,
                "codex_cli_reflection_timeout": codex_cli_reflection_timeout,
                "judge_backend": judge_backend,
                "codex_cli_judge_model": codex_cli_judge_model,
                "codex_cli_judge_profile": codex_cli_judge_profile,
                "codex_cli_judge_timeout": codex_cli_judge_timeout,
                "codex_cli_judge_sandbox": codex_cli_judge_sandbox,
                "run_profile": run_profile,
                "official_parity_status": official_parity_status,
            }
        )
        self.model_config = _compact(
            {
                "codex_exec_path": codex_exec_path,
                "codex_exec_sandbox": codex_exec_sandbox,
                "codex_exec_profile": codex_exec_profile,
                "codex_exec_full_auto": codex_exec_full_auto,
                "codex_exec_reasoning_effort": codex_exec_reasoning_effort,
                "codex_exec_use_sdk": codex_exec_use_sdk,
                "codex_exec_network_access": codex_exec_network_access,
                "codex_exec_web_search": codex_exec_web_search,
                "codex_exec_approval_policy": codex_exec_approval_policy,
                "target_backend": target_backend,
                "target_model": target_model,
                "optimizer_backend": optimizer_backend,
                "optimizer_model": optimizer_model,
                "exec_timeout": exec_timeout,
            }
        )
        self.config = {"env": dict(self.env_config), "model": dict(self.model_config)}
        self.dataloader = AgentSkillsDataLoader(
            split_dir=split_dir,
            data_path=data_path,
            split_mode=split_mode,
            split_ratio=split_ratio,
            split_seed=split_seed,
            split_output_dir=split_output_dir,
            seed=seed,
            limit=limit,
        )
        self.evaluator = AgentSkillsEvaluator(self._rollout_config())
        self.rollout_runner = AgentSkillsRollout(self._rollout_config())
        self.reflector = CodexCliReflector(self._rollout_config())
        if startup_config:
            self._sync_from_flat_config(startup_config)
            self._rebuild_components()

    def _rollout_config(self):
        merged = dict(self.model_config)
        merged.update(self.env_config)
        if hasattr(self, "_cfg"):
            merged.update(self._cfg)
        return merged

    def _sync_from_flat_config(self, cfg):
        env_keys = {
            "env",
            "split_dir",
            "data_path",
            "split_mode",
            "split_ratio",
            "split_seed",
            "split_output_dir",
            "visual_eval_policy",
            "out_root",
            "max_turns",
            "exec_timeout",
            "tool_rollout_for_visual_assertions",
            "require_drawio_cli_for_visual_rollouts",
            "visual_exec_timeout",
            "workers",
            "analyst_workers",
            "failure_only",
            "minibatch_size",
            "edit_budget",
            "seed",
            "limit",
            "reflection_backend",
            "codex_cli_reflection_model",
            "codex_cli_reflection_timeout",
            "judge_backend",
            "codex_cli_judge_model",
            "codex_cli_judge_profile",
            "codex_cli_judge_timeout",
            "codex_cli_judge_sandbox",
            "run_profile",
            "official_parity_status",
        }
        model_keys = {
            "codex_exec_path",
            "codex_exec_sandbox",
            "codex_exec_profile",
            "codex_exec_full_auto",
            "codex_exec_reasoning_effort",
            "codex_exec_use_sdk",
            "codex_exec_network_access",
            "codex_exec_web_search",
            "codex_exec_approval_policy",
            "target_backend",
            "target_model",
            "optimizer_backend",
            "optimizer_model",
        }
        for key in env_keys:
            if key in cfg:
                env_key = "name" if key == "env" else key
                self.env_config[env_key] = cfg[key]
        for key in model_keys:
            if key in cfg:
                self.model_config[key] = cfg[key]
        self.max_turns = int(self.env_config.get("max_turns") or 1)
        self.exec_timeout = int(self.env_config.get("exec_timeout") or 240)
        self.workers = int(self.env_config.get("workers") or 1)
        self.analyst_workers = int(self.env_config.get("analyst_workers") or 1)
        self.failure_only = bool(self.env_config.get("failure_only", False))
        self.minibatch_size = int(self.env_config.get("minibatch_size") or 1)
        self.edit_budget = int(self.env_config.get("edit_budget") or 2)
        self.config = {
            "env": dict(self.env_config),
            "model": dict(self.model_config),
            "flat": dict(cfg),
        }

    def _rebuild_components(self):
        self.dataloader = AgentSkillsDataLoader(
            split_dir=self.env_config.get("split_dir", ""),
            data_path=self.env_config.get("data_path", ""),
            split_mode=self.env_config.get("split_mode", "split_dir"),
            split_ratio=self.env_config.get("split_ratio", "2:1:7"),
            split_seed=self.env_config.get("split_seed", 42),
            split_output_dir=self.env_config.get("split_output_dir", ""),
            seed=self.env_config.get("seed", 42),
            limit=self.env_config.get("limit", 0),
        )
        self.evaluator = AgentSkillsEvaluator(self._rollout_config())
        self.rollout_runner = AgentSkillsRollout(self._rollout_config())
        self.reflector = CodexCliReflector(self._rollout_config())

    def setup(self, cfg):
        super().setup(cfg)
        flat_cfg = _flatten_config(cfg)
        self._sync_from_flat_config(flat_cfg)
        self._rebuild_components()
        self.dataloader.setup(flat_cfg)

    def get_dataloader(self):
        return self.dataloader

    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
        return list(batch.payload or [])

    def build_train_env(self, batch_size, seed, **kwargs):
        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
        return self.build_env_from_batch(batch, **kwargs)

    def build_eval_env(self, env_num, split, seed, **kwargs):
        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
        return self.build_env_from_batch(batch, **kwargs)

    def get_task_types(self):
        seen = []
        for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
            task_type = str(item.get("task_type") or item.get("skill_name") or "agent_skill_quality")
            if task_type not in seen:
                seen.append(task_type)
        return seen or ["agent_skill_quality"]

    def rollout(self, env_manager, skill_content, out_dir, **kwargs):
        items = list(env_manager or [])
        prediction_dir = Path(out_dir) / "predictions"
        prediction_dir.mkdir(parents=True, exist_ok=True)
        results = []
        print(f"    [agent_skills rollout] starting {len(items)} case(s)", flush=True)
        for index, case in enumerate(items, 1):
            print(
                f"      [agent_skills rollout] {index}/{len(items)} {case.get('id', '<unknown>')}",
                flush=True,
            )
            rollout = self.rollout_runner.run(case, skill_content)
            scored = self.evaluator.score(case, rollout)
            scored.update(
                {
                    "task_description": case.get("prompt", ""),
                    "target_user_prompt": case.get("prompt", ""),
                    "expected_behavior": case.get("expected_behavior", []),
                    "case_path": case.get("case_path", ""),
                    "returncode": rollout.get("returncode", 0),
                    "stderr": rollout.get("stderr", ""),
                }
            )
            case_dir = prediction_dir / _safe_id(scored["id"])
            case_dir.mkdir(parents=True, exist_ok=True)
            (case_dir / "conversation.json").write_text(
                self._conversation_json(case, rollout),
                encoding="utf-8",
            )
            (case_dir / "target_user_prompt.txt").write_text(
                str(case.get("prompt", "")),
                encoding="utf-8",
            )
            print(
                f"      [agent_skills rollout done] {scored['id']} hard={scored['hard']} returncode={rollout.get('returncode', 0)}",
                flush=True,
            )
            results.append(scored)
        return results

    def _conversation_json(self, case, rollout):
        import json

        return json.dumps(
            [
                {
                    "role": "user",
                    "content": case.get("prompt", ""),
                },
                {
                    "role": "assistant",
                    "content": rollout.get("response", ""),
                },
                {
                    "role": "system",
                    "content": f"returncode={rollout.get('returncode', 0)} stderr={rollout.get('stderr', '')}",
                },
            ],
            ensure_ascii=False,
            indent=2,
        )

    def reflect(self, results, skill_content, out_dir, **kwargs):
        backend = str(self.env_config.get("reflection_backend") or "native").strip().lower()
        if backend == "codex_cli":
            return self.reflector.reflect(results, skill_content, out_dir, **kwargs)

        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
        return run_minibatch_reflect(
            results=results,
            skill_content=skill_content,
            prediction_dir=prediction_dir,
            patches_dir=patches_dir,
            workers=self.analyst_workers,
            failure_only=self.failure_only,
            minibatch_size=self.minibatch_size,
            edit_budget=self.edit_budget,
            random_seed=kwargs.get("random_seed"),
            error_system=self.get_error_minibatch_prompt(),
            success_system=self.get_success_minibatch_prompt(),
            step_buffer_context=kwargs.get("step_buffer_context", ""),
            meta_skill_context=kwargs.get("meta_skill_context", ""),
            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
        )
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall