Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

assets/agent-skills-benchmark/dataloader.py.template

View on GitHub
import json
import posixpath
import re
from pathlib import Path

from skillopt.datasets.base import SplitDataLoader


REQUIRED_FIELDS = {
    "id",
    "skill_name",
    "case_path",
    "prompt",
    "expected_behavior",
    "rubric_path",
    "fixtures",
    "split_family",
    "split_group",
    "expected_artifacts",
    "deterministic_assertions",
    "visual_assertions",
    "tags",
    "should_trigger",
    "workspace_policy",
    "source_hash",
}
WORKSPACE_POLICIES = {"text-only", "isolated-artifact-write"}


def canonical_fixture_path(value):
    if (
        not isinstance(value, str)
        or not value
        or value != value.strip()
        or "\\" in value
        or "|" in value
        or any(ord(char) < 32 or ord(char) == 127 for char in value)
        or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", value)
        or posixpath.isabs(value)
    ):
        return None
    normalized = posixpath.normpath(value)
    if normalized in (".", "..") or normalized.startswith("../"):
        return None
    return normalized


def valid_split_family(value):
    if not isinstance(value, str) or value != value.strip():
        return False
    if re.fullmatch(r"[a-z0-9][a-z0-9-]{0,79}", value):
        return True
    if re.fullmatch(r"case:[a-z0-9][a-z0-9-]{0,127}", value):
        return True
    if not value.startswith("fixture:"):
        return False
    fixtures = value.removeprefix("fixture:").split("|")
    canonical = [canonical_fixture_path(fixture) for fixture in fixtures]
    return bool(fixtures) and all(canonical) and fixtures == sorted(set(canonical))


class AgentSkillsDataLoader(SplitDataLoader):
    def __init__(
        self,
        split_dir="",
        data_path="",
        split_mode="split_dir",
        split_ratio="2:1:7",
        split_seed=42,
        split_output_dir="",
        seed=42,
        limit=0,
    ):
        super().__init__(
            split_dir=split_dir,
            data_path=data_path,
            split_mode=split_mode,
            split_ratio=split_ratio,
            split_seed=split_seed,
            split_output_dir=split_output_dir,
            seed=seed,
            limit=limit,
        )

    def setup(self, cfg):
        super().setup(cfg)
        self._validate_split_boundaries(self._splits)

    @staticmethod
    def _validate_split_boundaries(splits):
        skill_names = {item["skill_name"] for items in splits.values() for item in items}
        if len(skill_names) > 1:
            raise ValueError("skill_name crosses dataset boundaries")
        for field in ("split_family", "split_group"):
            membership = {}
            for split, items in splits.items():
                for item in items:
                    previous = membership.setdefault(item[field], split)
                    if previous != split:
                        raise ValueError(
                            f"{field} {item[field]!r} crosses {previous} and {split} splits"
                        )
        fixture_membership = {}
        for split, items in splits.items():
            for item in items:
                for fixture in item["fixtures"]:
                    normalized = canonical_fixture_path(fixture)
                    if normalized is None:
                        raise ValueError(f"item {item.get('id', '<unknown>')} has invalid fixture")
                    previous = fixture_membership.setdefault(normalized, split)
                    if previous != split:
                        raise ValueError(
                            f"fixture {normalized!r} crosses {previous} and {split} splits"
                        )

    def load_split_items(self, split_path):
        path = Path(split_path) / "items.json"
        items = json.loads(path.read_text(encoding="utf-8"))
        for item in items:
            missing = sorted(REQUIRED_FIELDS - set(item))
            if missing:
                raise ValueError(f"{path}: item {item.get('id', '<unknown>')} missing {missing}")
            if (
                not isinstance(item["skill_name"], str)
                or re.fullmatch(r"[a-z0-9][a-z0-9-]{0,127}", item["skill_name"]) is None
                or not isinstance(item["id"], str)
                or re.fullmatch(
                    rf"{re.escape(item['skill_name'])}/[a-z0-9][a-z0-9-]{{0,127}}",
                    item["id"],
                )
                is None
            ):
                raise ValueError(
                    f"{path}: item {item.get('id', '<unknown>')} has invalid skill_name or id"
                )
            if not valid_split_family(item["split_family"]):
                raise ValueError(
                    f"{path}: item {item.get('id', '<unknown>')} has invalid split_family"
                )
            if (
                not isinstance(item["split_group"], str)
                or re.fullmatch(r"sha256:[0-9a-f]{64}", item["split_group"]) is None
            ):
                raise ValueError(
                    f"{path}: item {item.get('id', '<unknown>')} has invalid split_group"
                )
            if not isinstance(item["fixtures"], list) or any(
                canonical_fixture_path(fixture) is None
                for fixture in item["fixtures"]
            ):
                raise ValueError(
                    f"{path}: item {item.get('id', '<unknown>')} has invalid fixtures"
                )
            if item["workspace_policy"] not in WORKSPACE_POLICIES:
                raise ValueError(
                    f"{path}: item {item.get('id', '<unknown>')} has unsupported "
                    f"workspace_policy {item['workspace_policy']!r}"
                )
            has_visual_assertions = any(
                str(assertion or "").strip()
                for assertion in item["visual_assertions"]
            )
            if (
                item["workspace_policy"] == "isolated-artifact-write"
            ) != has_visual_assertions:
                raise ValueError(
                    f"{path}: item {item.get('id', '<unknown>')} has workspace_policy "
                    "that does not match its visual assertions"
                )
        return sorted(items, key=lambda item: item["id"])

    def load_split(self, split):
        if self._splits:
            return self.get_split_items(split)
        alias = {
            "valid_seen": "val",
            "selection": "val",
            "valid_unseen": "test",
        }.get(split, split)
        return self.load_split_items(Path(self.split_dir) / alias)

    def load_all(self):
        splits = {split: self.load_split(split) for split in ("train", "val", "test")}
        self._validate_split_boundaries(splits)
        return splits
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall