stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup
skillopt-setup
FlaggedSet up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.
assets/agent-skills-benchmark/dataloader.py.template
View on GitHubimport json
import posixpath
import re
from pathlib import Path
from skillopt.datasets.base import SplitDataLoader
REQUIRED_FIELDS = {
"id",
"skill_name",
"case_path",
"prompt",
"expected_behavior",
"rubric_path",
"fixtures",
"split_family",
"split_group",
"expected_artifacts",
"deterministic_assertions",
"visual_assertions",
"tags",
"should_trigger",
"workspace_policy",
"source_hash",
}
WORKSPACE_POLICIES = {"text-only", "isolated-artifact-write"}
def canonical_fixture_path(value):
if (
not isinstance(value, str)
or not value
or value != value.strip()
or "\\" in value
or "|" in value
or any(ord(char) < 32 or ord(char) == 127 for char in value)
or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", value)
or posixpath.isabs(value)
):
return None
normalized = posixpath.normpath(value)
if normalized in (".", "..") or normalized.startswith("../"):
return None
return normalized
def valid_split_family(value):
if not isinstance(value, str) or value != value.strip():
return False
if re.fullmatch(r"[a-z0-9][a-z0-9-]{0,79}", value):
return True
if re.fullmatch(r"case:[a-z0-9][a-z0-9-]{0,127}", value):
return True
if not value.startswith("fixture:"):
return False
fixtures = value.removeprefix("fixture:").split("|")
canonical = [canonical_fixture_path(fixture) for fixture in fixtures]
return bool(fixtures) and all(canonical) and fixtures == sorted(set(canonical))
class AgentSkillsDataLoader(SplitDataLoader):
def __init__(
self,
split_dir="",
data_path="",
split_mode="split_dir",
split_ratio="2:1:7",
split_seed=42,
split_output_dir="",
seed=42,
limit=0,
):
super().__init__(
split_dir=split_dir,
data_path=data_path,
split_mode=split_mode,
split_ratio=split_ratio,
split_seed=split_seed,
split_output_dir=split_output_dir,
seed=seed,
limit=limit,
)
def setup(self, cfg):
super().setup(cfg)
self._validate_split_boundaries(self._splits)
@staticmethod
def _validate_split_boundaries(splits):
skill_names = {item["skill_name"] for items in splits.values() for item in items}
if len(skill_names) > 1:
raise ValueError("skill_name crosses dataset boundaries")
for field in ("split_family", "split_group"):
membership = {}
for split, items in splits.items():
for item in items:
previous = membership.setdefault(item[field], split)
if previous != split:
raise ValueError(
f"{field} {item[field]!r} crosses {previous} and {split} splits"
)
fixture_membership = {}
for split, items in splits.items():
for item in items:
for fixture in item["fixtures"]:
normalized = canonical_fixture_path(fixture)
if normalized is None:
raise ValueError(f"item {item.get('id', '<unknown>')} has invalid fixture")
previous = fixture_membership.setdefault(normalized, split)
if previous != split:
raise ValueError(
f"fixture {normalized!r} crosses {previous} and {split} splits"
)
def load_split_items(self, split_path):
path = Path(split_path) / "items.json"
items = json.loads(path.read_text(encoding="utf-8"))
for item in items:
missing = sorted(REQUIRED_FIELDS - set(item))
if missing:
raise ValueError(f"{path}: item {item.get('id', '<unknown>')} missing {missing}")
if (
not isinstance(item["skill_name"], str)
or re.fullmatch(r"[a-z0-9][a-z0-9-]{0,127}", item["skill_name"]) is None
or not isinstance(item["id"], str)
or re.fullmatch(
rf"{re.escape(item['skill_name'])}/[a-z0-9][a-z0-9-]{{0,127}}",
item["id"],
)
is None
):
raise ValueError(
f"{path}: item {item.get('id', '<unknown>')} has invalid skill_name or id"
)
if not valid_split_family(item["split_family"]):
raise ValueError(
f"{path}: item {item.get('id', '<unknown>')} has invalid split_family"
)
if (
not isinstance(item["split_group"], str)
or re.fullmatch(r"sha256:[0-9a-f]{64}", item["split_group"]) is None
):
raise ValueError(
f"{path}: item {item.get('id', '<unknown>')} has invalid split_group"
)
if not isinstance(item["fixtures"], list) or any(
canonical_fixture_path(fixture) is None
for fixture in item["fixtures"]
):
raise ValueError(
f"{path}: item {item.get('id', '<unknown>')} has invalid fixtures"
)
if item["workspace_policy"] not in WORKSPACE_POLICIES:
raise ValueError(
f"{path}: item {item.get('id', '<unknown>')} has unsupported "
f"workspace_policy {item['workspace_policy']!r}"
)
has_visual_assertions = any(
str(assertion or "").strip()
for assertion in item["visual_assertions"]
)
if (
item["workspace_policy"] == "isolated-artifact-write"
) != has_visual_assertions:
raise ValueError(
f"{path}: item {item.get('id', '<unknown>')} has workspace_policy "
"that does not match its visual assertions"
)
return sorted(items, key=lambda item: item["id"])
def load_split(self, split):
if self._splits:
return self.get_split_items(split)
alias = {
"valid_seen": "val",
"selection": "val",
"valid_unseen": "test",
}.get(split, split)
return self.load_split_items(Path(self.split_dir) / alias)
def load_all(self):
splits = {split: self.load_split(split) for split in ("train", "val", "test")}
self._validate_split_boundaries(splits)
return splits
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |