avibebuilder/claude-prime / .claude/skills/skill-creator
skill-creator
VerifiedUse when the user wants to work on a Claude Code skill file (SKILL.md): writing one from scratch, testing whether an existing one works well, running evals or benchmarks, improving its instructions, or fixing why it isn't triggering. Triggers on: 'make a skill for X', 'test this skill', 'run evals on my SKILL.md', 'touch-skill', sharing a SKILL.md and asking if it's ready to ship. The key signal is intent to create, validate, or improve a skill — not just mention one. Do NOT trigger for general Claude Code questions, hook debugging, or CLAUDE.md configuration.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 32.3 KB |
| agents/analyzer.md | markdown | 10.1 KB |
| agents/comparator.md | markdown | 7.1 KB |
| agents/grader.md | markdown | 10.5 KB |
| agents/judge.md | markdown | 6.0 KB |
| assets/eval_review.html | other | 6.9 KB |
| eval-viewer/generate_report.py | script | 17.4 KB |
| eval-viewer/generate_review.py | script | 16.0 KB |
| eval-viewer/report.html | other | 91.8 KB |
| eval-viewer/viewer.html | other | 43.9 KB |
| references/available-skill-resources.md | markdown | 2.0 KB |
| references/schemas.md | markdown | 15.6 KB |
| scripts/__init__.py | script | 0 B |
| scripts/aggregate_benchmark.pyexecutable | script | 21.4 KB |
| scripts/generate_report.pyexecutable | script | 12.5 KB |
| scripts/improve_description.pyexecutable | script | 10.9 KB |
| scripts/package_skill.pyexecutable | script | 4.1 KB |
| scripts/quick_validate.pyexecutable | script | 4.4 KB |
| scripts/run_eval.pyexecutable | script | 13.1 KB |
| scripts/run_loop.pyexecutable | script | 13.7 KB |
| scripts/utils.py | script | 1.6 KB |
scripts/run_eval.py
View on GitHub#!/usr/bin/env python3
"""Run trigger evaluation for a skill description.
Tests whether a skill's description causes Claude to trigger (read the skill)
for a set of queries. Outputs results as JSON.
"""
import argparse
import atexit
import json
import os
import select
import shutil
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from scripts.utils import parse_skill_md
def find_project_root() -> Path:
"""Find the project root by walking up from cwd looking for .claude/.
Mimics how Claude Code discovers its project root, so the command file
we create ends up where claude -p will look for it.
"""
current = Path.cwd()
for parent in [current, *current.parents]:
if (parent / ".claude").is_dir():
return parent
return current
def _restore_backed_up_skills(skills_dir: Path, backup_dir: Path) -> None:
"""Restore any skills from the backup directory back to their original location."""
if not backup_dir.exists():
return
for skill_backup in backup_dir.iterdir():
if skill_backup.is_dir():
target = skills_dir / skill_backup.name
if target.exists():
shutil.rmtree(target)
shutil.move(str(skill_backup), str(target))
print(f"Restored skill from backup: {skill_backup.name}", file=sys.stderr)
# Clean up backup dir if empty
if backup_dir.exists() and not any(backup_dir.iterdir()):
backup_dir.rmdir()
def run_single_query(
query: str,
skill_name: str,
timeout: int,
project_root: str,
model: str | None = None,
) -> bool:
"""Run a single query and return whether the skill was triggered.
Expects that a command file already exists in .claude/commands/ (created
by run_eval). Runs `claude -p` with the raw query and checks whether
Claude invokes the Skill tool with the real skill name.
Uses --include-partial-messages to detect triggering early from
stream events (content_block_start) rather than waiting for the
full assistant message, which only arrives after tool execution.
"""
cmd = [
"claude",
"-p", query,
"--output-format", "stream-json",
"--verbose",
"--include-partial-messages",
]
if model:
cmd.extend(["--model", model])
# Remove CLAUDECODE env var to allow nesting claude -p inside a
# Claude Code session. The guard is for interactive terminal conflicts;
# programmatic subprocess usage is safe.
env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
env["CLAUDE_SILENT"] = "1"
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
cwd=project_root,
env=env,
)
triggered = False
start_time = time.time()
buffer = ""
# Track state for stream event detection
pending_tool_name = None
accumulated_json = ""
try:
while time.time() - start_time < timeout:
if process.poll() is not None:
remaining = process.stdout.read()
if remaining:
buffer += remaining.decode("utf-8", errors="replace")
break
ready, _, _ = select.select([process.stdout], [], [], 1.0)
if not ready:
continue
chunk = os.read(process.stdout.fileno(), 8192)
if not chunk:
break
buffer += chunk.decode("utf-8", errors="replace")
while "\n" in buffer:
line, buffer = buffer.split("\n", 1)
line = line.strip()
if not line:
continue
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
# Early detection via stream events
if event.get("type") == "stream_event":
se = event.get("event", {})
se_type = se.get("type", "")
if se_type == "content_block_start":
cb = se.get("content_block", {})
if cb.get("type") == "tool_use":
tool_name = cb.get("name", "")
if tool_name in ("Skill", "Read"):
pending_tool_name = tool_name
accumulated_json = ""
else:
return False
elif se_type == "content_block_delta" and pending_tool_name:
delta = se.get("delta", {})
if delta.get("type") == "input_json_delta":
accumulated_json += delta.get("partial_json", "")
if skill_name in accumulated_json:
return True
elif se_type in ("content_block_stop", "message_stop"):
if pending_tool_name:
return skill_name in accumulated_json
if se_type == "message_stop":
return False
# Fallback: full assistant message
elif event.get("type") == "assistant":
message = event.get("message", {})
for content_item in message.get("content", []):
if content_item.get("type") != "tool_use":
continue
tool_name = content_item.get("name", "")
tool_input = content_item.get("input", {})
if tool_name == "Skill" and skill_name in tool_input.get("skill", ""):
triggered = True
elif tool_name == "Read" and skill_name in tool_input.get("file_path", ""):
triggered = True
return triggered
elif event.get("type") == "result":
return triggered
finally:
# Clean up process on any exit path (return, exception, timeout)
if process.poll() is None:
process.kill()
process.wait()
return triggered
def run_eval(
eval_set: list[dict],
skill_name: str,
description: str,
num_workers: int,
timeout: int,
project_root: Path,
runs_per_query: int = 1,
trigger_threshold: float = 0.5,
model: str | None = None,
) -> dict:
"""Run the full eval set and return results.
Sets up a temporary command file with the real skill name and backs up
any existing skill folder so Claude sees only the test command file.
Restores everything in a finally block (and via atexit for crash safety).
"""
skills_dir = Path(project_root) / ".claude" / "skills"
backup_dir = skills_dir / ".eval-backup"
skill_dir = skills_dir / skill_name
commands_dir = Path(project_root) / ".claude" / "commands"
command_file = commands_dir / f"{skill_name}.md"
# --- Startup recovery: restore from any previous crashed run ---
if backup_dir.exists() and any(backup_dir.iterdir()):
print("Warning: found leftover .eval-backup from a crashed run, restoring...", file=sys.stderr)
_restore_backed_up_skills(skills_dir, backup_dir)
# --- Setup: backup real skill, create command file ---
atexit_registered = False
def _atexit_restore():
"""Crash recovery: restore backed-up skill and remove command file."""
if command_file.exists():
command_file.unlink()
_restore_backed_up_skills(skills_dir, backup_dir)
try:
# Backup existing skill if it exists
if skill_dir.exists():
backup_dir.mkdir(parents=True, exist_ok=True)
shutil.move(str(skill_dir), str(backup_dir / skill_name))
# Create command file with real skill name
commands_dir.mkdir(parents=True, exist_ok=True)
indented_desc = "\n ".join(description.split("\n"))
command_content = (
f"---\n"
f"description: |\n"
f" {indented_desc}\n"
f"---\n\n"
f"# {skill_name}\n\n"
f"This skill handles: {description}\n"
)
command_file.write_text(command_content)
# Register atexit handler for crash safety
atexit.register(_atexit_restore)
atexit_registered = True
# --- Run all queries in parallel ---
results = []
with ThreadPoolExecutor(max_workers=num_workers) as executor:
future_to_info = {}
for item in eval_set:
for run_idx in range(runs_per_query):
future = executor.submit(
run_single_query,
item["query"],
skill_name,
timeout,
str(project_root),
model,
)
future_to_info[future] = (item, run_idx)
query_triggers: dict[str, list[bool]] = {}
query_items: dict[str, dict] = {}
for future in as_completed(future_to_info):
item, _ = future_to_info[future]
query = item["query"]
query_items[query] = item
if query not in query_triggers:
query_triggers[query] = []
try:
query_triggers[query].append(future.result())
except Exception as e:
print(f"Warning: query failed: {e}", file=sys.stderr)
query_triggers[query].append(False)
finally:
# --- Teardown: remove command file, restore backup ---
if command_file.exists():
command_file.unlink()
_restore_backed_up_skills(skills_dir, backup_dir)
# Unregister atexit handler since we've already cleaned up
if atexit_registered:
atexit.unregister(_atexit_restore)
for query, triggers in query_triggers.items():
item = query_items[query]
trigger_rate = sum(triggers) / len(triggers)
should_trigger = item["should_trigger"]
if should_trigger:
did_pass = trigger_rate >= trigger_threshold
else:
did_pass = trigger_rate < trigger_threshold
results.append({
"query": query,
"should_trigger": should_trigger,
"trigger_rate": trigger_rate,
"triggers": sum(triggers),
"runs": len(triggers),
"pass": did_pass,
})
passed = sum(1 for r in results if r["pass"])
total = len(results)
return {
"skill_name": skill_name,
"description": description,
"results": results,
"summary": {
"total": total,
"passed": passed,
"failed": total - passed,
},
}
def main():
parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description")
parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
parser.add_argument("--skill-path", required=True, help="Path to skill directory")
parser.add_argument("--description", default=None, help="Override description to test")
parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)")
parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
args = parser.parse_args()
eval_set = json.loads(Path(args.eval_set).read_text())
skill_path = Path(args.skill_path)
if not (skill_path / "SKILL.md").exists():
print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
sys.exit(1)
name, original_description, content = parse_skill_md(skill_path)
description = args.description or original_description
project_root = find_project_root()
if args.verbose:
print(f"Evaluating: {description}", file=sys.stderr)
output = run_eval(
eval_set=eval_set,
skill_name=name,
description=description,
num_workers=args.num_workers,
timeout=args.timeout,
project_root=project_root,
runs_per_query=args.runs_per_query,
trigger_threshold=args.trigger_threshold,
model=args.model,
)
if args.verbose:
summary = output["summary"]
print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr)
for r in output["results"]:
status = "PASS" if r["pass"] else "FAIL"
rate_str = f"{r['triggers']}/{r['runs']}"
print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr)
print(json.dumps(output, indent=2))
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 197k | Updated | Claude CodeCodex | Verified | Install |