mgiovani/ai-arsenal / skills/create-skill
create-skill
VerifiedCreate a new agent skill (or Claude Code slash command) from a plain-language description, using live spec fetching, pattern research, and an approval-gated blueprint before any files are written. Use whenever the user wants to build, scaffold, or author a new skill, subagent capability, or slash command, including phrasings like 'make a command for X', 'create a slash command', 'turn this into a reusable skill', or 'package this workflow as a skill'. Not for editing CLAUDE.md/AGENTS.md memory rules (use create-rule) or discovering/installing existing third-party skills (use find-skills).
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 20.3 KB |
| evals/evals.json | config | 3.5 KB |
| evals/trigger-eval.json | config | 2.2 KB |
| references/agent-prompts.md | markdown | 4.5 KB |
| references/eval-system.md | markdown | 5.6 KB |
| references/frontmatter-fields.md | markdown | 2.8 KB |
| references/schemas.md | markdown | 6.3 KB |
| references/skill-anatomy.md | markdown | 10.2 KB |
| references/specification-urls.md | markdown | 2.6 KB |
| scripts/generate_report.py | script | 11.5 KB |
| scripts/improve_description.py | script | 16.7 KB |
| scripts/package_skill.py | script | 4.5 KB |
| scripts/quick_validate.pyexecutable | script | 11.1 KB |
| scripts/run_eval.py | script | 8.1 KB |
scripts/run_eval.py
View on GitHub#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.11"
# dependencies = [
# "click",
# "rich",
# ]
# ///
"""
Eval runner for agent skills.
Runs with-skill and baseline comparisons for all evals defined in evals/evals.json.
Requires the `claude` CLI to be installed and authenticated.
Usage:
uv run python skills/create-skill/scripts/run_eval.py <skill_path>
uv run python skills/create-skill/scripts/run_eval.py <skill_path> --eval eval-1
uv run python skills/create-skill/scripts/run_eval.py <skill_path> \\
--model claude-haiku-5-5
Exit codes:
0 - all evals ran successfully
1 - one or more evals failed to run
"""
from __future__ import annotations
import json
import subprocess
import sys
import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
import click
from rich.console import Console
from rich.progress import (
BarColumn,
Progress,
SpinnerColumn,
TaskProgressColumn,
TextColumn,
)
from rich.table import Table
console = Console()
DEFAULT_MODEL = 'claude-haiku-5-5'
MAX_WORKERS = 4
def load_evals(skill_path: Path) -> dict:
"""Load eval definitions from evals/evals.json."""
evals_file = skill_path / 'evals' / 'evals.json'
if not evals_file.exists():
msg = f'evals/evals.json not found in {skill_path}'
raise FileNotFoundError(msg)
return json.loads(evals_file.read_text(encoding='utf-8'))
def run_claude(
prompt: str,
model: str,
) -> tuple[str, float]:
"""Run a claude -p call and return (output, duration_seconds).
Args:
prompt: The prompt to send
model: Claude model to use
Returns:
Tuple of (output_text, duration_seconds)
"""
cmd = ['claude', '-p', prompt, '--model', model, '--output-format', 'text']
start = time.time()
try:
result = subprocess.run(
cmd,
check=False,
capture_output=True,
text=True,
timeout=300,
)
duration = time.time() - start
if result.returncode != 0:
error_msg = result.stderr.strip() or 'Unknown error'
return f'ERROR: claude -p failed: {error_msg}', duration
return result.stdout.strip(), duration
except subprocess.TimeoutExpired:
return 'ERROR: claude -p timed out after 300s', time.time() - start
except FileNotFoundError:
return (
'ERROR: claude CLI not found. Install from https://claude.ai/code',
time.time() - start,
)
def run_single_eval(
eval_case: dict,
_skill_path: Path,
model: str,
results_dir: Path,
) -> dict:
"""Run one eval case: with-skill + baseline.
Returns dict with eval results.
"""
eval_id = eval_case['id']
prompt = eval_case['prompt']
eval_dir = results_dir / eval_id
eval_dir.mkdir(parents=True, exist_ok=True)
# Run with-skill (skill active) and baseline (no skill active).
# skill_path is logged in results for traceability; claude -p determines
# skill loading from its own environment configuration.
with_skill_output, with_skill_duration = run_claude(prompt, model)
# Run baseline (no skill)
baseline_output, baseline_duration = run_claude(prompt, model)
(eval_dir / 'with_skill.txt').write_text(with_skill_output, encoding='utf-8')
(eval_dir / 'baseline.txt').write_text(baseline_output, encoding='utf-8')
timing = {
'eval_id': eval_id,
'with_skill_duration_s': round(with_skill_duration, 2),
'baseline_duration_s': round(baseline_duration, 2),
'model': model,
}
(eval_dir / 'timing.json').write_text(json.dumps(timing, indent=2), encoding='utf-8')
return {
'eval_id': eval_id,
'success': not with_skill_output.startswith('ERROR'),
'with_skill': with_skill_output,
'baseline': baseline_output,
'timing': timing,
}
def run_all_evals(
skill_path: Path,
evals_data: dict,
model: str,
only_eval: str | None = None,
) -> list[dict]:
"""Run all evals (or a single one if specified).
Uses ProcessPoolExecutor for parallel execution.
Returns list of result dicts.
"""
results_dir = skill_path / 'evals' / 'results'
results_dir.mkdir(parents=True, exist_ok=True)
eval_cases = evals_data.get('evals', [])
if only_eval:
eval_cases = [e for e in eval_cases if e['id'] == only_eval]
if not eval_cases:
console.print(f'[red]Eval not found: {only_eval}[/red]')
sys.exit(1)
results: list[dict] = []
with Progress(
SpinnerColumn(),
TextColumn('[progress.description]{task.description}'),
BarColumn(),
TaskProgressColumn(),
console=console,
) as progress:
task = progress.add_task('Running evals...', total=len(eval_cases))
# Each eval runs 2 claude calls, do them with limited parallelism
with ProcessPoolExecutor(
max_workers=min(MAX_WORKERS, len(eval_cases))
) as executor:
futures = {
executor.submit(
run_single_eval, eval_case, skill_path, model, results_dir
): eval_case['id']
for eval_case in eval_cases
}
for future in as_completed(futures):
eval_id = futures[future]
try:
result = future.result()
results.append(result)
status = '[green]✓[/green]' if result['success'] else '[red]✗[/red]'
progress.update(
task,
advance=1,
description=f'{status} {eval_id}',
)
except Exception as e: # noqa: BLE001
results.append(
{'eval_id': eval_id, 'success': False, 'error': str(e)}
)
progress.update(
task, advance=1, description=f'[red]✗ {eval_id}[/red]'
)
return results
def print_summary(results: list[dict]) -> None:
"""Print a summary table of eval run results."""
table = Table(title='Eval Run Summary', show_header=True)
table.add_column('Eval ID', style='cyan')
table.add_column('Status', justify='center')
table.add_column('With-Skill (s)', justify='right')
table.add_column('Baseline (s)', justify='right')
for result in sorted(results, key=lambda r: r['eval_id']):
status = '[green]OK[/green]' if result.get('success') else '[red]FAIL[/red]'
timing = result.get('timing', {})
ws_time = str(timing.get('with_skill_duration_s', '-'))
bl_time = str(timing.get('baseline_duration_s', '-'))
table.add_row(result['eval_id'], status, ws_time, bl_time)
console.print(table)
success_count = sum(1 for r in results if r.get('success'))
console.print(
f'\n[bold]Results:[/bold] {success_count}/{len(results)} evals ran successfully'
)
console.print('[dim]Next: run generate_report.py after grading is complete[/dim]')
@click.command()
@click.argument('skill_path', type=click.Path(exists=True, path_type=Path))
@click.option('--eval', 'only_eval', default=None, help='Run only this eval ID')
@click.option('--model', default=DEFAULT_MODEL, help='Claude model to use')
def main(skill_path: Path, only_eval: str | None, model: str) -> None:
"""Run evals for a skill: with-skill vs baseline comparison."""
console.print(f'[bold blue]Running evals for:[/bold blue] {skill_path.name}')
try:
evals_data = load_evals(skill_path)
except FileNotFoundError as e:
console.print(f'[red]{e}[/red]')
sys.exit(1)
skill_name = evals_data.get('skill', skill_path.name)
eval_count = len(evals_data.get('evals', []))
console.print(f'Skill: {skill_name} | Evals: {eval_count} | Model: {model}')
console.print()
results = run_all_evals(skill_path, evals_data, model, only_eval)
print_summary(results)
failed = [r for r in results if not r.get('success')]
sys.exit(1 if failed else 0)
if __name__ == '__main__':
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |