tzachbon/srulik-toolkit / plugins/srulik-toolkit/skills/skill-creator
skill-creator
VerifiedCreate or edit a skill, improve its instructions or triggers, evaluate it on real tasks, or benchmark a revision. Every skill creation or edit uses create-plan and implement-plan, including small changes; evaluation compares actual outputs and includes human review.
| File | Kind | Size |
|---|---|---|
| LICENSE.txt | markdown | 11.1 KB |
| SKILL.md | skill md | 4.2 KB |
| agents/analyzer.md | markdown | 10.3 KB |
| agents/grader.md | markdown | 9.1 KB |
| agents/openai.yaml | config | 118 B |
| assets/eval_review.html | other | 6.8 KB |
| eval-viewer/generate_review.py | script | 7.2 KB |
| eval-viewer/viewer.html | other | 40.7 KB |
| references/authoring.md | markdown | 2.8 KB |
| references/claude-code.md | markdown | 3.9 KB |
| references/codex.md | markdown | 2.1 KB |
| references/evaluation.md | markdown | 3.9 KB |
| references/hosts.md | markdown | 861 B |
| references/oh-my-pi.md | markdown | 1.2 KB |
| references/schemas.md | markdown | 3.9 KB |
| references/t3-code.md | markdown | 2.2 KB |
| scripts/__init__.py | script | 0 B |
| scripts/aggregate_benchmark.py | script | 9.3 KB |
| scripts/generate_report.py | script | 12.5 KB |
| scripts/improve_description.py | script | 11.3 KB |
| scripts/run_eval.py | script | 12.6 KB |
| scripts/run_loop.py | script | 13.9 KB |
| scripts/utils.py | script | 3.0 KB |
scripts/run_eval.py
View on GitHub#!/usr/bin/env python3
# Modified for Srulik Toolkit: portable evaluation contracts, isolated execution, and offline review. Upstream ca45fc496a3436251f191354091d557da31c15b5; Apache-2.0.
"""Run trigger evaluation for a skill description.
Tests whether a skill's description causes Claude to trigger (read the skill)
for a set of queries. Outputs results as JSON.
"""
import argparse
import json
import os
import select
import signal
import subprocess
import sys
import time
import uuid
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from scripts.utils import parse_skill_md, safe_path, workspace_root
def find_project_root() -> Path:
"""Require an explicit project-local command directory; never discover a parent."""
current = workspace_root(Path.cwd())
if not safe_path(current, current / ".claude").is_dir():
raise ValueError("run from a disposable project containing its own .claude directory")
return current
def run_single_query(
query: str,
skill_name: str,
skill_description: str,
timeout: int,
project_root: str,
model: str | None = None,
) -> bool:
"""Run a single query and return whether the skill was triggered.
Creates a command file in .claude/commands/ so it appears in Claude's
available_skills list, then runs `claude -p` with the raw query.
Uses --include-partial-messages to detect triggering early from
stream events (content_block_start) rather than waiting for the
full assistant message, which only arrives after tool execution.
"""
unique_id = uuid.uuid4().hex[:8]
clean_name = f"{skill_name}-skill-{unique_id}"
project_commands_dir = Path(project_root) / ".claude" / "commands"
command_file = project_commands_dir / f"{clean_name}.md"
try:
safe_path(Path(project_root), command_file, exists=False)
project_commands_dir.mkdir(parents=True, exist_ok=True)
# Use YAML block scalar to avoid breaking on quotes in description
indented_desc = "\n ".join(skill_description.split("\n"))
command_content = (
f"---\n"
f"description: |\n"
f" {indented_desc}\n"
f"---\n\n"
f"# {skill_name}\n\n"
f"This skill handles: {skill_description}\n"
)
command_file.write_text(command_content)
cmd = [
"claude",
"-p", query,
"--output-format", "stream-json",
"--verbose",
"--include-partial-messages",
]
if model:
cmd.extend(["--model", model])
# Remove CLAUDECODE env var to allow nesting claude -p inside a
# Claude Code session. The guard is for interactive terminal conflicts;
# programmatic subprocess usage is safe.
env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
cwd=project_root,
env=env,
start_new_session=True,
)
triggered = False
start_time = time.time()
buffer = ""
# Track state for stream event detection
pending_tool_name = None
accumulated_json = ""
try:
while time.time() - start_time < timeout:
if process.poll() is not None and process.returncode:
raise RuntimeError("claude process failed; trigger result is unknown")
exited = False
ready, _, _ = select.select([process.stdout], [], [], min(1.0, max(0, timeout-(time.time()-start_time))))
if not ready:
continue
chunk = os.read(process.stdout.fileno(), 8192)
if chunk:
buffer += chunk.decode("utf-8", errors="replace")
else:
process.wait(timeout=max(0.1, timeout-(time.time()-start_time)))
if process.returncode:
raise RuntimeError("claude process failed; trigger result is unknown")
exited = True
buffer += "\n"
while "\n" in buffer:
line, buffer = buffer.split("\n", 1)
line = line.strip()
if not line:
continue
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
# Early detection via stream events
if event.get("type") == "stream_event":
se = event.get("event", {})
se_type = se.get("type", "")
if se_type == "content_block_start":
cb = se.get("content_block", {})
tool_name = cb.get("name") if cb.get("type") == "tool_use" else None
pending_tool_name = tool_name if tool_name in ("Skill", "Read") else None
accumulated_json = ""
elif se_type == "content_block_delta" and pending_tool_name:
delta = se.get("delta", {})
if delta.get("type") == "input_json_delta":
accumulated_json += delta.get("partial_json", "")
if clean_name in accumulated_json:
return True
elif se_type in ("content_block_stop", "message_stop"):
if pending_tool_name and clean_name in accumulated_json:
return True
pending_tool_name = None
accumulated_json = ""
# Fallback: full assistant message
elif event.get("type") == "assistant":
message = event.get("message", {})
for content_item in message.get("content", []):
if content_item.get("type") != "tool_use":
continue
tool_name = content_item.get("name", "")
tool_input = content_item.get("input", {})
if tool_name == "Skill" and clean_name in tool_input.get("skill", ""):
triggered = True
elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""):
triggered = True
if triggered:
return True
elif event.get("type") == "result":
if event.get("is_error") or str(event.get("subtype", "")).startswith("error"):
raise RuntimeError("claude reported an error; trigger result is unknown")
return triggered
if exited:
raise RuntimeError("claude ended without a trigger result")
raise TimeoutError("claude trigger evaluation timed out")
finally:
# Clean up process on any exit path (return, exception, timeout)
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
return triggered
finally:
if command_file.exists():
command_file.unlink()
def run_eval(
eval_set: list[dict],
skill_name: str,
description: str,
num_workers: int,
timeout: int,
project_root: Path,
runs_per_query: int = 1,
trigger_threshold: float = 0.5,
model: str | None = None,
) -> dict:
"""Run the full eval set and return results."""
if (not isinstance(eval_set, list) or not eval_set
or any(not isinstance(item, dict) or not isinstance(item.get("query"), str)
or not item["query"].strip() or type(item.get("should_trigger")) is not bool
for item in eval_set)):
raise ValueError("eval set requires nonempty queries with boolean should_trigger")
if min(num_workers, timeout, runs_per_query) < 1 or not 0 <= trigger_threshold <= 1:
raise ValueError("positive limits and a trigger threshold between zero and one are required")
results = []
with ProcessPoolExecutor(max_workers=num_workers) as executor:
future_to_info = {}
for item in eval_set:
for run_idx in range(runs_per_query):
future = executor.submit(
run_single_query,
item["query"],
skill_name,
description,
timeout,
str(project_root),
model,
)
future_to_info[future] = (item, run_idx)
query_triggers: dict[str, list[bool]] = {}
query_items: dict[str, dict] = {}
for future in as_completed(future_to_info):
item, _ = future_to_info[future]
query = item["query"]
query_items[query] = item
if query not in query_triggers:
query_triggers[query] = []
try:
query_triggers[query].append(future.result())
except Exception as e:
raise RuntimeError("trigger evaluation failed; no benchmark produced") from e
for query, triggers in query_triggers.items():
item = query_items[query]
trigger_rate = sum(triggers) / len(triggers)
should_trigger = item["should_trigger"]
if should_trigger:
did_pass = trigger_rate >= trigger_threshold
else:
did_pass = trigger_rate < trigger_threshold
results.append({
"query": query,
"should_trigger": should_trigger,
"trigger_rate": trigger_rate,
"triggers": sum(triggers),
"runs": len(triggers),
"pass": did_pass,
})
passed = sum(1 for r in results if r["pass"])
total = len(results)
return {
"skill_name": skill_name,
"description": description,
"results": results,
"summary": {
"total": total,
"passed": passed,
"failed": total - passed,
},
}
def main():
parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description")
parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
parser.add_argument("--skill-path", required=True, help="Path to skill directory")
parser.add_argument("--description", default=None, help="Override description to test")
parser.add_argument("--num-workers", type=int, default=1, help="Number of parallel workers")
parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)")
parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
args = parser.parse_args()
eval_set = json.loads(Path(args.eval_set).read_text())
skill_path = Path(args.skill_path)
if not (skill_path / "SKILL.md").exists():
print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
sys.exit(1)
name, original_description, content = parse_skill_md(skill_path)
description = args.description or original_description
project_root = find_project_root()
if args.verbose:
print(f"Evaluating: {description}", file=sys.stderr)
output = run_eval(
eval_set=eval_set,
skill_name=name,
description=description,
num_workers=args.num_workers,
timeout=args.timeout,
project_root=project_root,
runs_per_query=args.runs_per_query,
trigger_threshold=args.trigger_threshold,
model=args.model,
)
if args.verbose:
summary = output["summary"]
print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr)
for r in output["results"]:
status = "PASS" if r["pass"] else "FAIL"
rate_str = f"{r['triggers']}/{r['runs']}"
print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr)
print(json.dumps(output, indent=2))
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |