yiouli/pixie-qa / skills/eval-driven-dev
eval-driven-dev
UnverifiedImprove AI application with evaluation-driven development. Define eval criteria, instrument the application, build golden datasets, observe and evaluate application runs, analyze results, and produce a concrete action plan for improvements. ALWAYS USE THIS SKILL when the user asks to set up QA, add tests, add evals, evaluate, benchmark, fix wrong behaviors, improve quality, or do quality assurance for any Python project that calls an LLM model.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 17.4 KB |
| references/1-a-project-analysis.md | markdown | 5.7 KB |
| references/1-b-entry-point.md | markdown | 2.3 KB |
| references/1-c-eval-criteria.md | markdown | 7.5 KB |
| references/2a-instrumentation.md | markdown | 7.3 KB |
| references/2b-implement-runnable.md | markdown | 7.4 KB |
| references/2c-capture-and-verify-trace.md | markdown | 5.7 KB |
| references/3-define-evaluators.md | markdown | 10.3 KB |
| references/4-build-dataset.md | markdown | 21.6 KB |
| references/5-run-tests.md | markdown | 6.0 KB |
| references/6-analyze-outcomes.md | markdown | 16.2 KB |
| references/evaluators.md | markdown | 17.6 KB |
| references/runnable-examples/cli-app.md | markdown | 2.0 KB |
| references/runnable-examples/fastapi-web-server.md | markdown | 4.1 KB |
| references/runnable-examples/standalone-function.md | markdown | 1.8 KB |
| references/testing-api.md | markdown | 16.3 KB |
| references/wrap-api.md | markdown | 8.8 KB |
| resources/setup.shexecutable | script | 3.0 KB |
| resources/verify_step6_completion.py | script | 4.4 KB |
resources/verify_step6_completion.py
View on GitHub#!/usr/bin/env python3
"""Validate that eval-driven-dev Step 6 artifacts are complete.
Usage:
python verify_step6_completion.py /path/to/pixie_qa/results/<test_id>
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ENTRY_REQUIRED_FILES = ("evaluations.jsonl",)
DATASET_ANALYSIS_FILES = ("analysis.md", "analysis-summary.md")
ROOT_ANALYSIS_FILES = ("action-plan.md", "action-plan-summary.md", "meta.json")
def _dataset_dirs(results_dir: Path) -> list[Path]:
return sorted(
path
for path in results_dir.iterdir()
if path.is_dir() and path.name.startswith("dataset-")
)
def _entry_dirs(dataset_dir: Path) -> list[Path]:
return sorted(
path
for path in dataset_dir.iterdir()
if path.is_dir() and path.name.startswith("entry-")
)
def _read_jsonl(path: Path, errors: list[str]) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
try:
for index, line in enumerate(
path.read_text(encoding="utf-8").splitlines(), start=1
):
if not line.strip():
continue
obj = json.loads(line)
if not isinstance(obj, dict):
errors.append(f"{path}: line {index} is not a JSON object")
continue
rows.append(obj)
except OSError as exc:
errors.append(f"{path}: could not read file ({exc})")
except json.JSONDecodeError as exc:
errors.append(f"{path}: invalid JSONL ({exc})")
return rows
def validate_results_dir(results_dir: Path) -> list[str]:
"""Return a list of validation errors for a pixie results directory."""
errors: list[str] = []
if not results_dir.is_dir():
return [f"{results_dir}: results directory not found"]
for file_name in ROOT_ANALYSIS_FILES:
if not (results_dir / file_name).is_file():
errors.append(f"Missing root artifact: {results_dir / file_name}")
datasets = _dataset_dirs(results_dir)
if not datasets:
errors.append(f"{results_dir}: no dataset-* directories found")
return errors
for dataset_dir in datasets:
for file_name in DATASET_ANALYSIS_FILES:
if not (dataset_dir / file_name).is_file():
errors.append(f"Missing dataset artifact: {dataset_dir / file_name}")
entry_dirs = _entry_dirs(dataset_dir)
if not entry_dirs:
errors.append(f"{dataset_dir}: no entry-* directories found")
continue
for entry_dir in entry_dirs:
for file_name in ENTRY_REQUIRED_FILES:
if not (entry_dir / file_name).is_file():
errors.append(f"Missing entry artifact: {entry_dir / file_name}")
evaluations_path = entry_dir / "evaluations.jsonl"
if not evaluations_path.is_file():
continue
evaluations = _read_jsonl(evaluations_path, errors)
for row in evaluations:
status = row.get("status")
if status == "pending":
errors.append(
"Pending evaluation remains: "
f"{evaluations_path} ({row.get('evaluator', 'unknown evaluator')})"
)
continue
if "score" not in row:
errors.append(
"Missing score in scored evaluation: "
f"{evaluations_path} ({row.get('evaluator', 'unknown evaluator')})"
)
if "reasoning" not in row:
errors.append(
"Missing reasoning in scored evaluation: "
f"{evaluations_path} ({row.get('evaluator', 'unknown evaluator')})"
)
return errors
def main(argv: list[str] | None = None) -> int:
"""CLI entry point."""
parser = argparse.ArgumentParser(
description="Validate Step 6 completion for a pixie results directory"
)
parser.add_argument(
"results_dir",
type=Path,
help="Path to pixie_qa/results/<test_id>",
)
args = parser.parse_args(argv)
errors = validate_results_dir(args.results_dir)
if errors:
print("Step 6 completion check failed:")
for error in errors:
print(f"- {error}")
return 1
print("Step 6 completion check passed.")
return 0
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |