Skip to content

yiouli/pixie-qa / skills/eval-driven-dev

eval-driven-dev

Unverified

Improve AI application with evaluation-driven development. Define eval criteria, instrument the application, build golden datasets, observe and evaluate application runs, analyze results, and produce a concrete action plan for improvements. ALWAYS USE THIS SKILL when the user asks to set up QA, add tests, add evals, evaluate, benchmark, fix wrong behaviors, improve quality, or do quality assurance for any Python project that calls an LLM model.

resources/verify_step6_completion.py

View on GitHub
#!/usr/bin/env python3
"""Validate that eval-driven-dev Step 6 artifacts are complete.

Usage:
    python verify_step6_completion.py /path/to/pixie_qa/results/<test_id>
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

ENTRY_REQUIRED_FILES = ("evaluations.jsonl",)
DATASET_ANALYSIS_FILES = ("analysis.md", "analysis-summary.md")
ROOT_ANALYSIS_FILES = ("action-plan.md", "action-plan-summary.md", "meta.json")


def _dataset_dirs(results_dir: Path) -> list[Path]:
    return sorted(
        path
        for path in results_dir.iterdir()
        if path.is_dir() and path.name.startswith("dataset-")
    )


def _entry_dirs(dataset_dir: Path) -> list[Path]:
    return sorted(
        path
        for path in dataset_dir.iterdir()
        if path.is_dir() and path.name.startswith("entry-")
    )


def _read_jsonl(path: Path, errors: list[str]) -> list[dict[str, object]]:
    rows: list[dict[str, object]] = []
    try:
        for index, line in enumerate(
            path.read_text(encoding="utf-8").splitlines(), start=1
        ):
            if not line.strip():
                continue
            obj = json.loads(line)
            if not isinstance(obj, dict):
                errors.append(f"{path}: line {index} is not a JSON object")
                continue
            rows.append(obj)
    except OSError as exc:
        errors.append(f"{path}: could not read file ({exc})")
    except json.JSONDecodeError as exc:
        errors.append(f"{path}: invalid JSONL ({exc})")
    return rows


def validate_results_dir(results_dir: Path) -> list[str]:
    """Return a list of validation errors for a pixie results directory."""
    errors: list[str] = []

    if not results_dir.is_dir():
        return [f"{results_dir}: results directory not found"]

    for file_name in ROOT_ANALYSIS_FILES:
        if not (results_dir / file_name).is_file():
            errors.append(f"Missing root artifact: {results_dir / file_name}")

    datasets = _dataset_dirs(results_dir)
    if not datasets:
        errors.append(f"{results_dir}: no dataset-* directories found")
        return errors

    for dataset_dir in datasets:
        for file_name in DATASET_ANALYSIS_FILES:
            if not (dataset_dir / file_name).is_file():
                errors.append(f"Missing dataset artifact: {dataset_dir / file_name}")

        entry_dirs = _entry_dirs(dataset_dir)
        if not entry_dirs:
            errors.append(f"{dataset_dir}: no entry-* directories found")
            continue

        for entry_dir in entry_dirs:
            for file_name in ENTRY_REQUIRED_FILES:
                if not (entry_dir / file_name).is_file():
                    errors.append(f"Missing entry artifact: {entry_dir / file_name}")

            evaluations_path = entry_dir / "evaluations.jsonl"
            if not evaluations_path.is_file():
                continue

            evaluations = _read_jsonl(evaluations_path, errors)
            for row in evaluations:
                status = row.get("status")
                if status == "pending":
                    errors.append(
                        "Pending evaluation remains: "
                        f"{evaluations_path} ({row.get('evaluator', 'unknown evaluator')})"
                    )
                    continue

                if "score" not in row:
                    errors.append(
                        "Missing score in scored evaluation: "
                        f"{evaluations_path} ({row.get('evaluator', 'unknown evaluator')})"
                    )
                if "reasoning" not in row:
                    errors.append(
                        "Missing reasoning in scored evaluation: "
                        f"{evaluations_path} ({row.get('evaluator', 'unknown evaluator')})"
                    )

    return errors


def main(argv: list[str] | None = None) -> int:
    """CLI entry point."""
    parser = argparse.ArgumentParser(
        description="Validate Step 6 completion for a pixie results directory"
    )
    parser.add_argument(
        "results_dir",
        type=Path,
        help="Path to pixie_qa/results/<test_id>",
    )
    args = parser.parse_args(argv)

    errors = validate_results_dir(args.results_dir)
    if errors:
        print("Step 6 completion check failed:")
        for error in errors:
            print(f"- {error}")
        return 1

    print("Step 6 completion check passed.")
    return 0


if __name__ == "__main__":
    sys.exit(main())
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 319kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 319kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 319kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 196kUpdated Claude CodeCodexVerifiedInstall