yiouli/pixie-qa / skills/eval-driven-dev
eval-driven-dev
UnverifiedImprove AI application with evaluation-driven development. Define eval criteria, instrument the application, build golden datasets, observe and evaluate application runs, analyze results, and produce a concrete action plan for improvements. ALWAYS USE THIS SKILL when the user asks to set up QA, add tests, add evals, evaluate, benchmark, fix wrong behaviors, improve quality, or do quality assurance for any Python project that calls an LLM model.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 17.4 KB |
| references/1-a-project-analysis.md | markdown | 5.7 KB |
| references/1-b-entry-point.md | markdown | 2.3 KB |
| references/1-c-eval-criteria.md | markdown | 7.5 KB |
| references/2a-instrumentation.md | markdown | 7.3 KB |
| references/2b-implement-runnable.md | markdown | 7.4 KB |
| references/2c-capture-and-verify-trace.md | markdown | 5.7 KB |
| references/3-define-evaluators.md | markdown | 10.3 KB |
| references/4-build-dataset.md | markdown | 21.6 KB |
| references/5-run-tests.md | markdown | 6.0 KB |
| references/6-analyze-outcomes.md | markdown | 16.2 KB |
| references/evaluators.md | markdown | 17.6 KB |
| references/runnable-examples/cli-app.md | markdown | 2.0 KB |
| references/runnable-examples/fastapi-web-server.md | markdown | 4.1 KB |
| references/runnable-examples/standalone-function.md | markdown | 1.8 KB |
| references/testing-api.md | markdown | 16.3 KB |
| references/wrap-api.md | markdown | 8.8 KB |
| resources/setup.shexecutable | script | 3.0 KB |
| resources/verify_step6_completion.py | script | 4.4 KB |
resources/setup.sh
View on GitHub#!/usr/bin/env bash
# Setup script for eval-driven-dev skill.
# Updates the skill, installs/upgrades pixie-qa[all], initializes the
# pixie working directory, and starts the web UI server in the background.
#
# Error handling:
# - Skill update failure → non-fatal (continue with existing version)
# - pixie-qa upgrade failure when already installed → non-fatal
# - pixie-qa NOT installed and install fails → FATAL (exit 1)
# - pixie init failure → FATAL (exit 1)
# - pixie start failure → FATAL (exit 1)
set -u
echo "=== Updating skill ==="
npx skills update yiouli/pixie-qa -g -y && npx skills update yiouli/pixie-qa -p -y || {
echo "(skill update failed — proceeding with existing version)"
}
echo ""
echo "=== Installing / upgrading pixie-qa[all] ==="
# Helper: check if pixie CLI is importable
_pixie_available() {
if [ -f uv.lock ]; then
uv run python -c "import pixie" 2>/dev/null
elif [ -f poetry.lock ]; then
poetry run python -c "import pixie" 2>/dev/null
else
python -c "import pixie" 2>/dev/null
fi
}
# Check if pixie is already installed before attempting upgrade
PIXIE_WAS_INSTALLED=false
if _pixie_available; then
PIXIE_WAS_INSTALLED=true
fi
INSTALL_OK=false
if [ -f uv.lock ]; then
# uv add does universal resolution across all Python versions in
# requires-python. If the host project supports a Python version
# where pixie-qa is unavailable (e.g. <3.10), uv add fails.
# Fall back to uv pip install which only targets the active interpreter.
if uv add "pixie-qa[all]>=0.8.4,<0.9.0" --upgrade 2>&1; then
INSTALL_OK=true
else
echo "(uv add failed — falling back to uv pip install)"
if uv pip install "pixie-qa[all]>=0.8.4,<0.9.0" 2>&1; then
INSTALL_OK=true
fi
fi
elif [ -f poetry.lock ]; then
if poetry add "pixie-qa[all]>=0.8.4,<0.9.0"; then
INSTALL_OK=true
fi
else
if pip install --upgrade "pixie-qa[all]>=0.8.4,<0.9.0"; then
INSTALL_OK=true
fi
fi
if [ "$INSTALL_OK" = false ]; then
if [ "$PIXIE_WAS_INSTALLED" = true ]; then
echo "(pixie-qa upgrade failed — proceeding with existing version)"
else
echo ""
echo "ERROR: pixie-qa is not installed and installation failed."
echo "The eval-driven-dev workflow requires the pixie-qa package."
echo "Please install it manually and re-run this script."
exit 1
fi
fi
echo ""
echo "=== Initializing pixie working directory ==="
if [ -f uv.lock ]; then
uv run pixie init
elif [ -f poetry.lock ]; then
poetry run pixie init
else
pixie init
fi
if [ $? -ne 0 ]; then
echo ""
echo "ERROR: Failed to initialize pixie working directory."
echo "Please check the error above and fix it before continuing."
exit 1
fi
echo ""
echo "=== Starting web UI server (background) ==="
if [ -f uv.lock ]; then
uv run pixie start
elif [ -f poetry.lock ]; then
poetry run pixie start
else
pixie start
fi
if [ $? -ne 0 ]; then
echo ""
echo "ERROR: Failed to start the web UI server."
echo "Please check the error above and fix it before continuing."
exit 1
fi
echo ""
echo "=== Setup complete ==="
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |