aiskillstore/marketplace / skills/7alexhale5-rgb/regression-test
regression-test
VerifiedManage golden dataset regression tests for LLM prompts using Promptfoo. Commands — init, add, run, report. Use when user says "regression", "golden dataset", "prompt regression", "eval run", "test my prompts", or after shipping prompt changes.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 5.8 KB |
| skill-report.json | config | 40.5 KB |
skill-report.json
View on GitHub{
"schema_version": "2.0",
"meta": {
"generated_at": "2026-08-21T01:02:35.960Z",
"slug": "7alexhale5-rgb-regression-test",
"source_url": "https://github.com/7alexhale5-rgb/alexhale-skills/tree/e0f44c48adcc2ac8123a4a8dec2c02503d1c5945/skills/regression-test",
"source_ref": "e0f44c48adcc2ac8123a4a8dec2c02503d1c5945",
"model": "codex",
"analysis_version": "3.0.0",
"source_type": "community",
"content_hash": "2e1e221850ad08c9c6912478029a97726f9bd97812a83c24c9827563bbc42c03",
"tree_hash": "243c3dd590a95db2cbb6a19990dea2e34348d3536a863981205d652da635341f",
"provenance": {
"model_effective": "codex",
"fallback_chain": [
{
"agent": "codex",
"outcome": "success"
}
],
"analysis_version": "3.0.0"
},
"upstream_version_raw": null,
"upstream_version_normalized": null,
"upstream_version_source": "none",
"upstream_version_status": "missing",
"upstream_commit_sha": "e0f44c48adcc2ac8123a4a8dec2c02503d1c5945"
},
"skill": {
"name": "regression-test",
"description": "Create and manage Promptfoo golden datasets for LLM prompt regression testing. Initialize files, add cases, run evaluations, and summarize recent trends.",
"summary": "Run repeatable Promptfoo checks to detect LLM prompt regressions.",
"icon": "📦",
"author": "7alexhale5-rgb",
"license": "MIT",
"tags": [
"regression-testing",
"promptfoo",
"llm-evaluation",
"golden-datasets",
"prompt-engineering"
],
"supported_tools": [
"claude",
"codex",
"claude-code"
],
"risk_factors": [
"external_commands",
"network",
"filesystem"
],
"category": "coding"
},
"security_audit": {
"risk_level": "safe",
"is_blocked": false,
"safe_to_publish": true,
"analysis_status": "ok",
"summary": "All 38 static findings are false positives caused by Markdown syntax, bounded shell examples, or a Promptfoo message template. No malicious intent, prompt injection, credential handling, or unauthorized network behavior was found.",
"remediation": [],
"risk_factor_evidence": [
{
"factor": "external_commands",
"evidence": [
{
"file": "SKILL.md",
"line_start": 14,
"line_end": 14
},
{
"file": "SKILL.md",
"line_start": 20,
"line_end": 20
},
{
"file": "SKILL.md",
"line_start": 25,
"line_end": 27
},
{
"file": "SKILL.md",
"line_start": 27,
"line_end": 29
},
{
"file": "SKILL.md",
"line_start": 29,
"line_end": 29
},
{
"file": "SKILL.md",
"line_start": 30,
"line_end": 33
},
{
"file": "SKILL.md",
"line_start": 33,
"line_end": 46
},
{
"file": "SKILL.md",
"line_start": 46,
"line_end": 48
},
{
"file": "SKILL.md",
"line_start": 48,
"line_end": 48
},
{
"file": "SKILL.md",
"line_start": 50,
"line_end": 50
},
{
"file": "SKILL.md",
"line_start": 54,
"line_end": 58
},
{
"file": "SKILL.md",
"line_start": 58,
"line_end": 58
},
{
"file": "SKILL.md",
"line_start": 67,
"line_end": 81
},
{
"file": "SKILL.md",
"line_start": 81,
"line_end": 83
},
{
"file": "SKILL.md",
"line_start": 83,
"line_end": 87
},
{
"file": "SKILL.md",
"line_start": 87,
"line_end": 92
},
{
"file": "SKILL.md",
"line_start": 92,
"line_end": 95
},
{
"file": "SKILL.md",
"line_start": 95,
"line_end": 98
},
{
"file": "SKILL.md",
"line_start": 98,
"line_end": 103
},
{
"file": "SKILL.md",
"line_start": 103,
"line_end": 110
},
{
"file": "SKILL.md",
"line_start": 110,
"line_end": 119
},
{
"file": "SKILL.md",
"line_start": 119,
"line_end": 126
},
{
"file": "SKILL.md",
"line_start": 126,
"line_end": 131
},
{
"file": "SKILL.md",
"line_start": 131,
"line_end": 133
},
{
"file": "SKILL.md",
"line_start": 133,
"line_end": 141
},
{
"file": "SKILL.md",
"line_start": 141,
"line_end": 150
},
{
"file": "SKILL.md",
"line_start": 150,
"line_end": 160
},
{
"file": "SKILL.md",
"line_start": 160,
"line_end": 161
},
{
"file": "SKILL.md",
"line_start": 161,
"line_end": 163
},
{
"file": "SKILL.md",
"line_start": 163,
"line_end": 173
},
{
"file": "SKILL.md",
"line_start": 173,
"line_end": 174
},
{
"file": "SKILL.md",
"line_start": 174,
"line_end": 175
},
{
"file": "SKILL.md",
"line_start": 175,
"line_end": 176
},
{
"file": "SKILL.md",
"line_start": 101,
"line_end": 101
},
{
"file": "SKILL.md",
"line_start": 98,
"line_end": 103
}
]
},
{
"factor": "network",
"evidence": [
{
"file": "SKILL.md",
"line_start": 12,
"line_end": 12
}
]
},
{
"factor": "filesystem",
"evidence": [
{
"file": "SKILL.md",
"line_start": 94,
"line_end": 94
}
]
}
],
"static_findings": [
{
"id": "external_commands:SKILL.md:14:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 14,
"line_end": 14,
"snippet": "**Prerequisite:** `npm install -g promptfoo` (check with `which promptfoo`)"
},
{
"id": "external_commands:SKILL.md:20:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 20,
"line_end": 20,
"snippet": "### `/regression-test init`"
},
{
"id": "external_commands:SKILL.md:25:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 25,
"line_end": 27,
"snippet": "```bash"
},
{
"id": "external_commands:SKILL.md:27:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 27,
"line_end": 29,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:29:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 29,
"line_end": 29,
"snippet": "2. Generate `.promptfoo/promptfooconfig.yaml` (the path matters: `tests: \"golden/*.yaml\"` below reso"
},
{
"id": "external_commands:SKILL.md:30:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 30,
"line_end": 33,
"snippet": "```yaml"
},
{
"id": "external_commands:SKILL.md:33:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 33,
"line_end": 46,
"snippet": "# The vars come from the golden files that `/regression-test add` writes."
},
{
"id": "external_commands:SKILL.md:46:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 46,
"line_end": 48,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:48:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 48,
"line_end": 48,
"snippet": "3. Add `.promptfoo/results/` to `.gitignore` if not already present. Ignore the results, commit the "
},
{
"id": "external_commands:SKILL.md:50:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 50,
"line_end": 50,
"snippet": "4. Report: \"Initialized `.promptfoo/golden/` — run `/regression-test add <name>` to capture test cas"
},
{
"id": "external_commands:SKILL.md:54:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 54,
"line_end": 58,
"snippet": "### `/regression-test add <name>`"
},
{
"id": "external_commands:SKILL.md:58:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 58,
"line_end": 58,
"snippet": "1. Parse `<name>` from user input (slug format, e.g., `auth-flow`, `error-handling`)"
},
{
"id": "external_commands:SKILL.md:67:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 67,
"line_end": 81,
"snippet": "```yaml"
},
{
"id": "external_commands:SKILL.md:81:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 81,
"line_end": 83,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:83:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 83,
"line_end": 87,
"snippet": "4. Report: \"Added golden test `{name}` — {N} total test cases.\""
},
{
"id": "external_commands:SKILL.md:87:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 87,
"line_end": 92,
"snippet": "### `/regression-test run`"
},
{
"id": "external_commands:SKILL.md:92:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 92,
"line_end": 95,
"snippet": "```bash"
},
{
"id": "external_commands:SKILL.md:95:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 95,
"line_end": 98,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:98:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 98,
"line_end": 103,
"snippet": "```bash"
},
{
"id": "external_commands:SKILL.md:103:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 103,
"line_end": 110,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:110:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 110,
"line_end": 119,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:119:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 119,
"line_end": 126,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:126:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 126,
"line_end": 131,
"snippet": "### `/regression-test report`"
},
{
"id": "external_commands:SKILL.md:131:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 131,
"line_end": 133,
"snippet": "```bash"
},
{
"id": "external_commands:SKILL.md:133:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 133,
"line_end": 141,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:141:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 141,
"line_end": 150,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:150:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 150,
"line_end": 160,
"snippet": "```"
},
{
"id": "external_commands:SKILL.md:160:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 160,
"line_end": 161,
"snippet": "| After changing any prompt | `/regression-test run` before you commit. This is the one that catches"
},
{
"id": "external_commands:SKILL.md:161:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 161,
"line_end": 163,
"snippet": "| Before a release | Run it in CI and fail the build on a new failure. `promptfoo eval` returns non-"
},
{
"id": "external_commands:SKILL.md:163:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 163,
"line_end": 173,
"snippet": "| Weekly, on a schedule | `/regression-test report` to see whether quality is drifting rather than b"
},
{
"id": "external_commands:SKILL.md:173:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 173,
"line_end": 174,
"snippet": "| promptfoo CLI | Not installed | Prompt user: `npm install -g promptfoo` |"
},
{
"id": "external_commands:SKILL.md:174:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 174,
"line_end": 175,
"snippet": "| .promptfoo/ directory | Not initialized | Prompt: `/regression-test init` |"
},
{
"id": "external_commands:SKILL.md:175:ruby-shell-backtick-execution",
"category": "external_commands",
"severity": "medium",
"pattern": "Ruby/shell backtick execution",
"file": "SKILL.md",
"line_start": 175,
"line_end": 176,
"snippet": "| Golden datasets | No .yaml files | Skip with note: \"No golden tests — add with `/regression-test a"
},
{
"id": "external_commands:SKILL.md:101:shell-command-substitution",
"category": "external_commands",
"severity": "medium",
"pattern": "Shell command substitution",
"file": "SKILL.md",
"line_start": 101,
"line_end": 101,
"snippet": "--output .promptfoo/results/eval-$(date +%Y%m%d-%H%M%S).json \\"
},
{
"id": "external_commands:SKILL.md:98:template-literal-with-command-substitution",
"category": "external_commands",
"severity": "medium",
"pattern": "Template literal with command substitution",
"file": "SKILL.md",
"line_start": 98,
"line_end": 103,
"snippet": "```bash"
},
{
"id": "network:SKILL.md:12:hardcoded-url",
"category": "network",
"severity": "low",
"pattern": "Hardcoded URL",
"file": "SKILL.md",
"line_start": 12,
"line_end": 12,
"snippet": "https://github.com/7alexhale5-rgb/alexhale-skills/tree/main/example"
},
{
"id": "filesystem:SKILL.md:94:standard-device-file-access",
"category": "filesystem",
"severity": "low",
"pattern": "Standard device file access",
"file": "SKILL.md",
"line_start": 94,
"line_end": 94,
"snippet": "ls .promptfoo/golden/*.yaml 2>/dev/null | wc -l # must be > 0"
},
{
"id": "blocker:SKILL.md:36:template-injection-targeting-system-prompt",
"category": "blocker",
"severity": "medium",
"pattern": "Template injection targeting system prompt",
"file": "SKILL.md",
"line_start": 36,
"line_end": 36,
"snippet": "[{\"role\":\"system\",\"content\":\"{{system_prompt}}\"},{\"role\":\"user\",\"content\":\"{{user_input}}\"}]"
}
],
"finding_verdicts": [
{
"id": "external_commands:SKILL.md:14:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:20:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:25:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:27:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:29:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:30:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:33:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:46:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:48:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:50:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:54:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:58:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:67:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:81:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:83:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:87:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:92:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:95:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:98:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:103:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:110:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:119:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:126:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:131:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:133:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:141:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:150:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:160:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:161:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:163:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:173:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:174:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:175:ruby-shell-backtick-execution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The detector matched Markdown backticks used for command names, fenced examples, placeholders, or prose, not Ruby or shell backtick execution. The documented operations are bounded to Promptfoo setup and reporting."
},
{
"id": "external_commands:SKILL.md:101:shell-command-substitution",
"verdict": "false_positive",
"confidence": 0.98,
"reason": "The substitution invokes date with a fixed format only to create a timestamped local results filename. It contains no user-controlled command or argument."
},
{
"id": "external_commands:SKILL.md:98:template-literal-with-command-substitution",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "Lines 98-103 are a fenced Bash example, not a JavaScript template literal. Its only command substitution is a fixed date invocation for the output filename."
},
{
"id": "network:SKILL.md:12:hardcoded-url",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The URL is a visible GitHub documentation link to a walkthrough. The skill does not instruct the agent to fetch it or transmit data."
},
{
"id": "filesystem:SKILL.md:94:standard-device-file-access",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The command redirects expected ls errors to /dev/null while counting local YAML files. It neither reads sensitive device data nor writes to a device."
},
{
"id": "blocker:SKILL.md:36:template-injection-targeting-system-prompt",
"verdict": "false_positive",
"confidence": 0.99,
"reason": "The cited content is a Promptfoo chat-message template that places declared test variables into system and user roles. It does not address the auditing agent or attempt to override instructions."
}
],
"semantic_findings": [],
"critical_findings": [],
"high_findings": [],
"medium_findings": [],
"low_findings": [],
"dangerous_patterns": [],
"files_scanned": 1,
"total_lines": 177,
"audit_model": "codex",
"audited_at": "2026-08-21T01:02:35.961Z",
"agent_auto_install_policy": "allowed",
"manual_install_policy": "allowed"
},
"content": {
"user_title": "Detect LLM Prompt Regressions with Promptfoo",
"value_statement": "Prompt changes can silently reduce output quality. This skill builds golden datasets, runs Promptfoo evaluations, and summarizes failures and recent trends.",
"seo_keywords": [
"Claude",
"Codex",
"Claude Code",
"Promptfoo",
"LLM regression testing",
"golden dataset",
"prompt evaluation",
"AI quality testing",
"prompt testing"
],
"actual_capabilities": [
"Initializes a Promptfoo directory, golden dataset folder, configuration file, and results ignore rule.",
"Captures named test cases with system prompts, user inputs, expected behavior, and optional variables.",
"Runs Promptfoo evaluations without cache and stores timestamped local result files.",
"Reports total, passed, and failed cases with assertion details and duration.",
"Summarizes pass rates and new failures across the five most recent evaluations."
],
"limitations": [
"Requires the Promptfoo CLI, provider access, and any credentials required by the configured model.",
"The generated configuration targets one Anthropic model and uses general rubric assertions.",
"Trend reports cover only the five most recent local evaluation files.",
"Shell examples assume a Unix environment with common command-line utilities."
],
"use_cases": [
{
"title": "Validate prompt changes",
"description": "Compare updated prompts against recorded expectations before merging code changes.",
"target_user": "Prompt engineers"
},
{
"title": "Check model migrations",
"description": "Rerun a stable golden dataset after changing a model or provider version.",
"target_user": "Machine learning engineers"
},
{
"title": "Enforce release quality",
"description": "Run prompt evaluations in continuous integration and stop releases when assertions fail.",
"target_user": "Release engineers"
}
],
"prompt_templates": [
{
"title": "Initialize a test suite",
"prompt": "Initialize Promptfoo regression testing in this repository. Show the files created and explain any prerequisites that are missing.",
"scenario": "Starting a golden dataset in an existing project"
},
{
"title": "Add a golden case",
"prompt": "Add a golden test named {name}. Ask for the system prompt, user input, expected behavior, and optional variables before writing it.",
"scenario": "Recording one expected prompt behavior"
},
{
"title": "Run regression tests",
"prompt": "Run all golden prompt tests. Summarize the pass rate, duration, and each failed assertion with expected and actual behavior.",
"scenario": "Checking a prompt change before merge"
},
{
"title": "Analyze release trends",
"prompt": "Review the last five evaluations, identify new failures, compare pass-rate trends, and recommend quality gates for {release_workflow}.",
"scenario": "Designing advanced continuous integration checks"
}
],
"output_examples": [
{
"input": "Initialize regression testing for this prompt project.",
"output": "Initialized the Promptfoo configuration and golden dataset directory. The results directory is ignored, and the suite is ready for its first case."
},
{
"input": "Run the golden tests after my system prompt change.",
"output": "Regression results: 12 total, 11 passed, and 1 failed. The auth-flow case failed its expected behavior rubric and needs review."
},
{
"input": "Show recent prompt quality trends.",
"output": "Recent trend: four of five runs passed completely. One auth-flow failure appeared in the latest run, so quality is declining."
}
],
"best_practices": [
"Review every golden case before committing it, and exclude prompts containing secrets or sensitive data.",
"Describe expected behavior precisely enough that rubric assertions produce consistent results.",
"Run the suite after prompt or model changes and before every release."
],
"anti_patterns": [
"Do not treat one current model response as a permanent definition of correct behavior.",
"Do not commit API keys, private prompts, personal data, or confidential outputs in golden files.",
"Do not ignore unstable rubric failures without reviewing the assertion and provider behavior."
],
"faq": [
{
"question": "What must be installed?",
"answer": "Install the Promptfoo CLI and configure access for the model provider used by the evaluation."
},
{
"question": "Which files does the skill create?",
"answer": "It creates Promptfoo configuration, golden YAML cases, local result files, and a results entry in .gitignore."
},
{
"question": "Can these tests run in continuous integration?",
"answer": "Yes. Run the Promptfoo evaluation in continuous integration and fail the job when an assertion fails."
},
{
"question": "Should sensitive prompts be stored as golden cases?",
"answer": "No. Keep sensitive cases outside the repository and avoid storing credentials or confidential content."
},
{
"question": "Can I use a different model?",
"answer": "Yes. Edit the generated Promptfoo provider configuration before running the suite."
},
{
"question": "How much evaluation history is reported?",
"answer": "The report summarizes the five most recent timestamped result files available in the local results directory."
}
]
},
"file_structure": [
{
"name": "SKILL.md",
"type": "file",
"path": "SKILL.md",
"lines": 177
}
]
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |