borghei/Claude-Skills / engineering/chaos-engineering
chaos-engineering
VerifiedDesigns hypothesis-driven fault-injection experiments, sizes their blast radius and plans gamedays. Use when designing a chaos experiment, planning a gameday, choosing what to inject, or building a chaos maturity model.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 10.8 KB |
| references/chaos-principles-and-maturity.md | markdown | 16.5 KB |
| references/experiment-design-loop.md | markdown | 8.8 KB |
| references/fault-injection-catalog.md | markdown | 23.4 KB |
| references/gameday-playbook.md | markdown | 16.9 KB |
| references/gameday-workflows-and-antipatterns.md | markdown | 6.2 KB |
| scripts/blast_radius_calculator.py | script | 10.8 KB |
| scripts/chaos_experiment_designer.py | script | 20.4 KB |
| scripts/gameday_planner.py | script | 16.4 KB |
scripts/chaos_experiment_designer.py
View on GitHub#!/usr/bin/env python3
"""
chaos_experiment_designer.py — Scaffold a chaos experiment document.
Given a target service, a fault type, and a few parameters, emits a complete
experiment template covering steady state, hypothesis, blast radius, abort
criteria, observation checklist, and the run/analyze/document loop.
Designed to make "let's do a chaos experiment" turn into a real plan in 60 seconds.
Stdlib only. Markdown or JSON output.
Usage:
python3 chaos_experiment_designer.py --target payment-svc --fault dependency-timeout
python3 chaos_experiment_designer.py --target search --fault pod-kill --duration 5m --maturity-level L2
python3 chaos_experiment_designer.py --target api --fault network-latency --inject-value 200ms --format json
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass, field, asdict
from datetime import datetime, timezone
from pathlib import Path
# Fault catalog with default parameters and standard observation lists
FAULT_CATALOG: dict[str, dict[str, object]] = {
"pod-kill": {
"description": "Kill one or more instances/pods of the target service",
"real_world_analog": "Spot interruption, hypervisor reboot, OOM-kill",
"default_inject_value": "1 pod",
"hypothesis_template": "Killing {inject_value} of {target}'s pods causes < 30 seconds of degraded latency (within 2× baseline), error rate stays under 0.5%, replacement instance is healthy within 60 seconds.",
"watch": ["per-pod error rate", "total error rate", "replica count", "in-flight request count", "autoscaler decisions"],
"tools": ["Chaos Mesh PodChaos pod-kill", "Litmus pod-delete", "kubectl delete pod"],
"abort_thresholds": {
"total_error_rate_pct": 5.0,
"p99_latency_multiplier": 3.0,
"duration_seconds": 300,
},
"blast_radius_default": "1 of N pods",
"common_bugs": [
"in-flight requests aborted (graceful drain not honored)",
"long startup time for replacement",
"connection pool stuck on dead pod",
"client retries cascade to other pods",
],
},
"network-latency": {
"description": "Inject network latency on the target's outbound or inbound traffic",
"real_world_analog": "Cross-region routing, congested link, vendor latency spike",
"default_inject_value": "200ms",
"hypothesis_template": "Adding {inject_value} of latency to {target}'s downstream calls increases end-to-end p99 by ≤ {inject_value} + 50ms. No timeouts fire. Retry rate increases by < 10%.",
"watch": ["latency p50/p95/p99", "timeout count", "retry count", "circuit breaker state", "queue depth"],
"tools": ["Chaos Mesh NetworkChaos delay", "Litmus pod-network-latency", "tc qdisc netem delay"],
"abort_thresholds": {
"timeout_rate_pct": 1.0,
"circuit_breaker_state": "any open",
"duration_seconds": 600,
},
"blast_radius_default": "All traffic from/to target during experiment",
"common_bugs": [
"hard-coded timeouts shorter than expected latency",
"retry storms when timeout fires",
"client-side queue overflow",
"metric pollution from per-call timers",
],
},
"network-partition": {
"description": "Block all traffic between target and a specified peer/network",
"real_world_analog": "AZ network failure, security group misconfiguration, BGP issue",
"default_inject_value": "full partition for 60s",
"hypothesis_template": "When traffic between {target} and dependency is blocked, {target} serves cached data for ≤ cache TTL, then returns clear errors with retry-after. No hung connections. Health check reflects degraded state within 30s.",
"watch": ["connection establishment rate", "connection timeout rate", "open file descriptors", "cache hit rate", "downstream call rate"],
"tools": ["Chaos Mesh NetworkChaos partition", "Litmus pod-network-partition", "iptables / security groups"],
"abort_thresholds": {
"open_fd_growth_rate": "no monotonic growth",
"unrelated_feature_error_rate_pct": 0.5,
"duration_seconds": 300,
},
"blast_radius_default": "All traffic to specified peer",
"common_bugs": [
"sockets leaked on partition (eventual OOM)",
"cache returns stale data without indicating staleness",
"health check still reports healthy",
"cascade to upstream when downstream is partitioned",
],
},
"dependency-timeout": {
"description": "Mock the target's downstream to return slowly (above timeout)",
"real_world_analog": "Downstream overloaded, GC pause, cross-region call",
"default_inject_value": "2000ms response time",
"hypothesis_template": "When {target}'s downstream returns in {inject_value}, circuit breaker trips after N requests, fallback engages, no requests hang.",
"watch": ["downstream p95/p99 latency", "downstream timeout count", "circuit breaker state", "fallback path metrics", "thread pool utilization"],
"tools": ["Toxiproxy latency toxic", "Mountebank with delayed responses", "Chaos Mesh NetworkChaos delay between specific services"],
"abort_thresholds": {
"thread_pool_saturation_pct": 90,
"fallback_unavailable": "abort if fallback doesn't engage",
"duration_seconds": 300,
},
"blast_radius_default": "All calls to mocked downstream from target",
"common_bugs": [
"no caller-side timeout",
"timeout too long",
"in-flight call count grows unboundedly",
"thread pool saturates and degrades other work",
],
},
"dependency-error": {
"description": "Mock the target's downstream to return errors (5xx)",
"real_world_analog": "Downstream outage, deploy hiccup, vendor incident",
"default_inject_value": "100% 503 responses for 5min",
"hypothesis_template": "When {target}'s downstream returns errors at {inject_value}, fallback path engages, error rate on parent feature stays < 0.5%, alerting fires within 2 minutes.",
"watch": ["per-dependency error rate", "fallback path utilization", "retry count", "circuit breaker state", "end-user-visible metrics on dependent features"],
"tools": ["Toxiproxy", "Mountebank", "custom controllable proxy"],
"abort_thresholds": {
"user_visible_error_rate_pct": 1.0,
"retry_storm_threshold": "retry rate > 100× normal",
"duration_seconds": 600,
},
"blast_radius_default": "Only calls to mocked downstream from target",
"common_bugs": [
"no fallback at all",
"fallback itself broken",
"retries amplify load on degraded downstream",
"circuit breaker never opens or never closes",
],
},
"cpu-saturation": {
"description": "Saturate CPU on the target instance(s)",
"real_world_analog": "Runaway worker, GC death-spiral, noisy neighbor",
"default_inject_value": "100% on all cores",
"hypothesis_template": "Saturating {target}'s CPU at {inject_value} causes latency increase but no errors. Other co-located processes are throttled (cgroup limits) but functional.",
"watch": ["CPU per pod", "CPU per host", "latency", "error rate", "throttle metrics", "GC time"],
"tools": ["Chaos Mesh StressChaos", "Litmus pod-cpu-hog", "stress-ng"],
"abort_thresholds": {
"error_rate_pct": 5.0,
"GC_pause_ms": 5000,
"duration_seconds": 300,
},
"blast_radius_default": "1 instance",
"common_bugs": [
"no CPU limits (one runaway eats host)",
"thread starvation in fixed pools",
"long GC pauses misattributed",
],
},
"memory-pressure": {
"description": "Consume memory on the target instance(s)",
"real_world_analog": "Memory leak, large request, batch job, cache growth",
"default_inject_value": "90% memory used",
"hypothesis_template": "Under {inject_value} memory pressure on {target}, kernel pages out cleanly; app continues with slight latency increase; OOM-killer doesn't fire unless cgroup limit exceeded.",
"watch": ["memory per pod", "swap usage", "OOM events", "latency", "error rate"],
"tools": ["Chaos Mesh StressChaos", "Litmus pod-memory-hog", "stress-ng"],
"abort_thresholds": {
"OOM_kill_count": 0,
"error_rate_pct": 2.0,
"duration_seconds": 300,
},
"blast_radius_default": "1 instance",
"common_bugs": [
"process OOM-killed instead of throttled",
"cache doesn't evict under pressure",
"large allocations fail mid-request",
],
},
"disk-fill": {
"description": "Fill the target's disk to high utilization",
"real_world_analog": "Log rotation failure, runaway cache, large uploads",
"default_inject_value": "95% disk used",
"hypothesis_template": "At {inject_value} disk on {target}: alert fires within 1 min; app rejects writes with 4xx; no data corruption; recovery is automatic when space is freed.",
"watch": ["disk usage percent", "write success rate", "log write rate", "application error rate"],
"tools": ["dd if=/dev/zero", "Chaos Mesh IOChaos"],
"abort_thresholds": {
"process_crash": "abort on any crash",
"data_corruption": "abort on any detected corruption",
"duration_seconds": 600,
},
"blast_radius_default": "1 instance disk volume",
"common_bugs": [
"app crashes on write failure",
"logger silently drops when /var/log full",
"alert fires only at 100%",
],
},
"traffic-spike": {
"description": "Generate synthetic traffic at multiplied rate",
"real_world_analog": "Viral moment, bot attack, product launch",
"default_inject_value": "5x normal RPS for 5min",
"hypothesis_template": "At {inject_value} for 5min against {target}: autoscaler engages within 90s, latency stays under SLO, error rate stays < 1%, cost returns to baseline within 10min after spike ends.",
"watch": ["RPS", "autoscaler decisions", "latency", "error rate", "queue depth at every layer", "cost per request"],
"tools": ["k6", "Locust", "Vegeta", "JMeter"],
"abort_thresholds": {
"real_user_error_rate_pct": 1.0,
"cost_multiplier": 5.0,
"duration_seconds": 600,
},
"blast_radius_default": "Synthetic traffic only (do not replay real user data)",
"common_bugs": [
"autoscaler too slow",
"cold cache amplifies origin load",
"rate-limiter fires on legitimate spike",
"downstream services overload before primary autoscales",
],
},
}
def parse_duration(s: str) -> int:
"""Parse a duration like '5m', '300s', '1h' into seconds."""
if not s:
return 300
s = s.strip().lower()
if s.endswith("ms"):
return max(1, int(float(s[:-2]) / 1000))
if s.endswith("s"):
return int(float(s[:-1]))
if s.endswith("m"):
return int(float(s[:-1]) * 60)
if s.endswith("h"):
return int(float(s[:-1]) * 3600)
return int(float(s))
@dataclass
class Experiment:
name: str
target: str
fault_type: str
fault_description: str
real_world_analog: str
inject_value: str
duration_seconds: int
maturity_level: str
hypothesis: str
steady_state_template: list[str]
watch_metrics: list[str]
tools: list[str]
abort_thresholds: dict[str, object]
blast_radius: str
common_bugs: list[str]
pre_run_checklist: list[str] = field(default_factory=list)
post_run_actions: list[str] = field(default_factory=list)
metadata: dict[str, str] = field(default_factory=dict)
def build_experiment(args: argparse.Namespace) -> Experiment:
fault = args.fault.lower()
if fault not in FAULT_CATALOG:
raise ValueError(f"Unknown fault type: {fault}. Available: {', '.join(FAULT_CATALOG)}")
cat = FAULT_CATALOG[fault]
inject_value = args.inject_value or str(cat["default_inject_value"])
duration = parse_duration(args.duration)
hyp = str(cat["hypothesis_template"]).format(target=args.target, inject_value=inject_value)
name = f"chaos-{args.target}-{fault}-{datetime.now(timezone.utc).strftime('%Y%m%d')}"
return Experiment(
name=name,
target=args.target,
fault_type=fault,
fault_description=str(cat["description"]),
real_world_analog=str(cat["real_world_analog"]),
inject_value=inject_value,
duration_seconds=duration,
maturity_level=args.maturity_level,
hypothesis=hyp,
steady_state_template=[
"Quantify the current baseline before injection. Capture as a screenshot or metric snapshot.",
"Error rate (parent feature): <value> (e.g., 0.05%)",
"p95 latency: <value> (e.g., 180ms)",
"p99 latency: <value> (e.g., 380ms)",
"Request rate (relevant endpoint): <value> rps",
"Business KPI (if any): <value>",
"Capacity headroom: <value> (e.g., CPU 35% avg)",
],
watch_metrics=list(cat["watch"]),
tools=list(cat["tools"]),
abort_thresholds=dict(cat["abort_thresholds"]),
blast_radius=str(cat["blast_radius_default"]),
common_bugs=list(cat["common_bugs"]),
pre_run_checklist=[
"Maturity prerequisite met (L1: staging only; L2: tooling automated; L3: prod approved)",
"Observer assigned to watch dashboards",
"Abort mechanism tested in last 7 days",
"Customer support / oncall informed (for prod runs)",
"Steady-state metrics captured (screenshot)",
"Slack thread open for the experiment",
"Hypothesis written before injection",
"Duration cap agreed",
],
post_run_actions=[
"Within 1 hour: scribe writes raw observations",
"Within 1 day: hypothesis status (confirmed / partial / disproven) recorded",
"Within 3 days: action items filed with owners and due dates",
"Within 1 week: experiment artifact added to team's chaos catalog",
"Schedule re-run after fixes",
],
metadata={
"generated_at": datetime.now(timezone.utc).isoformat(),
"designer_version": "1.0",
},
)
def render_markdown(exp: Experiment) -> str:
out: list[str] = []
out.append(f"# Chaos experiment: `{exp.name}`")
out.append("")
out.append(f"_Generated: {exp.metadata['generated_at']}_")
out.append("")
out.append("## Overview")
out.append("")
out.append(f"- **Target service**: `{exp.target}`")
out.append(f"- **Fault type**: `{exp.fault_type}`")
out.append(f"- **Inject value**: {exp.inject_value}")
out.append(f"- **Duration**: {exp.duration_seconds} seconds ({exp.duration_seconds // 60}m {exp.duration_seconds % 60}s)")
out.append(f"- **Maturity level**: {exp.maturity_level}")
out.append(f"- **Description**: {exp.fault_description}")
out.append(f"- **Real-world analog**: {exp.real_world_analog}")
out.append("")
out.append("## Hypothesis (falsifiable)")
out.append("")
out.append(f"> {exp.hypothesis}")
out.append("")
out.append("Edit this to match your specific SLOs and thresholds before running.")
out.append("")
out.append("## Steady state (capture before injection)")
out.append("")
for line in exp.steady_state_template:
out.append(f"- {line}")
out.append("")
out.append("## Blast radius")
out.append("")
out.append(f"- **Default scope**: {exp.blast_radius}")
out.append("- **Caps by maturity**:")
out.append(" - L1 (staging): full target service in staging")
out.append(" - L2 (staging-auto): full target service in staging, scheduled")
out.append(" - L3 (prod-canary): 1-5% of users / one canary cell / time-boxed")
out.append(" - L4 (continuous): bounded by error-budget consumption")
out.append("")
out.append("Run `scripts/blast_radius_calculator.py` for quantitative estimates.")
out.append("")
out.append("## Watch metrics")
out.append("")
for m in exp.watch_metrics:
out.append(f"- {m}")
out.append("")
out.append("## Tools (pick one)")
out.append("")
for t in exp.tools:
out.append(f"- {t}")
out.append("")
out.append("## Abort criteria")
out.append("")
for k, v in exp.abort_thresholds.items():
out.append(f"- **{k}**: {v}")
out.append("- Operator or observer can call abort at any time")
out.append("")
out.append("## Pre-run checklist")
out.append("")
for c in exp.pre_run_checklist:
out.append(f"- [ ] {c}")
out.append("")
out.append("## Post-run actions")
out.append("")
for a in exp.post_run_actions:
out.append(f"- {a}")
out.append("")
out.append("## Common bugs surfaced by this fault")
out.append("")
for b in exp.common_bugs:
out.append(f"- {b}")
out.append("")
out.append("---")
out.append("")
out.append("## Run log (fill in during/after the experiment)")
out.append("")
out.append("### Pre-injection baseline (captured at <time>)")
out.append("- error rate: <>")
out.append("- p95 latency: <>")
out.append("- p99 latency: <>")
out.append("- request rate: <>")
out.append("")
out.append("### Injection")
out.append("- Tool used: <>")
out.append("- Command: `<>`")
out.append("- Started: <time>")
out.append("- Ended: <time>")
out.append("")
out.append("### Observations (chronological)")
out.append("- T+0:00 — <event>")
out.append("- T+0:30 — <event>")
out.append("- T+1:00 — <event>")
out.append("")
out.append("### Result")
out.append("- [ ] Hypothesis confirmed")
out.append("- [ ] Hypothesis partially confirmed")
out.append("- [ ] Hypothesis disproven")
out.append("")
out.append("### Surprises")
out.append("- ")
out.append("")
out.append("### Action items")
out.append("| ID | Action | Owner | Due | Priority | Ticket |")
out.append("|----|--------|-------|-----|----------|--------|")
out.append("| AI-1 | | | | P? | |")
return "\n".join(out)
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(
description="Scaffold a chaos experiment document",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
p.add_argument("--target", help="Target service name (e.g., payment-svc)")
p.add_argument(
"--fault",
choices=sorted(FAULT_CATALOG.keys()),
help="Fault type to inject. Run --list-faults for catalog.",
)
p.add_argument("--inject-value", help="Magnitude (e.g., '200ms' for latency, '1 pod' for pod-kill)")
p.add_argument("--duration", default="5m", help="Duration (e.g., 5m, 300s, 1h) (default: 5m)")
p.add_argument(
"--maturity-level",
choices=["L1", "L2", "L3", "L4"],
default="L2",
help="Target maturity level — informs blast radius caps (default: L2)",
)
p.add_argument("--format", choices=["markdown", "json"], default="markdown")
p.add_argument("--output", help="Output file path")
p.add_argument("--list-faults", action="store_true", help="List all fault types in catalog")
args = p.parse_args()
if not args.list_faults and not (args.target and args.fault):
p.error("--target and --fault are required (or use --list-faults)")
return args
def main() -> int:
args = parse_args()
if args.list_faults:
print("Available fault types:\n")
for k, v in FAULT_CATALOG.items():
print(f" {k:<22} {v['description']}")
return 0
try:
exp = build_experiment(args)
except ValueError as e:
print(f"error: {e}", file=sys.stderr)
return 2
if args.format == "json":
out = json.dumps(asdict(exp), indent=2, default=str)
else:
out = render_markdown(exp)
if args.output:
Path(args.output).write_text(out)
print(f"wrote {args.output}", file=sys.stderr)
else:
print(out)
return 0
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| brainstormingobra/superpowersYou MUST use this before any creative work - creating features, building components, adding functionality, modifying behavior, or planning anything new, in software or out of it (a talk, a business, a renovation). | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| brand-guidelinesanthropics/skillsApplies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| frontend-designanthropics/skillsGuidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| mcp-builderanthropics/skillsGuide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK). | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |