Skip to content

HabitaNexus/monorepo / skills/system-reliability-engineering/chaos-engineering

chaos-engineering

Verified

FileKindSize
SKILL.mdskill md8.0 KB
scripts/README.mdmarkdown1.7 KB
scripts/chaos_monkey.pyscript8.6 KB
scripts/experiments.pyscript7.8 KB
scripts/requirements.txtmanifest90 B

scripts/chaos_monkey.py

View on GitHub
#!/usr/bin/env python3
"""
Chaos Monkey

Randomly terminate pods and resources to test system resilience.

Usage:
    # Run chaos monkey
    python chaos_monkey.py run --experiment-type pod-delete

    # Enable/disable
    python chaos_monkey.py enable
    python chaos_monkey.py disable

    # Set probability
    python chaos_monkey.py set-probability --probability 0.1
"""

import argparse
import random
import sys
from typing import Dict, List, Optional

try:
    import kubernetes
    from kubernetes.client.rest import ApiException
except ImportError:
    print("❌ Error: kubernetes not installed. Install with: pip install kubernetes")
    sys.exit(1)


class ChaosMonkey:
    """Chaos Monkey for Kubernetes - Randomly terminate resources."""

    def __init__(self, enabled: bool = True, probability: float = 0.1, kubeconfig: Optional[str] = None):
        """
        Initialize Chaos Monkey.

        Args:
            enabled: Whether chaos monkey is enabled
            probability: Probability of chaos (0.0 to 1.0)
            kubeconfig: Path to kubeconfig file (optional)
        """
        self.enabled = enabled
        self.probability = probability

        if kubeconfig:
            kubernetes.config.load_kube_config(config_file=kubeconfig)
        else:
            try:
                kubernetes.config.load_incluster_config()
            except:
                kubernetes.config.load_kube_config()

        self.core_v1 = kubernetes.client.CoreV1Api()
        self.apps_v1 = kubernetes.client.AppsV1Api()

    def run_experiment(self, experiment_type: str) -> Dict:
        """
        Run a chaos experiment.

        Args:
            experiment_type: Type of experiment (pod-delete, node-drain, cpu-stress, etc.)

        Returns:
            Dictionary with experiment results
        """
        if not self.enabled:
            return {
                'status': 'skipped',
                'reason': 'Chaos Monkey is disabled',
            }

        if random.random() > self.probability:
            return {
                'status': 'skipped',
                'reason': f'Random skip (probability: {self.probability})',
            }

        try:
            if experiment_type == 'pod-delete':
                return self._delete_random_pod()
            elif experiment_type == 'node-drain':
                return self._drain_random_node()
            elif experiment_type == 'cpu-stress':
                return self._stress_cpu()
            elif experiment_type == 'memory-stress':
                return self._stress_memory()
            elif experiment_type == 'network-partition':
                return self._partition_network()
            else:
                return {
                    'status': 'error',
                    'error': f'Unknown experiment type: {experiment_type}',
                }
        except Exception as e:
            return {
                'status': 'error',
                'error': str(e),
            }

    def _delete_random_pod(self) -> Dict:
        """Delete a random pod from a deployment."""
        try:
            # Get all deployments
            deployments = self.apps_v1.list_deployment_for_all_namespaces()

            # Filter to deployments with chaos annotation
            chaos_deployments = [
                d for d in deployments.items
                if d.metadata.annotations and
                d.metadata.annotations.get('chaos.enabled') == 'true'
            ]

            if not chaos_deployments:
                return {
                    'status': 'skipped',
                    'reason': 'No deployments with chaos.enabled=true annotation found',
                }

            # Pick random deployment
            deployment = random.choice(chaos_deployments)
            namespace = deployment.metadata.namespace
            app_label = deployment.metadata.labels.get('app') or deployment.metadata.name

            # Get pods for this deployment
            pods = self.core_v1.list_namespaced_pod(
                namespace,
                label_selector=f"app={app_label}"
            )

            if not pods.items:
                return {
                    'status': 'skipped',
                    'reason': f'No pods found for deployment {deployment.metadata.name}',
                }

            # Pick random pod
            pod = random.choice(pods.items)
            pod_name = pod.metadata.name

            # Delete pod
            self.core_v1.delete_namespaced_pod(
                pod_name,
                namespace,
                grace_period_seconds=0
            )

            print(f"💥 Chaos Monkey: Deleted pod {pod_name} in namespace {namespace}")

            return {
                'status': 'success',
                'experiment': 'pod-delete',
                'pod': pod_name,
                'namespace': namespace,
                'deployment': deployment.metadata.name,
            }

        except ApiException as e:
            return {
                'status': 'error',
                'error': f'Kubernetes API error: {e}',
            }

    def _drain_random_node(self) -> Dict:
        """Drain a random node (placeholder - requires node access)."""
        return {
            'status': 'not_implemented',
            'message': 'Node draining requires additional permissions and implementation',
        }

    def _stress_cpu(self) -> Dict:
        """Stress CPU on random pod (placeholder)."""
        return {
            'status': 'not_implemented',
            'message': 'CPU stress requires additional tools (stress-ng)',
        }

    def _stress_memory(self) -> Dict:
        """Stress memory on random pod (placeholder)."""
        return {
            'status': 'not_implemented',
            'message': 'Memory stress requires additional tools',
        }

    def _partition_network(self) -> Dict:
        """Create network partition (placeholder)."""
        return {
            'status': 'not_implemented',
            'message': 'Network partition requires network policy manipulation',
        }


def main():
    """CLI entry point."""
    parser = argparse.ArgumentParser(
        description="Chaos Monkey - Randomly terminate resources to test resilience",
        formatter_class=argparse.RawDescriptionHelpFormatter
    )

    parser.add_argument(
        "--kubeconfig",
        help="Path to kubeconfig file"
    )

    parser.add_argument(
        "--probability",
        type=float,
        default=0.1,
        help="Probability of chaos (0.0 to 1.0, default: 0.1)"
    )

    subparsers = parser.add_subparsers(dest="command", help="Command to execute")

    # Run command
    run_parser = subparsers.add_parser("run", help="Run chaos experiment")
    run_parser.add_argument(
        "--experiment-type",
        choices=['pod-delete', 'node-drain', 'cpu-stress', 'memory-stress', 'network-partition'],
        default='pod-delete',
        help="Type of experiment to run"
    )

    # Enable/disable
    subparsers.add_parser("enable", help="Enable Chaos Monkey")
    subparsers.add_parser("disable", help="Disable Chaos Monkey")

    # Set probability
    prob_parser = subparsers.add_parser("set-probability", help="Set chaos probability")
    prob_parser.add_argument("--probability", type=float, required=True, help="Probability (0.0 to 1.0)")

    args = parser.parse_args()

    if not args.command:
        parser.print_help()
        return 1

    try:
        enabled = args.command != 'disable'
        monkey = ChaosMonkey(
            enabled=enabled,
            probability=args.probability,
            kubeconfig=args.kubeconfig
        )

        if args.command == "run":
            result = monkey.run_experiment(args.experiment_type)

            if result['status'] == 'success':
                print(f"✅ Experiment completed: {result.get('experiment')}")
                print(f"   Pod: {result.get('pod')}")
                print(f"   Namespace: {result.get('namespace')}")
            elif result['status'] == 'skipped':
                print(f"⏭️  Experiment skipped: {result.get('reason')}")
            else:
                print(f"❌ Experiment failed: {result.get('error', 'Unknown error')}")

        elif args.command in ["enable", "disable"]:
            status = "enabled" if enabled else "disabled"
            print(f"✅ Chaos Monkey {status}")

        elif args.command == "set-probability":
            monkey.probability = args.probability
            print(f"✅ Probability set to {args.probability}")

        return 0

    except Exception as e:
        print(f"❌ Error: {e}", file=sys.stderr)
        import traceback
        traceback.print_exc()
        return 1


if __name__ == "__main__":
    exit(main())
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall