Skip to content

HabitaNexus/monorepo / skills/system-reliability-engineering/disaster-recovery-business-continuity

disaster-recovery-business-continuity

Verified

FileKindSize
SKILL.mdskill md10.0 KB
scripts/README.mdmarkdown2.4 KB
scripts/backup-strategy.shscript7.5 KB
scripts/dr-test.shscript6.8 KB
scripts/failover_procedures.pyscript13.0 KB
scripts/requirements.txtmanifest398 B

scripts/dr-test.sh

View on GitHub
#!/bin/bash
#
# Disaster Recovery Test Script
#
# Comprehensive DR test that simulates a disaster and verifies recovery procedures.
#
# Usage:
#   ./dr-test.sh [service-name]
#
# Environment variables:
#   DR_TEST_SKIP_SIMULATION - Skip disaster simulation (for testing procedures only)
#   DR_TEST_SKIP_FAILBACK   - Skip failback procedure
#

set -euo pipefail

# Configuration
SERVICE_NAME="${1:-payment-service}"
SKIP_SIMULATION="${DR_TEST_SKIP_SIMULATION:-false}"
SKIP_FAILBACK="${DR_TEST_SKIP_FAILBACK:-false}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_section() {
    echo ""
    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
    echo -e "${BLUE}$1${NC}"
    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
}

log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
}

# Pre-test checklist
verify_backups() {
    log_info "Verifying backups..."

    if [ -d "/backup" ]; then
        BACKUP_COUNT=$(find /backup -name "*.gz" -o -name "*.tar.gz" | wc -l)
        if [ "$BACKUP_COUNT" -gt 0 ]; then
            log_info "Found $BACKUP_COUNT backup(s)"
        else
            log_warn "No backups found"
        fi
    else
        log_warn "Backup directory not found"
    fi
}

verify_standby_health() {
    log_info "Verifying standby health..."

    # Check if standby service is running
    if command -v kubectl &> /dev/null; then
        if kubectl get deployment "${SERVICE_NAME}-standby" &> /dev/null; then
            log_info "Standby deployment found"
        else
            log_warn "Standby deployment not found"
        fi
    else
        log_warn "kubectl not available, skipping standby health check"
    fi
}

verify_documentation() {
    log_info "Verifying documentation..."

    if [ -f "failover_procedures.md" ] || [ -f "DR_PLAN.md" ]; then
        log_info "Documentation found"
    else
        log_warn "DR documentation not found"
    fi
}

# Simulate disaster
simulate_disaster() {
    log_info "Simulating disaster (stopping primary service)..."

    if [ "$SKIP_SIMULATION" = "true" ]; then
        log_warn "Skipping disaster simulation (DR_TEST_SKIP_SIMULATION=true)"
        return
    fi

    if command -v kubectl &> /dev/null; then
        log_info "Scaling down primary deployment..."
        # kubectl scale deployment "${SERVICE_NAME}-primary" --replicas=0 || {
        #     log_warn "Failed to scale down primary (may not exist)"
        # }
        log_info "Primary service stopped (simulated)"
    else
        log_warn "kubectl not available, skipping disaster simulation"
    fi
}

# Execute failover
execute_failover() {
    log_info "Executing failover for $SERVICE_NAME..."

    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

    if [ -f "$SCRIPT_DIR/failover_procedures.py" ]; then
        python3 "$SCRIPT_DIR/failover_procedures.py" failover "$SERVICE_NAME" || {
            log_error "Failover failed"
            return 1
        }
    else
        log_error "failover_procedures.py not found"
        return 1
    fi
}

# Check service health
check_service_health() {
    log_info "Checking service health..."

    # Wait for service to stabilize
    sleep 10

    # Check if service is responding
    if command -v curl &> /dev/null; then
        # Replace with actual health check endpoint
        # if curl -f http://service-endpoint/health &> /dev/null; then
        #     log_info "Service health check: OK"
        # else
        #     log_warn "Service health check: FAILED"
        # fi
        log_info "Service health check: OK (simulated)"
    else
        log_warn "curl not available, skipping health check"
    fi
}

# Restore primary
restore_primary() {
    log_info "Restoring primary service..."

    if [ "$SKIP_SIMULATION" = "true" ]; then
        log_warn "Skipping primary restoration (DR_TEST_SKIP_SIMULATION=true)"
        return
    fi

    if command -v kubectl &> /dev/null; then
        log_info "Scaling up primary deployment..."
        # kubectl scale deployment "${SERVICE_NAME}-primary" --replicas=3 || {
        #     log_warn "Failed to scale up primary"
        # }
        log_info "Primary service restored (simulated)"
    else
        log_warn "kubectl not available, skipping primary restoration"
    fi
}

# Execute failback
execute_failback() {
    log_info "Executing failback for $SERVICE_NAME..."

    if [ "$SKIP_FAILBACK" = "true" ]; then
        log_warn "Skipping failback (DR_TEST_SKIP_FAILBACK=true)"
        return
    fi

    # Failback logic (reverse of failover)
    log_info "Failback procedure (implement based on your requirements)"

    # Example:
    # 1. Verify primary is healthy
    # 2. Update routing back to primary
    # 3. Verify traffic is flowing to primary
    # 4. Update standby to sync from primary
}

# Post-test verification
verify_service_health() {
    log_info "Verifying service health after test..."
    check_service_health
}

verify_data_consistency() {
    log_info "Verifying data consistency..."

    # Data consistency checks
    # - Compare primary and standby data
    # - Check replication lag
    # - Verify no data loss

    log_info "Data consistency check: OK (implement based on your requirements)"
}

# Main test execution
main() {
    log_section "🚨 Disaster Recovery Test - $SERVICE_NAME"

    START_TIME=$(date +%s)

    # 1. Pre-test checklist
    log_section "1️⃣  Pre-Test Checklist"
    verify_backups
    verify_standby_health
    verify_documentation

    # 2. Simulate disaster
    log_section "2️⃣  Simulating Disaster"
    simulate_disaster

    # 3. Execute failover
    log_section "3️⃣  Executing Failover"
    if ! execute_failover; then
        log_error "Failover failed, aborting test"
        exit 1
    fi

    # 4. Verify service availability
    log_section "4️⃣  Verifying Service Availability"
    check_service_health

    # 5. Restore primary
    log_section "5️⃣  Restoring Primary"
    restore_primary

    # 6. Failback
    log_section "6️⃣  Executing Failback"
    execute_failback

    # 7. Post-test verification
    log_section "7️⃣  Post-Test Verification"
    verify_service_health
    verify_data_consistency

    END_TIME=$(date +%s)
    DURATION=$((END_TIME - START_TIME))

    log_section "✅ DR Test Completed"
    log_info "Duration: ${DURATION}s"
    log_info "Service: $SERVICE_NAME"
    log_info "Status: PASSED"
}

# Run main
main
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 196kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 196kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 196kUpdated Claude CodeCodexVerifiedInstall