HabitaNexus/monorepo / skills/system-reliability-engineering/disaster-recovery-business-continuity
disaster-recovery-business-continuity
Verified| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 10.0 KB |
| scripts/README.md | markdown | 2.4 KB |
| scripts/backup-strategy.sh | script | 7.5 KB |
| scripts/dr-test.sh | script | 6.8 KB |
| scripts/failover_procedures.py | script | 13.0 KB |
| scripts/requirements.txt | manifest | 398 B |
scripts/dr-test.sh
View on GitHub#!/bin/bash
#
# Disaster Recovery Test Script
#
# Comprehensive DR test that simulates a disaster and verifies recovery procedures.
#
# Usage:
# ./dr-test.sh [service-name]
#
# Environment variables:
# DR_TEST_SKIP_SIMULATION - Skip disaster simulation (for testing procedures only)
# DR_TEST_SKIP_FAILBACK - Skip failback procedure
#
set -euo pipefail
# Configuration
SERVICE_NAME="${1:-payment-service}"
SKIP_SIMULATION="${DR_TEST_SKIP_SIMULATION:-false}"
SKIP_FAILBACK="${DR_TEST_SKIP_FAILBACK:-false}"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
log_section() {
echo ""
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BLUE}$1${NC}"
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
}
log_info() {
echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1" >&2
}
# Pre-test checklist
verify_backups() {
log_info "Verifying backups..."
if [ -d "/backup" ]; then
BACKUP_COUNT=$(find /backup -name "*.gz" -o -name "*.tar.gz" | wc -l)
if [ "$BACKUP_COUNT" -gt 0 ]; then
log_info "Found $BACKUP_COUNT backup(s)"
else
log_warn "No backups found"
fi
else
log_warn "Backup directory not found"
fi
}
verify_standby_health() {
log_info "Verifying standby health..."
# Check if standby service is running
if command -v kubectl &> /dev/null; then
if kubectl get deployment "${SERVICE_NAME}-standby" &> /dev/null; then
log_info "Standby deployment found"
else
log_warn "Standby deployment not found"
fi
else
log_warn "kubectl not available, skipping standby health check"
fi
}
verify_documentation() {
log_info "Verifying documentation..."
if [ -f "failover_procedures.md" ] || [ -f "DR_PLAN.md" ]; then
log_info "Documentation found"
else
log_warn "DR documentation not found"
fi
}
# Simulate disaster
simulate_disaster() {
log_info "Simulating disaster (stopping primary service)..."
if [ "$SKIP_SIMULATION" = "true" ]; then
log_warn "Skipping disaster simulation (DR_TEST_SKIP_SIMULATION=true)"
return
fi
if command -v kubectl &> /dev/null; then
log_info "Scaling down primary deployment..."
# kubectl scale deployment "${SERVICE_NAME}-primary" --replicas=0 || {
# log_warn "Failed to scale down primary (may not exist)"
# }
log_info "Primary service stopped (simulated)"
else
log_warn "kubectl not available, skipping disaster simulation"
fi
}
# Execute failover
execute_failover() {
log_info "Executing failover for $SERVICE_NAME..."
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [ -f "$SCRIPT_DIR/failover_procedures.py" ]; then
python3 "$SCRIPT_DIR/failover_procedures.py" failover "$SERVICE_NAME" || {
log_error "Failover failed"
return 1
}
else
log_error "failover_procedures.py not found"
return 1
fi
}
# Check service health
check_service_health() {
log_info "Checking service health..."
# Wait for service to stabilize
sleep 10
# Check if service is responding
if command -v curl &> /dev/null; then
# Replace with actual health check endpoint
# if curl -f http://service-endpoint/health &> /dev/null; then
# log_info "Service health check: OK"
# else
# log_warn "Service health check: FAILED"
# fi
log_info "Service health check: OK (simulated)"
else
log_warn "curl not available, skipping health check"
fi
}
# Restore primary
restore_primary() {
log_info "Restoring primary service..."
if [ "$SKIP_SIMULATION" = "true" ]; then
log_warn "Skipping primary restoration (DR_TEST_SKIP_SIMULATION=true)"
return
fi
if command -v kubectl &> /dev/null; then
log_info "Scaling up primary deployment..."
# kubectl scale deployment "${SERVICE_NAME}-primary" --replicas=3 || {
# log_warn "Failed to scale up primary"
# }
log_info "Primary service restored (simulated)"
else
log_warn "kubectl not available, skipping primary restoration"
fi
}
# Execute failback
execute_failback() {
log_info "Executing failback for $SERVICE_NAME..."
if [ "$SKIP_FAILBACK" = "true" ]; then
log_warn "Skipping failback (DR_TEST_SKIP_FAILBACK=true)"
return
fi
# Failback logic (reverse of failover)
log_info "Failback procedure (implement based on your requirements)"
# Example:
# 1. Verify primary is healthy
# 2. Update routing back to primary
# 3. Verify traffic is flowing to primary
# 4. Update standby to sync from primary
}
# Post-test verification
verify_service_health() {
log_info "Verifying service health after test..."
check_service_health
}
verify_data_consistency() {
log_info "Verifying data consistency..."
# Data consistency checks
# - Compare primary and standby data
# - Check replication lag
# - Verify no data loss
log_info "Data consistency check: OK (implement based on your requirements)"
}
# Main test execution
main() {
log_section "🚨 Disaster Recovery Test - $SERVICE_NAME"
START_TIME=$(date +%s)
# 1. Pre-test checklist
log_section "1️⃣ Pre-Test Checklist"
verify_backups
verify_standby_health
verify_documentation
# 2. Simulate disaster
log_section "2️⃣ Simulating Disaster"
simulate_disaster
# 3. Execute failover
log_section "3️⃣ Executing Failover"
if ! execute_failover; then
log_error "Failover failed, aborting test"
exit 1
fi
# 4. Verify service availability
log_section "4️⃣ Verifying Service Availability"
check_service_health
# 5. Restore primary
log_section "5️⃣ Restoring Primary"
restore_primary
# 6. Failback
log_section "6️⃣ Executing Failback"
execute_failback
# 7. Post-test verification
log_section "7️⃣ Post-Test Verification"
verify_service_health
verify_data_consistency
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
log_section "✅ DR Test Completed"
log_info "Duration: ${DURATION}s"
log_info "Service: $SERVICE_NAME"
log_info "Status: PASSED"
}
# Run main
main
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |