Disaster Recovery

                    ┌─────────────────────────────────────────┐
                    │           Backup Architecture           │
                    ├─────────────────────────────────────────┤
                    │                                         │
                    │  PostgreSQL (RDS)                       │
                    │  ├── Automated snapshots (daily, 7d)    │
                    │  ├── Continuous WAL archiving (5m RPO)  │
                    │  └── Manual snapshots (pre-migration)   │
                    │                                         │
                    │  Redis (ElastiCache)                    │
                    │  ├── Daily snapshots (retained 7d)      │
                    │  └── AOF persistence (1s sync)          │
                    │                                         │
                    │  Application State                      │
                    │  ├── Git (infrastructure as code)       │
                    │  ├── S3 (pipeline artifacts, exports)   │
                    │  └── Secrets Manager (credentials)      │
                    │                                         │
                    │  All backups replicated to us-west-2    │
                    └─────────────────────────────────────────┘

# Pre-migration snapshot
aws rds create-db-snapshot \
  --db-instance-identifier kaireon-prod \
  --db-snapshot-identifier "kaireon-pre-migration-$(date +%Y%m%d-%H%M%S)"

# Cross-region copy for DR
aws rds copy-db-snapshot \
  --source-db-snapshot-identifier "kaireon-pre-migration-20260223-120000" \
  --target-db-snapshot-identifier "kaireon-pre-migration-20260223-120000" \
  --source-region us-east-1 \
  --region us-west-2

# Daily logical backup (in addition to RDS snapshots)
make backup

# Stored in: s3://kaireon-backups/daily/
# Cross-region replicated to: s3://kaireon-backups-dr/daily/ (us-west-2)

# Manual snapshot
aws elasticache create-snapshot \
  --replication-group-id kaireon-redis \
  --snapshot-name "kaireon-redis-$(date +%Y%m%d-%H%M%S)"

# Export to S3 for cross-region access
aws elasticache copy-snapshot \
  --source-snapshot-name "kaireon-redis-20260223-120000" \
  --target-snapshot-name "kaireon-redis-20260223-120000-dr" \
  --target-bucket "kaireon-backups-dr"

appendonly yes
appendfsync everysec
save 900 1
save 300 10
save 60 10000

# Automated backup verification script
#!/bin/bash
set -euo pipefail

echo "=== KaireonAI Backup Verification ==="
echo "Date: $(date -u)"

# 1. Verify RDS snapshot exists
LATEST_SNAPSHOT=$(aws rds describe-db-snapshots \
  --db-instance-identifier kaireon-prod \
  --query 'DBSnapshots | sort_by(@, &SnapshotCreateTime) | [-1].DBSnapshotIdentifier' \
  --output text)
echo "Latest RDS snapshot: $LATEST_SNAPSHOT"

# 2. Verify S3 backup exists
LATEST_S3=$(aws s3 ls s3://kaireon-backups/daily/ --recursive | sort | tail -1)
echo "Latest S3 backup: $LATEST_S3"

# 3. Verify cross-region replica
LATEST_DR=$(aws s3 ls s3://kaireon-backups-dr/daily/ --recursive --region us-west-2 | sort | tail -1)
echo "Latest DR backup: $LATEST_DR"

# 4. Verify Redis snapshot
LATEST_REDIS=$(aws elasticache describe-snapshots \
  --replication-group-id kaireon-redis \
  --query 'Snapshots | sort_by(@, &NodeSnapshots[0].SnapshotCreateTime) | [-1].SnapshotName' \
  --output text)
echo "Latest Redis snapshot: $LATEST_REDIS"

echo "=== Verification Complete ==="

# Trigger failover
aws rds reboot-db-instance \
  --db-instance-identifier kaireon-prod \
  --force-failover

# Monitor failover progress
watch -n 5 "aws rds describe-db-instances \
  --db-instance-identifier kaireon-prod \
  --query 'DBInstances[0].[DBInstanceStatus,AvailabilityZone]'"

# 1. Check the new AZ
aws rds describe-db-instances \
  --db-instance-identifier kaireon-prod \
  --query 'DBInstances[0].AvailabilityZone'

# 2. Verify application connectivity
kubectl exec -it deploy/kaireon-api -n kaireon -- curl -s localhost:9090/healthz

# 3. Check PgBouncer reconnected
kubectl exec -it deploy/kaireon-pgbouncer -n kaireon -- psql -p 6432 pgbouncer -c "SHOW POOLS;"

# 4. Monitor error rate for 10 minutes
kubectl logs -l app=kaireon-api -n kaireon --since=5m | grep -c "error" || echo "0 errors"

# Trigger failover
aws elasticache test-failover \
  --replication-group-id kaireon-redis \
  --node-group-id 0001

# Monitor
watch -n 5 "aws elasticache describe-replication-groups \
  --replication-group-id kaireon-redis \
  --query 'ReplicationGroups[0].Status'"

# 1. Verify Redis is responding
kubectl exec -it deploy/kaireon-api -n kaireon -- redis-cli -h kaireon-redis PING

# 2. Check for data loss (compare key count)
kubectl exec -it deploy/kaireon-api -n kaireon -- redis-cli -h kaireon-redis DBSIZE

# 3. Verify application is using the new primary
kubectl logs -l app=kaireon-api -n kaireon --since=2m | grep -i redis

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: kaireon-api-pdb
  namespace: kaireon
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: kaireon-api
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: kaireon-worker-pdb
  namespace: kaireon
spec:
  minAvailable: 1
  selector:
    matchLabels:
      app: kaireon-worker

# 1. Check node status
kubectl get nodes
kubectl describe node <failed-node>

# 2. Check affected pods
kubectl get pods --all-namespaces --field-selector spec.nodeName=<failed-node>

# 3. If node is NotReady for >5 minutes, drain it
kubectl drain <failed-node> --ignore-daemonsets --delete-emptydir-data --force

# 4. Verify pods rescheduled
kubectl get pods -n kaireon -o wide

# 5. If cluster autoscaler is not adding nodes, add manually
aws eks update-nodegroup-config \
  --cluster-name kaireon-prod \
  --nodegroup-name kaireon-workers \
  --scaling-config desiredSize=5

# 1. Verify which AZ is affected
kubectl get nodes -o custom-columns=NAME:.metadata.name,ZONE:.metadata.labels."topology\.kubernetes\.io/zone"

# 2. Ensure node group spans multiple AZs
aws eks describe-nodegroup \
  --cluster-name kaireon-prod \
  --nodegroup-name kaireon-workers \
  --query 'nodegroup.subnets'

# 3. Scale up to compensate for lost capacity
kubectl patch hpa kaireon-api-hpa -n kaireon --type merge -p \
  '{"spec":{"minReplicas":6}}'

# === STEP 1: Confirm primary region is down ===
# Verify from multiple sources before declaring a regional failure
aws health describe-events --region us-east-1
curl -s -o /dev/null -w "%{http_code}" https://api.kaireon.com/healthz

# === STEP 2: Activate DR database ===
# Restore from the latest cross-region snapshot
LATEST_DR_SNAPSHOT=$(aws rds describe-db-snapshots \
  --region us-west-2 \
  --query 'DBSnapshots | sort_by(@, &SnapshotCreateTime) | [-1].DBSnapshotIdentifier' \
  --output text)

aws rds restore-db-instance-from-db-snapshot \
  --region us-west-2 \
  --db-instance-identifier kaireon-dr \
  --db-snapshot-identifier "$LATEST_DR_SNAPSHOT" \
  --db-instance-class db.r6g.xlarge \
  --multi-az \
  --vpc-security-group-ids sg-dr-xxxxxxxx

aws rds wait db-instance-available \
  --region us-west-2 \
  --db-instance-identifier kaireon-dr

# === STEP 3: Activate DR Redis ===
LATEST_REDIS_SNAPSHOT=$(aws elasticache describe-snapshots \
  --region us-west-2 \
  --query 'Snapshots | sort_by(@, &NodeSnapshots[0].SnapshotCreateTime) | [-1].SnapshotName' \
  --output text)

aws elasticache create-replication-group \
  --region us-west-2 \
  --replication-group-id kaireon-redis-dr \
  --replication-group-description "KaireonAI Redis DR" \
  --snapshot-name "$LATEST_REDIS_SNAPSHOT" \
  --cache-node-type cache.r6g.large \
  --automatic-failover-enabled \
  --multi-az-enabled

# === STEP 4: Update application configuration ===
kubectl --context kaireon-dr config use-context kaireon-dr

kubectl set env deploy/kaireon-api -n kaireon \
  DATABASE_URL="postgresql://$DB_USER:$DB_PASSWORD@kaireon-dr.xxxxxxxx.us-west-2.rds.amazonaws.com:5432/kaireon" \
  REDIS_URL="redis://kaireon-redis-dr.xxxxxxxx.usw2.cache.amazonaws.com:6379"

kubectl set env deploy/kaireon-worker -n kaireon \
  DATABASE_URL="postgresql://$DB_USER:$DB_PASSWORD@kaireon-dr.xxxxxxxx.us-west-2.rds.amazonaws.com:5432/kaireon" \
  REDIS_URL="redis://kaireon-redis-dr.xxxxxxxx.usw2.cache.amazonaws.com:6379"

# === STEP 5: Scale up DR workloads ===
kubectl --context kaireon-dr scale deploy/kaireon-api -n kaireon --replicas=5
kubectl --context kaireon-dr scale deploy/kaireon-worker -n kaireon --replicas=3

# === STEP 6: Switch DNS ===
aws route53 change-resource-record-sets \
  --hosted-zone-id ZXXXXXXXXXXXXX \
  --change-batch '{
    "Changes": [{
      "Action": "UPSERT",
      "ResourceRecordSet": {
        "Name": "api.kaireon.com",
        "Type": "A",
        "AliasTarget": {
          "HostedZoneId": "Z1H1FL5HABSF5",
          "DNSName": "kaireon-dr-alb.us-west-2.elb.amazonaws.com",
          "EvaluateTargetHealth": true
        }
      }
    }]
  }'

# === STEP 7: Verify ===
curl -s https://api.kaireon.com/healthz
echo "DR failover complete at $(date -u)"

# === STEP 1: Verify primary region is stable ===
aws health describe-events --region us-east-1
# Wait at least 24 hours after region recovery before failing back

# === STEP 2: Sync data from DR to primary ===
# Create a snapshot of the DR database
aws rds create-db-snapshot \
  --region us-west-2 \
  --db-instance-identifier kaireon-dr \
  --db-snapshot-identifier "kaireon-failback-$(date +%Y%m%d-%H%M%S)"

# Copy to primary region
aws rds copy-db-snapshot \
  --region us-east-1 \
  --source-region us-west-2 \
  --source-db-snapshot-identifier "kaireon-failback-20260223-120000" \
  --target-db-snapshot-identifier "kaireon-failback-20260223-120000"

# Restore in primary region
aws rds restore-db-instance-from-db-snapshot \
  --region us-east-1 \
  --db-instance-identifier kaireon-prod-restored \
  --db-snapshot-identifier "kaireon-failback-20260223-120000" \
  --db-instance-class db.r6g.xlarge \
  --multi-az

# === STEP 3: Switch traffic back ===
# Follow the same DNS update procedure as failover but point back to us-east-1

# === STEP 4: Decommission DR resources ===
# After 48 hours of stable operation on primary:
aws rds delete-db-instance --region us-west-2 --db-instance-identifier kaireon-dr --skip-final-snapshot
aws elasticache delete-replication-group --region us-west-2 --replication-group-id kaireon-redis-dr

-- Compare record counts between backup and current
-- (Run against a restored copy of the pre-failure backup)

-- Decision logs gap analysis
SELECT
  date_trunc('hour', created_at) AS hour,
  count(*) AS decisions
FROM decision_logs
WHERE created_at > NOW() - INTERVAL '48 hours'
GROUP BY hour
ORDER BY hour;

-- Identify the gap window
SELECT
  min(created_at) AS first_record,
  max(created_at) AS last_record,
  count(*) AS total_records
FROM decision_logs
WHERE created_at > NOW() - INTERVAL '48 hours';

# DR Test Report - [Date]

## Summary
- **Test type:** [Regional failover / RDS failover / Redis failover / etc.]
- **Date:** [YYYY-MM-DD]
- **Duration:** [HH:MM]
- **Participants:** [Names]

## Results
| Metric | Target | Actual | Pass/Fail |
|--------|--------|--------|-----------|
| RTO | [target] | [actual] | [P/F] |
| RPO | [target] | [actual] | [P/F] |
| Data integrity | 100% | [actual]% | [P/F] |

## Timeline
| Time | Action | Result |
|------|--------|--------|
| HH:MM | [action] | [result] |

## Issues Found
1. [Issue description, severity, owner, deadline]

## Runbook Updates Required
1. [Update description]

## Action Items
- [ ] [Action] - Owner: [name] - Due: [date]