Skip to main content

High availability

Table of contents​

  1. High availability concepts
  2. Load balancing
  3. Clustering with Keepalived
  4. Pacemaker and Corosync
  5. Data replication
  6. Practical exercises

1 - High availability concepts​

What is HA?​

SPOF = Single Point of Failure

Availability levels​

LevelDowntime/yearArchitecture
99%3.65 daysBackup + restore
99.9%8.76 hoursActive/passive redundancy
99.99%52.6 minutesActive/active cluster
99.999%5.26 minutesGeo-distribution

HA patterns​

PatternDescription
Active/PassiveOnly one active node, manual or automatic failover
Active/ActiveAll nodes active, load balancing
N+1N active servers + 1 spare
N+MN active + M spares

Components of an HA solution​

🔝 Back to table of contents​


2 - Load balancing​

HAProxy - Basic configuration​

# /etc/haproxy/haproxy.cfg

global
    daemon
    maxconn 4096

defaults
    mode http
    timeout connect 5s
    timeout client 50s
    timeout server 50s
    option httplog

frontend http_front
    bind *:80
    default_backend http_back

backend http_back
    balance roundrobin
    option httpchk GET /health
    server web1 192.168.1.10:80 check
    server web2 192.168.1.11:80 check
    server web3 192.168.1.12:80 check backup

Load balancing algorithms​

AlgorithmDescription
roundrobinSequential distribution
leastconnTo the server with the fewest connections
sourceSticky by source IP (hash)
uriHash based on the URI

Health checks​

backend http_back
    option httpchk GET /health HTTP/1.1\r\nHost:\ localhost
    http-check expect status 200
    
    server web1 192.168.1.10:80 check inter 3s fall 3 rise 2
    # inter: intervalle entre checks
    # fall: nombre d'échecs avant down
    # rise: nombre de succès avant up

HAProxy stats​

listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 10s
    stats auth admin:password

🔝 Back to table of contents​


3 - Clustering with Keepalived​

VRRP architecture​

Installation​

apt install keepalived

Master configuration​

# /etc/keepalived/keepalived.conf (Master)

global_defs {
    router_id LB_MASTER
}

vrrp_script check_haproxy {
    script "/usr/bin/killall -0 haproxy"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 101
    advert_int 1
    
    authentication {
        auth_type PASS
        auth_pass secret123
    }
    
    virtual_ipaddress {
        192.168.1.100/24
    }
    
    track_script {
        check_haproxy
    }
}

Backup configuration​

# /etc/keepalived/keepalived.conf (Backup)

global_defs {
    router_id LB_BACKUP
}

vrrp_script check_haproxy {
    script "/usr/bin/killall -0 haproxy"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    
    authentication {
        auth_type PASS
        auth_pass secret123
    }
    
    virtual_ipaddress {
        192.168.1.100/24
    }
    
    track_script {
        check_haproxy
    }
}

Notification scripts​

# /etc/keepalived/notify.sh
#!/bin/bash

TYPE=$1
NAME=$2
STATE=$3

case $STATE in
    "MASTER")
        echo "$(date) Becoming MASTER" >> /var/log/keepalived-state.log
        # Actions: démarrer services, etc.
        ;;
    "BACKUP")
        echo "$(date) Becoming BACKUP" >> /var/log/keepalived-state.log
        ;;
    "FAULT")
        echo "$(date) FAULT state" >> /var/log/keepalived-state.log
        ;;
esac
# Dans keepalived.conf
vrrp_instance VI_1 {
    ...
    notify /etc/keepalived/notify.sh
}

🔝 Back to table of contents​


4 - Pacemaker and Corosync​

Architecture​

Installation​

apt install pacemaker corosync pcs
systemctl enable --now pcsd

# Définir mot de passe hacluster
passwd hacluster

Initial configuration​

# Sur tous les nœuds, authentifier
pcs host auth node1 node2

# Créer le cluster
pcs cluster setup ha-cluster node1 node2

# Démarrer
pcs cluster start --all
pcs cluster enable --all

# Vérifier
pcs status

Create resources​

# VIP
pcs resource create VirtualIP ocf:heartbeat:IPaddr2 \
    ip=192.168.1.100 cidr_netmask=24 \
    op monitor interval=30s

# Service
pcs resource create WebServer systemd:nginx \
    op monitor interval=30s

# Groupe (ressources ensemble)
pcs resource group add WebGroup VirtualIP WebServer

Constraints​

# Colocation (ressources ensemble)
pcs constraint colocation add WebServer with VirtualIP INFINITY

# Ordre (VIP avant WebServer)
pcs constraint order VirtualIP then WebServer

# Localisation (préférence de nœud)
pcs constraint location WebServer prefers node1=100

Cluster management​

# Statut
pcs status
crm_mon -1

# Mettre en maintenance
pcs node standby node1
pcs node unstandby node1

# Déplacer une ressource
pcs resource move VirtualIP node2
pcs resource clear VirtualIP

# Logs
journalctl -u pacemaker
journalctl -u corosync

🔝 Back to table of contents​


5 - Data replication​

DRBD - Block device replication​

# Installation
apt install drbd-utils

# /etc/drbd.d/data.res
resource data {
    protocol C;
    
    on node1 {
        device /dev/drbd0;
        disk /dev/sdb1;
        address 192.168.1.10:7789;
        meta-disk internal;
    }
    on node2 {
        device /dev/drbd0;
        disk /dev/sdb1;
        address 192.168.1.11:7789;
        meta-disk internal;
    }
}

# Initialiser
drbdadm create-md data
drbdadm up data
drbdadm primary --force data  # Sur le primaire

# Statut
drbdadm status
cat /proc/drbd

GlusterFS - Distributed file system​

# Installation
apt install glusterfs-server

# Créer le cluster
gluster peer probe node2
gluster pool list

# Créer un volume répliqué
gluster volume create gv0 replica 2 \
    node1:/data/brick1 \
    node2:/data/brick1

gluster volume start gv0

# Monter
mount -t glusterfs node1:/gv0 /mnt/gluster

PostgreSQL replication​

# postgresql.conf (Primary)
wal_level = replica
max_wal_senders = 3
wal_keep_size = 64MB

# pg_hba.conf
host replication replicator 192.168.1.0/24 md5

# Sur le replica
pg_basebackup -h primary -D /var/lib/postgresql/14/main -U replicator -P

# postgresql.conf (Replica)
primary_conninfo = 'host=primary user=replicator password=secret'

🔝 Back to table of contents​


6 - Practical exercises​

Exercise 1: Basic HAProxy​

Configure HAProxy to balance between 2 web servers:

Solution
# /etc/haproxy/haproxy.cfg
frontend web
    bind *:80
    default_backend servers

backend servers
    balance roundrobin
    option httpchk GET /
    server web1 192.168.1.10:80 check
    server web2 192.168.1.11:80 check

Exercise 2: Keepalived VIP​

Configure a VIP between 2 servers:

Solution
# Master (priority 101)
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 101
    virtual_ipaddress {
        192.168.1.100/24
    }
}

# Backup (priority 100)
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 100
    virtual_ipaddress {
        192.168.1.100/24
    }
}

Quiz​

Q1. What is the difference between Active/Passive and Active/Active?

Answer
  • Active/Passive: Only one node handles the traffic, the other is on standby
  • Active/Active: All nodes handle the traffic simultaneously

Q2. Which protocol does Keepalived use for the VIP?

Answer

VRRP (Virtual Router Redundancy Protocol)

🔝 Back to table of contents​


Key takeaways​

  • Eliminate SPOFs (Single Points of Failure)
  • HAProxy for load balancing
  • Keepalived for a simple VIP with VRRP
  • Pacemaker/Corosync for complex clusters
  • DRBD/GlusterFS for data replication
  • Always test failover scenarios

🔝 Back to table of contents​

← Previous chapter | Next chapter: Virtualization →