Comprehensive IPVS (IP Virtual Server) Guide¶

Client Request
     ↓
[Virtual IP (VIP)]
     ↓
[IPVS Director/Load Balancer]
     ↓
[Scheduling Algorithm]
     ↓
[Real Servers] → RS1, RS2, RS3

1. Client → VIP:80
2. Director rewrites: VIP:80 → RS1:80
3. RS1 processes request
4. RS1 → Director
5. Director rewrites: RS1:80 → VIP:80
6. Director → Client

1. Client → VIP:80
2. Director forwards to RS1 (MAC address change)
3. RS1 receives packet (VIP on loopback)
4. RS1 → Client (directly, not through director)

1. Client → VIP:80
2. Director encapsulates packet with RS1's IP
3. RS1 decapsulates and processes
4. RS1 → Client (directly)

# Debian/Ubuntu
sudo apt-get update
sudo apt-get install ipvsadm

# CentOS/RHEL
sudo yum install ipvsadm

# Arch Linux
sudo pacman -S ipvsadm

# From source
wget http://www.linuxvirtualserver.org/software/kernel-2.6/ipvsadm-1.31.tar.gz
tar xzf ipvsadm-1.31.tar.gz
cd ipvsadm-1.31
make
sudo make install

# Load module
sudo modprobe ip_vs

# Verify module is loaded
lsmod | grep ip_vs

# Load additional modules for different schedulers
sudo modprobe ip_vs_rr      # Round-robin
sudo modprobe ip_vs_wrr     # Weighted round-robin
sudo modprobe ip_vs_lc      # Least-connection
sudo modprobe ip_vs_wlc     # Weighted least-connection
sudo modprobe ip_vs_lblc    # Locality-based least-connection
sudo modprobe ip_vs_lblcr   # Locality-based least-connection with replication
sudo modprobe ip_vs_dh      # Destination hashing
sudo modprobe ip_vs_sh      # Source hashing
sudo modprobe ip_vs_sed     # Shortest expected delay
sudo modprobe ip_vs_nq      # Never queue

# Forwarding methods
sudo modprobe ip_vs_ftp     # FTP support

# Load at boot
echo "ip_vs" | sudo tee -a /etc/modules

# Temporary
sudo sysctl -w net.ipv4.ip_forward=1

# Permanent
echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf
sudo sysctl -p

# For IPv6
sudo sysctl -w net.ipv6.conf.all.forwarding=1
echo "net.ipv6.conf.all.forwarding = 1" | sudo tee -a /etc/sysctl.conf

# Increase connection tracking table size
sudo sysctl -w net.netfilter.nf_conntrack_max=1048576
sudo sysctl -w net.ipv4.netfilter.ip_conntrack_max=1048576

# Connection tracking timeouts
sudo sysctl -w net.netfilter.nf_conntrack_tcp_timeout_established=7200
sudo sysctl -w net.ipv4.vs.conn_reuse_mode=1

# Enable IPVS connection sync (for HA setups)
sudo sysctl -w net.ipv4.vs.sync_threshold=3

# Persist changes
sudo tee -a /etc/sysctl.conf << EOF
net.netfilter.nf_conntrack_max = 1048576
net.ipv4.netfilter.ip_conntrack_max = 1048576
net.netfilter.nf_conntrack_tcp_timeout_established = 7200
net.ipv4.vs.conn_reuse_mode = 1
net.ipv4.vs.sync_threshold = 3
EOF

sudo sysctl -p

# Simplest method - distributes requests evenly
ipvsadm -A -t 10.0.0.100:80 -s rr

# Distributes based on server capacity/weight
ipvsadm -A -t 10.0.0.100:80 -s wrr
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -w 3 -m
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -w 1 -m

# Sends new connections to server with fewest active connections
ipvsadm -A -t 10.0.0.100:80 -s lc

# Least-connection with server weights
ipvsadm -A -t 10.0.0.100:80 -s wlc
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -w 3 -m
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -w 1 -m

# Routes based on source IP (session persistence)
ipvsadm -A -t 10.0.0.100:80 -s sh

# Routes based on destination IP/port
ipvsadm -A -t 10.0.0.100:80 -s dh

# Routes to server with shortest expected delay
ipvsadm -A -t 10.0.0.100:80 -s sed

# If server idle, send; otherwise use SED
ipvsadm -A -t 10.0.0.100:80 -s nq

# On Director
# Add virtual service
ipvsadm -A -t 10.0.0.100:80 -s wlc

# Add real servers with NAT (-m)
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -m
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -m

# Configure VIP
ip addr add 10.0.0.100/24 dev eth0

# Enable masquerading for return traffic
iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -j MASQUERADE

# On Real Servers
# Set director as default gateway
ip route add default via 192.168.1.1  # Director's IP

# On Director
# Add virtual service
ipvsadm -A -t 10.0.0.100:80 -s wlc

# Add real servers with DR (-g)
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -g
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -g

# Configure VIP on director
ip addr add 10.0.0.100/32 dev eth0

# On Real Servers
# Configure VIP on loopback (hidden from ARP)
ip addr add 10.0.0.100/32 dev lo

# Disable ARP on loopback for VIP
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce

# Persist ARP settings
cat >> /etc/sysctl.conf << EOF
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
EOF

sysctl -p

# On Director
# Add virtual service
ipvsadm -A -t 10.0.0.100:80 -s wlc

# Add real servers with tunneling (-i)
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -i
ipvsadm -a -t 10.0.0.100:80 -r 203.0.113.20:80 -i  # Can be remote

# Configure VIP
ip addr add 10.0.0.100/32 dev eth0

# On Real Servers
# Create tunnel interface
ip tunnel add tunl0 mode ipip
ip link set tunl0 up

# Configure VIP on tunnel
ip addr add 10.0.0.100/32 dev tunl0

# Disable ARP
echo 1 > /proc/sys/net/ipv4/conf/tunl0/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/tunl0/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce

#!/bin/bash
# Simple HTTP load balancer using NAT

# Clear existing rules
ipvsadm -C

# Enable IP forwarding
sysctl -w net.ipv4.ip_forward=1

# Add virtual service (VIP:80)
ipvsadm -A -t 10.0.0.100:80 -s wlc

# Add real servers
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -m -w 1
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -m -w 1
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.12:80 -m -w 2

# Configure VIP on interface
ip addr add 10.0.0.100/24 dev eth0

# Enable masquerading
iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -j MASQUERADE

# Save configuration
ipvsadm-save > /etc/ipvsadm.rules

# Display configuration
ipvsadm -Ln

#!/bin/bash
# High-performance web load balancer using DR

# Director configuration
ipvsadm -C
ipvsadm -A -t 10.0.0.100:80 -s wlc
ipvsadm -A -t 10.0.0.100:443 -s wlc

# Add real servers for HTTP
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -g -w 10
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -g -w 10
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.12:80 -g -w 5

# Add real servers for HTTPS
ipvsadm -a -t 10.0.0.100:443 -r 192.168.1.10:443 -g -w 10
ipvsadm -a -t 10.0.0.100:443 -r 192.168.1.11:443 -g -w 10
ipvsadm -a -t 10.0.0.100:443 -r 192.168.1.12:443 -g -w 5

# Configure VIP
ip addr add 10.0.0.100/32 dev eth0

ipvsadm-save > /etc/ipvsadm.rules

#!/bin/bash
# Real server configuration for DR

VIP="10.0.0.100"

# Add VIP to loopback
ip addr add ${VIP}/32 dev lo

# Configure ARP behavior
sysctl -w net.ipv4.conf.lo.arp_ignore=1
sysctl -w net.ipv4.conf.lo.arp_announce=2
sysctl -w net.ipv4.conf.all.arp_ignore=1
sysctl -w net.ipv4.conf.all.arp_announce=2

# Make permanent
cat >> /etc/sysctl.conf << EOF
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
EOF

#!/bin/bash
# DNS load balancer

ipvsadm -C

# Add UDP virtual service for DNS
ipvsadm -A -u 10.0.0.100:53 -s lc

# Add DNS servers
ipvsadm -a -u 10.0.0.100:53 -r 192.168.1.10:53 -m
ipvsadm -a -u 10.0.0.100:53 -r 192.168.1.11:53 -m

# Configure VIP
ip addr add 10.0.0.100/24 dev eth0

ipvsadm-save > /etc/ipvsadm.rules

#!/bin/bash
# Load balancer with session persistence

ipvsadm -C

# Use source hashing for sticky sessions
ipvsadm -A -t 10.0.0.100:80 -s sh

# Add real servers
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -g
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.11:80 -g
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.12:80 -g

# Enable connection persistence (timeout in seconds)
ipvsadm -A -t 10.0.0.100:80 -s sh -p 360

ip addr add 10.0.0.100/32 dev eth0

#!/bin/bash
# FTP load balancer with passive mode support

# Load FTP helper module
modprobe ip_vs_ftp

ipvsadm -C

# FTP control connection
ipvsadm -A -t 10.0.0.100:21 -s wlc -p 600

# Add FTP servers
ipvsadm -a -t 10.0.0.100:21 -r 192.168.1.10:21 -m
ipvsadm -a -t 10.0.0.100:21 -r 192.168.1.11:21 -m

# Passive FTP data ports (example range)
ipvsadm -A -t 10.0.0.100:30000-30100 -s wlc
ipvsadm -a -t 10.0.0.100:30000-30100 -r 192.168.1.10:30000-30100 -m
ipvsadm -a -t 10.0.0.100:30000-30100 -r 192.168.1.11:30000-30100 -m

ip addr add 10.0.0.100/24 dev eth0

# On Primary Director
# Start connection sync daemon
ipvsadm --start-daemon=master --mcast-interface=eth1

# On Backup Director
# Start backup sync daemon
ipvsadm --start-daemon=backup --mcast-interface=eth1

# View sync status
ipvsadm -Ln --stats

# Mark packets with iptables
iptables -t mangle -A PREROUTING -p tcp --dport 80 -j MARK --set-mark 1
iptables -t mangle -A PREROUTING -p tcp --dport 443 -j MARK --set-mark 1

# Create IPVS service based on mark
ipvsadm -A -f 1 -s wlc

# Add real servers
ipvsadm -a -f 1 -r 192.168.1.10:0 -g
ipvsadm -a -f 1 -r 192.168.1.11:0 -g

# Now both HTTP and HTTPS go to same backend pool

# For UDP services, enable one-packet scheduling
# Useful for DNS, syslog, etc.
ipvsadm -A -u 10.0.0.100:53 -s wlc -O

# Add real servers
ipvsadm -a -u 10.0.0.100:53 -r 192.168.1.10:53 -m

# Set connection template timeout (for persistence)
echo 300 > /proc/sys/net/ipv4/vs/conn_template_timeout

# Or via sysctl
sysctl -w net.ipv4.vs.conn_template_timeout=300

# Limit connections per second to backend
ipvsadm -E -t 10.0.0.100:80 -s wlc --ops-min-conn 100 --ops-max-conn 1000

# View all services and servers
ipvsadm -Ln

# View with statistics
ipvsadm -Ln --stats

# View with rate information
ipvsadm -Ln --rate

# View connection table
ipvsadm -Ln -c

# View detailed connection table
cat /proc/net/ip_vs_conn

# View IPVS statistics
cat /proc/net/ip_vs_stats

# Check if service is defined
ipvsadm -Ln

# Check if real server is active
ipvsadm -Ln | grep "192.168.1.10"

# Check connection table
ipvsadm -Ln -c | grep "192.168.1.10"

# Capture traffic on director
tcpdump -i eth0 -nn host 192.168.1.10

# Capture on real server
tcpdump -i eth0 -nn host 10.0.0.100

# Verify real server is reachable
ping 192.168.1.10

# Check if service is running on real server
nc -zv 192.168.1.10 80

# Verify forwarding method is correct
ipvsadm -Ln  # Check for -m, -g, or -i flag

# Check weight (might be 0)
ipvsadm -Ln | grep "192.168.1.10"
# If weight is 0, set it:
ipvsadm -e -t 10.0.0.100:80 -r 192.168.1.10:80 -g -w 1

# Check connection timeout values
cat /proc/sys/net/ipv4/vs/timeout_*

# View active timeouts
sysctl -a | grep "net.ipv4.vs.timeout"

# Monitor connection table
watch -n 1 'ipvsadm -Ln -c | wc -l'

# Increase TCP timeout
sysctl -w net.ipv4.vs.timeout_established=7200

# Adjust other timeouts
sysctl -w net.ipv4.vs.timeout_close=60
sysctl -w net.ipv4.vs.timeout_timewait=120

# Make permanent
cat >> /etc/sysctl.conf << EOF
net.ipv4.vs.timeout_established = 7200
net.ipv4.vs.timeout_close = 60
net.ipv4.vs.timeout_timewait = 120
EOF

sysctl -p

# Check ARP table on director
arp -an

# Check if VIP responds to ARP from real servers
arping -I eth0 10.0.0.100

# Check ARP settings on real server
sysctl -a | grep arp_ignore
sysctl -a | grep arp_announce

# On Real Server - fix ARP configuration
sysctl -w net.ipv4.conf.lo.arp_ignore=1
sysctl -w net.ipv4.conf.lo.arp_announce=2
sysctl -w net.ipv4.conf.all.arp_ignore=1
sysctl -w net.ipv4.conf.all.arp_announce=2

# Verify VIP is on loopback
ip addr show lo | grep 10.0.0.100

# If not, add it
ip addr add 10.0.0.100/32 dev lo

# Check CPU usage
top -bn1 | grep "Cpu(s)"

# Check interrupt distribution
cat /proc/interrupts | grep eth

# Check connection table size
wc -l /proc/net/ip_vs_conn

# Check if conntrack table is full
dmesg | grep conntrack

# View IPVS stats
cat /proc/net/ip_vs_stats

# Increase conntrack table
sysctl -w net.netfilter.nf_conntrack_max=1048576

# Enable connection reuse
sysctl -w net.ipv4.vs.conn_reuse_mode=1

# Adjust scheduler for better distribution
ipvsadm -E -t 10.0.0.100:80 -s wlc

# Enable RPS (Receive Packet Steering)
echo f > /sys/class/net/eth0/queues/rx-0/rps_cpus

# Increase receive buffer
sysctl -w net.core.rmem_max=134217728
sysctl -w net.core.rmem_default=134217728

# Enable IPVS debugging
echo 1 > /proc/sys/net/ipv4/vs/debug_level
# Levels: 0=none, 1=basic, 2=detailed, 3=verbose

# View debug messages
dmesg | tail -f

# Disable debug
echo 0 > /proc/sys/net/ipv4/vs/debug_level

#!/bin/bash
# Test load distribution

VIP="10.0.0.100"
PORT="80"

# Send 100 requests and count backend responses
for i in {1..100}; do
    curl -s http://${VIP}:${PORT}/ | grep -o "Server: .*" &
done | sort | uniq -c

# Expected output (for 3 backends with equal weight):
# 33 Server: backend1
# 33 Server: backend2
# 34 Server: backend3

#!/bin/bash
# health-check.sh - Monitor real server health

VIP="10.0.0.100"
PORT="80"
REAL_SERVERS=("192.168.1.10" "192.168.1.11" "192.168.1.12")

check_server() {
    local server=$1
    local port=$2

    if nc -zw3 "$server" "$port" >/dev/null 2>&1; then
        echo "[OK] $server:$port is responsive"
        # Ensure server is in pool
        ipvsadm -Ln | grep -q "$server" || \
            ipvsadm -a -t ${VIP}:${PORT} -r ${server}:${port} -g
    else
        echo "[FAIL] $server:$port is down"
        # Remove from pool
        ipvsadm -d -t ${VIP}:${PORT} -r ${server}:${port} 2>/dev/null
    fi
}

# Check all servers
for server in "${REAL_SERVERS[@]}"; do
    check_server "$server" "$PORT"
done

# Use DR mode for production (best performance)
ipvsadm -A -t 10.0.0.100:80 -s wlc
ipvsadm -a -t 10.0.0.100:80 -r 192.168.1.10:80 -g

# Use weighted least-connection for varying server capacity
ipvsadm -E -t 10.0.0.100:80 -s wlc

# Enable persistence for stateful applications
ipvsadm -A -t 10.0.0.100:80 -s wlc -p 360

# Use keepalived for director failover
apt-get install keepalived

# /etc/keepalived/keepalived.conf
cat > /etc/keepalived/keepalived.conf << 'EOF'
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass secret
    }

    virtual_ipaddress {
        10.0.0.100/24
    }
}

virtual_server 10.0.0.100 80 {
    delay_loop 6
    lb_algo wlc
    lb_kind DR
    protocol TCP

    real_server 192.168.1.10 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 3
            connect_port 80
        }
    }

    real_server 192.168.1.11 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 3
            connect_port 80
        }
    }
}
EOF

#!/bin/bash
# monitor-ipvs.sh - Monitoring script

while true; do
    clear
    echo "=== IPVS Status at $(date) ==="
    echo ""

    echo "Virtual Services:"
    ipvsadm -Ln

    echo ""
    echo "Statistics:"
    ipvsadm -Ln --stats

    echo ""
    echo "Active Connections:"
    wc -l /proc/net/ip_vs_conn

    echo ""
    echo "System Stats:"
    cat /proc/net/ip_vs_stats

    sleep 5
done

# Limit connections from single IP
iptables -A INPUT -p tcp --dport 80 -m connlimit \
    --connlimit-above 100 --connlimit-mask 32 -j REJECT

# Rate limit SYN packets
iptables -A INPUT -p tcp --syn -m limit \
    --limit 10/s --limit-burst 20 -j ACCEPT

# Drop invalid packets
iptables -A INPUT -m state --state INVALID -j DROP

# Only allow established connections to backends
iptables -A FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT

# Optimal sysctl settings
cat >> /etc/sysctl.conf << EOF
# IPVS Performance
net.ipv4.vs.conn_reuse_mode = 1
net.ipv4.vs.expire_nodest_conn = 1
net.ipv4.vs.expire_quiescent_template = 1

# Connection Tracking
net.netfilter.nf_conntrack_max = 1048576
net.netfilter.nf_conntrack_tcp_timeout_established = 7200

# Network Stack
net.core.netdev_max_backlog = 5000
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.ipv4.tcp_rmem = 4096 87380 134217728
net.ipv4.tcp_wmem = 4096 65536 134217728

# TCP Optimization
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_tw_reuse = 1
EOF

sysctl -p

# Enable IPVS mode in kube-proxy

# Load modules on all nodes
cat > /etc/modules-load.d/ipvs.conf << EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF

# Load now
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack

# Configure kube-proxy
kubectl edit configmap kube-proxy -n kube-system

# Set mode to ipvs
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-proxy
data:
  config.conf: |-
    mode: "ipvs"
    ipvs:
      scheduler: "wlc"
      minSyncPeriod: 0s
      syncPeriod: 30s

# Restart kube-proxy pods
kubectl delete pod -n kube-system -l k8s-app=kube-proxy

# Verify IPVS mode
ipvsadm -Ln

# PostgreSQL read replica load balancing

ipvsadm -C
ipvsadm -A -t 10.0.0.100:5432 -s lc -p 3600

# Add read replicas
ipvsadm -a -t 10.0.0.100:5432 -r 192.168.1.10:5432 -g -w 1
ipvsadm -a -t 10.0.0.100:5432 -r 192.168.1.11:5432 -g -w 1
ipvsadm -a -t 10.0.0.100:5432 -r 192.168.1.12:5432 -g -w 1

# Long persistence timeout for database connections
sysctl -w net.ipv4.vs.timeout_established=28800  # 8 hours

# Using tunneling for geographically distributed servers

ipvsadm -C
ipvsadm -A -t 10.0.0.100:80 -s wlc

# Europe server
ipvsadm -a -t 10.0.0.100:80 -r 203.0.113.10:80 -i -w 10

# Asia server
ipvsadm -a -t 10.0.0.100:80 -r 198.51.100.20:80 -i -w 10

# US server
ipvsadm -a -t 10.0.0.100:80 -r 192.0.2.30:80 -i -w 10