Nginx+Keepalived实现双机主备和双主热备

发表于 分类于 阅读次数： 本文字数： 1.6k 阅读时长 ≈ 6 分钟

Nginx+Keepalived实现双机主备和双主热备

准备

两台服务器

192.168.1.104
192.168.1.105
192.168.1.110 VIP
192.168.1.111 VIP

开启 112 端口，此为 Keepalived 通信端口

安装Nginx

下载
1
2
wget https://nginx.org/download/nginx-1.20.1.tar.gz
tar -zxvf nginx-1.20.1.tar.gz
依赖
1
2
3
4
yum install gcc-c++
yum install -y pcre pcre-devel
yum install -y zlib zlib-devel
yum install -y openssl openssl-devel
配置
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 创建临时目录
mkdir /var/temp/nginx -p

# 进入到nginx目录后执行此命令，为了创建makefile文件
./configure \
--prefix=/usr/local/nginx \
--pid-path=/var/run/nginx/nginx.pid \
--lock-path=/var/lock/nginx.lock \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/temp/nginx/client \
--http-proxy-temp-path=/var/temp/nginx/proxy \
--http-fastcgi-temp-path=/var/temp/nginx/fastcgi \
--http-uwsgi-temp-path=/var/temp/nginx/uwsgi \
--http-scgi-temp-path=/var/temp/nginx/scgi \
--with-http_ssl_module

# 编译并安装
make && make install

# 设置软链接
ln -s /usr/local/nginx/sbin/nginx /usr/bin/nginx
ln -s /usr/local/nginx/conf/ /etc/nginx
运行
1
2
cd /usr/local/nginx
sbin/nginx
修改两个 Nginx 的 index.html
1
2
3
vim html/index.html
将 h1 标签下 Welcome to nginx! 后添加当前主机的 ip，主要为了和另外一台机器做区分
sbin/nginx -s reload
访问 Nginx

image.png
image.png

安装 Keepalived

下载
1
2
wget https://www.keepalived.org/software/keepalived-2.2.4.tar.gz
tar -zxvf keepalived-2.2.4.tar.gz
配置
1
2
3
cd keepalived-2.2.4
./configure --prefix=/usr/local/keepalived --sysconf=/etc
make && make install
将 Keepalived 设为服务
1
2
3
4
5
cd /data/software/keepalived-2.2.4/keepalived/etc
cp init.d/keepalived /etc/init.d/
cp sysconfig/keepalived /etc/sysconfig/
systemctl daemon-reload
systemctl start keepalived.service

双机主备

修改配置文件（主）
1
vim /etc/keepalived/keepalived.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
! Configuration File for keepalived
global_defs {
   # 路由id：当前安装keepalived节点主机的标识符，全局唯一
   router_id keep_104
}

vrrp_instance VI_1 {
    # 表示的状态，当前的104为Nginx主节点，MASTER/BACKUP
    state MASTER
    # 当前实例绑定的网卡
    interface ens33
    # 保证主备节点一致
    virtual_router_id 51
    # 优先级/权重，谁的优先级高，在MASTER挂掉之后，就会成为MASTER
    priority 100
    # 主备之间同步检查的时间间隔，默认1秒
    advert_int 1
    # 认证授权的密码，防止非法节点的进入
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.110
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[root@localhost keepalived]# ip addr # 可以看到，我们的 VIP 已经设置成功
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:3a:50:37 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.104/24 brd 192.168.1.255 scope global noprefixroute dynamic ens33
       valid_lft 7148sec preferred_lft 7148sec
    inet 192.168.1.110/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::6734:7460:379a:910f/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:4e:e6:fa:e4 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:4eff:fee6:fae4/64 scope link 
       valid_lft forever preferred_lft forever
修改配置文件（备）
1
vim /etc/keepalived/keepalived.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
! Configuration File for keepalived
global_defs {
   # 路由id：当前安装keepalived节点主机的标识符，全局唯一
   router_id keep_105
}

vrrp_instance VI_1 {
    # 表示的状态，当前的104为Nginx主节点，MASTER/BACKUP
    state BACKUP
    # 当前实例绑定的网卡
    interface ens33
    # 保证主备节点一致
    virtual_router_id 51
    # 优先级/权重，谁的优先级高，在MASTER挂掉之后，就会成为MASTER
    priority 80
    # 主备之间同步检查的时间间隔，默认1秒
    advert_int 1
    # 认证授权的密码，防止非法节点的进入
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # 虚拟ip
    virtual_ipaddress {
        192.168.1.110
    }
}
重启 Keepalived 后进行测试

访问 VIP，http://192.168.1.110

image.png

关闭 104 节点 Keepalived，继续访问 VIP

image.png

配置 Nginx 自动重启

在 Keepalived 配置文件加添加

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
vrrp_script check_nginx_alive {
    script "/etc/keepalived/check_nginx_alive_or_not.sh"
    # 每隔两秒运行上一行脚本
    interval 2 
    # 如果脚本运行成功，则加权重10
    weight 10 
    # 如果脚本运行失败，则减权重10
    # weight -10 
}

vrrp_instance VI_1 {
    track_script {
        check_nginx_alive # 追踪 nginx 脚本
    }
}
1
2
3
4
5
6
7
8
9
10
11
#!/bin/bash
A=`ps -C nginx --no-header |wc -l`
# 判断nginx是否宕机，如果宕机了，尝试重启
if [ $A -eq 0 ];then
    /usr/local/nginx/sbin/nginx
    # 等待一小会再次检查nginx，如果没有启动成功，则停止keepalived，使其启动备用机
    sleep 3
    if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then
        killall keepalived
    fi
fi

双主热备

修改配置文件（104）
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
global_defs {
   # 路由id：当前安装keepalived节点主机的标识符，全局唯一
   router_id keep_104
}

vrrp_script check_nginx_alive {
    script "/etc/keepalived/check_nginx_alive_or_not.sh"
    interval 2 # 每隔两秒运行上一行脚本
    weight 10 # 如果脚本运行成功，则升级权重为10
    # weight -10 # 如果脚本运行失败，则降低权重为-10
}

vrrp_instance VI_1 {
    # 表示的状态，当前的104为Nginx主节点，MASTER/BACKUP
    state MASTER
    # 当前实例绑定的网卡
    interface ens33
    # 保证主备节点一致
    virtual_router_id 51
    # 优先级/权重，谁的优先级高，在MASTER挂掉之后，就会成为MASTER
    priority 100
    # 主备之间同步检查的时间间隔，默认1秒
    advert_int 1
    # 认证授权的密码，防止非法节点的进入
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        check_nginx_alive # 追踪 nginx 脚本
    }
    virtual_ipaddress {
        192.168.1.110
    }
}

vrrp_instance VI_2 {
    state BACKUP
    interface ens33
    virtual_router_id 52
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.111
    }
}
修改配置文件（105）
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
global_defs {
   # 路由id：当前安装keepalived节点主机的标识符，全局唯一
   router_id keep_105
}

vrrp_script check_nginx_alive {
    script "/etc/keepalived/check_nginx_alive_or_not.sh"
    interval 2 # 每隔两秒运行上一行脚本
    weight 10 # 如果脚本运行成功，则升级权重为10
    # weight -10 # 如果脚本运行失败，则降低权重为-10
}

vrrp_instance VI_1 {
    # 表示的状态，当前的104为Nginx主节点，MASTER/BACKUP
    state BACKUP
    # 当前实例绑定的网卡
    interface ens33
    # 保证主备节点一致
    virtual_router_id 51
    # 优先级/权重，谁的优先级高，在MASTER挂掉之后，就会成为MASTER
    priority 80
    # 主备之间同步检查的时间间隔，默认1秒
    advert_int 1
    # 认证授权的密码，防止非法节点的进入
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        check_nginx_alive # 追踪 nginx 脚本
    }
    # 虚拟ip
    virtual_ipaddress {
        192.168.1.110
    }
}

vrrp_instance VI_2 {
    state MASTER
    interface ens33
    virtual_router_id 52
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.111
    }
}

其他

问题处理
  • nginx: [emerg] mkdir() “/var/temp/nginx/client” failed (2: No such file or directory)
    解决方案：mkdir -p /var/temp/nginx
  • nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
    解决方案：检查防火墙，kill 掉 nginx 进程并重新启动
  • nginx: [emerg] open() “/var/run/nginx/nginx.pid” failed (2: No such file or directory)
    解决方案：touch /var/run/nginx/nginx.pid
日志文件定时切割
1
2
3
touch cut_my_log.sh
chmod +x cut_my_log.sh
vim cut_my_log.sh
1
2
3
4
5
6
7
8
#!/bin/bash
LOG_PATH="/var/log/nginx/"
RECORD_TIME=$(date -d "yesterday" +%Y-%m-%d+%H:%M)
PID=/var/run/nginx/nginx.pid
mv ${LOG_PATH}/access.log ${LOG_PATH}/access.${RECORD_TIME}.log
mv ${LOG_PATH}/error.log ${LOG_PATH}/error.${RECORD_TIME}.log
# 向Nginx主进程发送信号，用于重新打开日志文件
kill -USR1 `cat $PID`
1
2
3
4
5
6
7
8
yum install crontabs

crontab -e
# 每天凌晨1点执行
0 1 * * * /usr/local/nginx/sbin/cut_my_log.sh

crontab -l
service crond restart