IP Tunneling 基础案例错误日志

原创已于 2026-04-22 15:40:31 修改 · 242 阅读

3 ·

本内容遵循CC 4.0 BY-SA版权协议

GEO检测

标签

#tcp/ip #网络协议 #网络

于 2026-04-20 17:37:51 首次发布

运维错误日志专栏收录该内容

1 篇文章

订阅专栏

主机规划

主机角色	ip
rs1	RIP:192.168.122.7 tunl0: 192.168.122.100
rs2	RIP:192.168.122.17 tunl0:192.168.122.100
client	IP: 192.168.197.100
lvs	DIP: 192.168.122.8 VIP: 192.168.122.100
router	enp1s0: 192.168.122.200 enp7s0:192.168.197.200

内核参数修改

1. rs1 和 rs2 的内核参数修改

[root@rs1 ~]# vim /etc/sysctl.conf
...
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.tunl0.rp_filter=0
net.ipv4.conf.all.arp_ignore=1
net.ipv4.conf.tunl0.arp_ignore=1
net.ipv4.conf.all.arp_announce=2
net.ipv4.conf.tunl0.arp_announce=2

[root@rs2 ~]# vim /etc/sysctl.conf
...
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.tunl0.rp_filter=0
net.ipv4.conf.all.arp_ignore=1
net.ipv4.conf.tunl0.arp_ignore=1
net.ipv4.conf.all.arp_announce=2
net.ipv4.conf.tunl0.arp_announce=2

使上面的配置生效

sysctl -p

启用内核的ipip功能

modprobe ipip
lsmod | grep ipip

2. 启用router`直连路由`功能.

echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf 
sysctl -p

3. 启用lvs的ipip功能。

lsmod | grep ipip
modprobe ipip
lsmod | grep ipip

`lvs` 主机配置

ipvsadm-save /etc/sysconfig/ipvsadm
ipvsadm -A -t 192.168.122.100:80 -s rr
ipvsadm -a -t 192.168.122.100:80 -r 192.168.122.7:80 -i
ipvsadm -a -t 192.168.122.100:80 -r 192.168.122.17:80 -i

ipvsadm -Ln

rs1 和 rs2 的配置

两台rs都配置了nginx服务并启动。

全局配置

所有的主机都已经关闭了防火墙
所有主机的selinux 都是permission模式。

异常描述

client主机 curl 192.168.122.100 时报出错误：

curl: (7) Failed to connect to 192.168.122.100 port 80: 拒绝连接

在lvs 查看ipvsadm -Ln时，发现字段ActiveConn的数字会因为客户端的请求轮询的增加

[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.122.100:80 rr
  -> 192.168.122.7:80             Tunnel  1      1          0         
  -> 192.168.122.17:80            Tunnel  1      1          0         
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.122.100:80 rr
  -> 192.168.122.7:80             Tunnel  1      2          0         
  -> 192.168.122.17:80            Tunnel  1      1          0

从这里可以判断出流量从客户端到rs时，路线是没有问题的，问题是在流量返回时。

问题排查

网络连通性检查

[root@client ~]# curl 192.168.122.17
nginxrs2 192.168.122.17 
[root@client ~]# curl 192.168.122.7
nginxrs1 192.168.122.7 
[root@client ~]# curl 192.168.122.100
curl: (7) Failed to connect to 192.168.122.100 port 80: 拒绝连接

三层网络是没有问题的。

检查内核的信息

[root@nginxrs1 ~]# for i in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 0 > $i; done
# 然后再验证一下
sysctl -a | grep rp_filter
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.enp1s0.arp_filter = 0
net.ipv4.conf.enp1s0.rp_filter = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.rp_filter = 0
net.ipv4.conf.tunl0.arp_filter = 0
net.ipv4.conf.tunl0.rp_filter = 0

[root@nginxrs2 ~]# for i in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 0 > $i; done
# 然后再验证一下
sysctl -a | grep rp_filter
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.enp1s0.arp_filter = 0
net.ipv4.conf.enp1s0.rp_filter = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.rp_filter = 0
net.ipv4.conf.tunl0.arp_filter = 0
net.ipv4.conf.tunl0.rp_filter = 0

检查nginx到底在监听什么？

[root@nginxrs2 ~]# ss -ntlp | grep :80
LISTEN 0      511          0.0.0.0:80        0.0.0.0:*    users:(("nginx",pid=914,fd=6),("nginx",pid=913,fd=6),("nginx",pid=912,fd=6))
LISTEN 0      511             [::]:80           [::]:*    users:(("nginx",pid=914,fd=7),("nginx",pid=913,fd=7),("nginx",pid=912,fd=7))

[root@nginxrs1 ~]# ss -ntlp | grep :80
LISTEN 0      511          0.0.0.0:80        0.0.0.0:*    users:(("nginx",pid=931,fd=6),("nginx",pid=930,fd=6),("nginx",pid=929,fd=6))
LISTEN 0      511             [::]:80           [::]:*    users:(("nginx",pid=931,fd=7),("nginx",pid=930,fd=7),("nginx",pid=929,fd=7))

检查rs的tunl0网卡

[root@nginxrs2 ~]# ip addr show 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp1s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 52:54:00:d9:4a:e0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.17/24 brd 192.168.122.255 scope global noprefixroute enp1s0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:fed9:4ae0/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    inet 192.168.122.100/32 scope global tunl0
       valid_lft forever preferred_lft forever
[root@nginxrs1 ~]# ip addr show 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp1s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 52:54:00:e8:1a:c6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.7/24 brd 192.168.122.255 scope global noprefixroute enp1s0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:fee8:1ac6/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    inet 192.168.122.100/32 scope global tunl0
       valid_lft forever preferred_lft forever

定位问题：

3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    inet 192.168.122.100/32 scope global tunl0
       valid_lft forever preferred_lft forever

tunlo网卡的状态是DOWN ,这意味着虽然包到了rs但是解包到三层网络层时，linux发现请求包的目的ip在tunlo网卡上，但是tunlo网卡的状态时down,于是就直接丢包了。

p link set tunl0 up

在`client`上进行测试


[root@client ~]# curl 192.168.122.100
curl: (7) Failed to connect to 192.168.122.100 port 80: 拒绝连接
[root@client ~]# curl 192.168.122.100
curl: (7) Failed to connect to 192.168.122.100 port 80: 拒绝连接
[root@client ~]# curl 192.168.122.100
nginxrs2 192.168.122.17 
[root@client ~]# curl 192.168.122.100
nginxrs1 192.168.122.7 
[root@client ~]# curl 192.168.122.100
nginxrs2 192.168.122.17 
[root@client ~]# curl 192.168.122.100
nginxrs1 192.168.122.7 
[root@client ~]# curl 192.168.122.100
nginxrs2 192.168.122.17 
[root@client ~]# curl 192.168.122.100
nginxrs1 192.168.122.7 
[root@client ~]# curl 192.168.122.100
nginxrs2 192.168.122.17 
[root@client ~]# curl 192.168.122.100
nginxrs1 192.168.122.7