Docker 容器跨主机通信 overlay
一.Overlay网络概述
Overlay网络是指在不改变现有网络基础设施的前提下，通过某种约定通信协议，把二层报文封装在IP报文之上的新的数据格式。Overlay网络采用VXLAN（Virtual Extensible LAN）技术创建一个虚拟网络，将不同主机上的容器连接到同一个逻辑网络中。这样，容器就可以像在同一台主机上一样进行通信，而无需关心底层的网络细节。
二.Overlay网络的优势
跨主机通信：Overlay网络允许在不同主机上的容器之间进行通信，打破了主机之间的隔离。
扩展性：Overlay网络可以支持大量的容器和主机，满足大规模容器化部署的需求。
隔离性：通过Overlay网络，可以为不同的容器提供独立的网络环境，避免网络冲突和干扰。
灵活性：Overlay网络支持动态添加和删除容器，无需重新配置网络。
三、实现Overlay网络的步骤
1.准备环境
dockerfile
docker01 192.168.73.128 ens33 centos7
docker02 192.168.73.129 ens33 centos72.初始化一个swarm集群
dockerfile
# 初始化swarm集群
[root@docker01 ~]# docker swarm init --advertise-addr 192.168.73.128
Swarm initialized: current node (9fn9iyxhkxvjey06lwhgv7zhb) is now a manager.
To add a worker to this swarm, run the following command:
docker swarm join --token SWMTKN-1-1rqzq4jhf4wm77e8yn6s347sd189u84mr0u042kwxxv05n8bbx-1rv4ae95p4e4y6aztlhkvqtfx 192.168.73.128:2377
To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
# 查看集群
[root@docker01 ~]# docker node ls
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
9fn9iyxhkxvjey06lwhgv7zhb * docker01 Ready Active Leader 26.1.4
# 查看网络
[root@docker01 ~]# docker network ls
NETWORK ID NAME DRIVER SCOPE
6ad75bb1b055 bridge bridge local
f12f6a0f8bbf docker_gwbridge bridge local
b1c70e3e1ded host host local
116fnum2qea1 ingress overlay swarm
e9eddadcf473 none null local3.创建 overlay network
dockerfile
# 创建ovelay，自定义子网和网关，不输入系统会自动生成
[root@docker01 ~]# docker network create -d overlay --subnet=192.168.100.0/24 --gateway=192.168.100.1 --attachable my-overlay
ue8rewtwd72difwr86gi5wwsl
# ---attachable：允许集群服务间的容器交互连接或者独立的容器之间能够连接。swarm在设计之初是为了service(一组container)而服务的，因此通过swarm创建的overlay网络在一开始并不支持单独的container加入其中。但是在docker1.13, 我们可以通过"--attach" 参数声明当前创建的overlay网络可以被container直接加入。
[root@docker01 ~]# docker network inspect my-overlay
[
{
"Name": "my-overlay",
"Id": "ue8rewtwd72difwr86gi5wwsl",
"Created": "2024-10-12T06:04:04.784641012Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "192.168.100.0/24", # 自定义子网
"Gateway": "192.168.100.1" # 自定义网关
}
]
},
"Internal": false,
"Attachable": true,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": null,
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4098"
},
"Labels": null
}
]4.将docker02加入集群
dockerfile
# 获取加入集群命令
[root@docker01 ~]# docker swarm join-token worker
To add a worker to this swarm, run the following command:
docker swarm join --token SWMTKN-1-1rqzq4jhf4wm77e8yn6s347sd189u84mr0u042kwxxv05n8bbx-1rv4ae95p4e4y6aztlhkvqtfx 192.168.73.128:2377
# 在docker02上执行
[root@docker02 ~]# docker swarm join --token SWMTKN-1-1rqzq4jhf4wm77e8yn6s347sd189u84mr0u042kwxxv05n8bbx-1rv4ae95p4e4y6aztlhkvqtfx 192.168.73.128:2377
This node joined a swarm as a worker.
# docker01 上查询是否加入成功
[root@docker01 ~]# docker node ls
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
9fn9iyxhkxvjey06lwhgv7zhb * docker01 Ready Active Leader 26.1.4
4m1xin9edttdt1ljxtzy3q2ab docker02 Ready Active 26.1.45.docker01创建容器
dockerfile
[root@docker01 ~]# docker run -d --name=busybox1 --network=my-overlay harbor.linux.com/k8s/busybox:latest /bin/sleep 3600
d8b2749823f570d6c08eccf2df369774323a72628e18e09bc9dfb09735bd00f0
# 查询容器ip
[root@docker01 ~]# docker exec -it busybox1 /bin/sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
14: eth0@if15: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue
link/ether 02:42:c0:a8:64:02 brd ff:ff:ff:ff:ff:ff
inet 192.168.100.2/24 brd 192.168.100.255 scope global eth0
valid_lft forever preferred_lft forever
16: eth1@if17: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff
inet 172.18.0.3/16 brd 172.18.255.255 scope global eth1
valid_lft forever preferred_lft forever
# 查看网络，可以发现有了容器的IP及LB IP
[root@docker01 ~]# docker network inspect my-overlay
[
{
"Name": "my-overlay",
"Id": "ue8rewtwd72difwr86gi5wwsl",
"Created": "2024-10-12T14:15:21.812458427+08:00",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "192.168.100.0/24",
"Gateway": "192.168.100.1"
}
]
},
"Internal": false,
"Attachable": true,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"d8b2749823f570d6c08eccf2df369774323a72628e18e09bc9dfb09735bd00f0": {
"Name": "busybox1",
"EndpointID": "225db615a5d654cd813171ed4d07c7f485b362551507dd18125a4405c45b18fb",
"MacAddress": "02:42:c0:a8:64:02",
"IPv4Address": "192.168.100.2/24",
"IPv6Address": ""
},
"lb-my-overlay": {
"Name": "my-overlay-endpoint",
"EndpointID": "cce907ae37301e39dbd55abcba03e50e562f589f7d08e6c57ace8891d4785747",
"MacAddress": "02:42:c0:a8:64:03",
"IPv4Address": "192.168.100.3/24",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4098"
},
"Labels": {},
"Peers": [
{
"Name": "9fc55aae1502",
"IP": "192.168.73.128"
}
]
}
]6.docker02创建容器
dockerfile
[root@docker02 ~]# docker run -d --name=busybox2 --network=my-overlay harbor.linux.com/k8s/busybox:latest /bin/sleep 3600
3dd7b088c68f0247d312070bc63e15a697d5e8aa198f7bf53450ea96843cc41b
# 查询容器ip
[root@docker02 ~]# docker exec -it busybox2 /bin/sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
13: eth0@if14: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue
link/ether 02:42:c0:a8:64:04 brd ff:ff:ff:ff:ff:ff
inet 192.168.100.4/24 brd 192.168.100.255 scope global eth0
valid_lft forever preferred_lft forever
15: eth1@if16: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff
inet 172.18.0.3/16 brd 172.18.255.255 scope global eth1
valid_lft forever preferred_lft forever
# 查看网络，可以发现有了容器的IP及LB IP
[root@docker02 ~]# docker network inspect my-overlay
[
{
"Name": "my-overlay",
"Id": "ue8rewtwd72difwr86gi5wwsl",
"Created": "2024-10-12T14:21:11.23116209+08:00",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "192.168.100.0/24",
"Gateway": "192.168.100.1"
}
]
},
"Internal": false,
"Attachable": true,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"3dd7b088c68f0247d312070bc63e15a697d5e8aa198f7bf53450ea96843cc41b": {
"Name": "busybox2",
"EndpointID": "e0b40de6b74e610d1e13318f8420fcaa5b7e8b9bcd9c68f6faaa9931ea72f992",
"MacAddress": "02:42:c0:a8:64:04",
"IPv4Address": "192.168.100.4/24",
"IPv6Address": ""
},
"lb-my-overlay": {
"Name": "my-overlay-endpoint",
"EndpointID": "4fe0523b017be80ce24bddd8b034633f998e36630b100d3a0468ce02ba4e533d",
"MacAddress": "02:42:c0:a8:64:05",
"IPv4Address": "192.168.100.5/24",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4098"
},
"Labels": {},
"Peers": [
{
"Name": "9fc55aae1502",
"IP": "192.168.73.128"
},
{
"Name": "473174a88eb9",
"IP": "192.168.73.129"
}
]
}
] 7.在容器内部互相ping对方
dockerfile
# docker01 ping内外网测试
[root@docker01 ~]# docker exec -it busybox1 /bin/sh
/ # ping 192.168.100.4
PING 192.168.100.4 (192.168.100.4): 56 data bytes
64 bytes from 192.168.100.4: seq=0 ttl=64 time=1.033 ms
64 bytes from 192.168.100.4: seq=1 ttl=64 time=1.702 ms
64 bytes from 192.168.100.4: seq=2 ttl=64 time=2.209 ms
^C
--- 192.168.100.4 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 1.033/1.648/2.209 ms
/ # ping 192.168.100.5
PING 192.168.100.5 (192.168.100.5): 56 data bytes
64 bytes from 192.168.100.5: seq=0 ttl=64 time=0.897 ms
64 bytes from 192.168.100.5: seq=1 ttl=64 time=1.988 ms
64 bytes from 192.168.100.5: seq=2 ttl=64 time=0.893 ms
^C
--- 192.168.100.5 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.893/1.259/1.988 ms
/ # ping www.baidu.com
PING www.baidu.com (182.61.200.7): 56 data bytes
64 bytes from 182.61.200.7: seq=0 ttl=127 time=21.864 ms
64 bytes from 182.61.200.7: seq=1 ttl=127 time=23.486 ms
64 bytes from 182.61.200.7: seq=2 ttl=127 time=23.225 ms
^C
--- www.baidu.com ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 21.864/22.858/23.486 ms
# docker02 ping内外网测试
[root@docker02 ~]# docker exec -it busybox2 /bin/sh
/ # ping 192.168.100.2
PING 192.168.100.2 (192.168.100.2): 56 data bytes
64 bytes from 192.168.100.2: seq=0 ttl=64 time=1.001 ms
64 bytes from 192.168.100.2: seq=1 ttl=64 time=1.374 ms
64 bytes from 192.168.100.2: seq=2 ttl=64 time=1.990 ms
^C
--- 192.168.100.2 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 1.001/1.455/1.990 ms
/ # ping 192.168.100.3
PING 192.168.100.3 (192.168.100.3): 56 data bytes
64 bytes from 192.168.100.3: seq=0 ttl=64 time=0.572 ms
64 bytes from 192.168.100.3: seq=1 ttl=64 time=0.999 ms
64 bytes from 192.168.100.3: seq=2 ttl=64 time=1.199 ms
^C
--- 192.168.100.3 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.572/0.923/1.199 ms
/ # ping www.baidu.com
PING www.baidu.com (182.61.200.6): 56 data bytes
64 bytes from 182.61.200.6: seq=0 ttl=127 time=22.134 ms
64 bytes from 182.61.200.6: seq=1 ttl=127 time=22.375 ms
64 bytes from 182.61.200.6: seq=2 ttl=127 time=22.372 ms
^C
--- www.baidu.com ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 22.134/22.293/22.375 ms
# Overlay网络ping通内外网的原理
# 内部网络通信：
1.在Overlay网络中，虚拟机之间的通信通过虚拟隧道进行。每个虚拟机都有一个唯一的IP地址，这个地址在Overlay网络中有效。
2.当一个虚拟机需要向另一个虚拟机发送数据时，数据会被封装在Overlay报文中，并通过虚拟隧道传输到目标虚拟机。
3.由于Overlay网络是在物理网络之上构建的，因此虚拟机之间的通信不会受到物理网络细节的影响。
# 外部网络通信：
1.Overlay网络通常与物理网络相连，以便虚拟机能够访问外部网络。
2.在这种情况下，虚拟机发出的数据会被封装在Overlay报文中，并通过物理网络的网关或路由器发送到外部网络。
3.外部网络收到的数据会被解封装，并根据目标IP地址进行路由和转发。
VXLAN（Virtual eXtensible Local Area Network）是一种网络虚拟化技术，它使用隧道协议将二层以太网帧封装在三层IP报文中，从而实现跨物理网络的二层连接。这里的vxlan:vethXXX表示通过VXLAN隧道连接的虚拟以太网接口，它可能用于将容器的流量封装并通过宿主机发送到外部网络或其他容器。vxlan才是实体，overlay只是网络模型；一句话就是vxlan是overlay网络的一种实现。