Kubernetes部署v1.17.4

原创已于 2023-05-18 16:18:56 修改 · 613 阅读

1 ·

本内容遵循CC 4.0 BY-SA版权协议

标签

#kubernetes

于 2023-01-06 11:37:10 首次发布

k8s 专栏收录该内容

4 篇文章

订阅专栏

一、准备环境

角色	IP地址	组件
master	192.168.0.10	docker，kubectl，kubeadm，kubelet
node01	192.168.0.11	docker，kubectl，kubeadm，kubelet
node02	192.168.0.12	docker，kubectl，kubeadm，kubelet

二、部署搭建（master和node都要初始化，我这里举例master）

1 检查操作系统的版本

# 安装kubernetes集群要求Centos版本要在7.5或之上
[root@master ~]# cat /etc/redhat-release
CentOS Linux release 7.6.1810 (Core)

2 主机名解析

[root@master ~]# vi /etc/hosts
::1     localhost       localhost.localdomain   localhost6      localhost6.localdomain6
127.0.0.1       localhost       localhost.localdomain   localhost4      localhost4.localdomain4
127.0.0.1       ecs-master-0001 ecs-master-0001
192.168.0.10    master
192.168.0.11    node01
192.168.0.12    node02

3 时间同步

# 启动chronyd服务
[root@master ~]# systemctl enable --now chronyd
[root@master ~]# date
Fri Jan  6 09:27:09 CST 2023

4 停止iptable和firewalld服务

[root@master ~]# systemctl disable --now firewalld
[root@master ~]# systemctl disable --now iptables
Failed to execute operation: No such file or directory
[root@master ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
   Active: inactive (dead)
     Docs: man:firewalld(1)
[root@master ~]# systemctl status iptables
Unit iptables.service could not be found.

5 停止SeLinux

# 改成SELINUX=disabled
[root@master ~]# vi /etc/selinux/config 
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted
[root@master ~]# setenforce 0

6 禁用swap分区

# 注释掉/dev/mapper/centos-swap这一行
[root@master ~]# vi /etc/fstab 
#
# /etc/fstab
# Created by anaconda on Thu Feb 10 08:57:10 2022
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
UUID=4fde6d8c-b0bb-4d16-a95f-c578b5f9c2a6 /                       ext4    defaults        1 1
# /dev/mapper/centos-swap                swap                     swap    defaults        0 0

7 修改linux的内核参数

# 添加网桥过滤和地址转发功能
[root@master ~]# vi /etc/sysctl.d/kubernetes.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
[root@master ~]# sysctl -p
vm.swappiness = 0
net.core.somaxconn = 1024
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_max_syn_backlog = 1024
[root@master ~]# modprobe br_netfilter
[root@master ~]# lsmod | grep br_netfilter
br_netfilter           22256  0 
bridge                151336  1 br_netfilter

8 配置ipvs功能

# 安装ipset和ipvadm
[root@master ~]# yum -y install ipset ipvsadm
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
Package ipset-7.1-1.el7.x86_64 already installed and latest version
Resolving Dependencies
--> Running transaction check
---> Package ipvsadm.x86_64 0:1.27-8.el7 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

=============================================================================================================================================
 Package                          Arch                            Version                                Repository                     Size
=============================================================================================================================================
Installing:
 ipvsadm                          x86_64                          1.27-8.el7                             base                           45 k

Transaction Summary
=============================================================================================================================================
Install  1 Package

Total download size: 45 k
Installed size: 75 k
Downloading packages:
ipvsadm-1.27-8.el7.x86_64.rpm                                                                                         |  45 kB  00:00:00     
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Warning: RPMDB altered outside of yum.
  Installing : ipvsadm-1.27-8.el7.x86_64                                                                                                 1/1 
  Verifying  : ipvsadm-1.27-8.el7.x86_64                                                                                                 1/1 

Installed:
  ipvsadm.x86_64 0:1.27-8.el7                                                                                                                

Complete!

# 添加写入脚本文件
[root@master ~]# cat <<EOF> /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
[root@master ~]# ls /etc/sysconfig/modules/
ipvs.modules

# 添加权限执行脚本
[root@master ~]# chmod +x /etc/sysconfig/modules/ipvs.modules
[root@master ~]# ll /etc/sysconfig/modules/ipvs.modules 
-rwxr-xr-x 1 root root 124 Jan  5 16:18 /etc/sysconfig/modules/ipvs.modules
[root@master ~]# sh /etc/sysconfig/modules/ipvs.modules
[root@master ~]# lsmod | grep -e ip_vs -e nf_conntrack_ipv4
nf_conntrack_ipv4      15053  0 
nf_defrag_ipv4         12729  1 nf_conntrack_ipv4
ip_vs_sh               12688  0 
ip_vs_wrr              12697  0 
ip_vs_rr               12600  0 
ip_vs                 145458  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          139264  2 ip_vs,nf_conntrack_ipv4
libcrc32c              12644  2 ip_vs,nf_conntrack

9 重启

[root@master ~]# reboot

10 安装docker

# 下载yum源
[root@master ~]# wget https://repo.huaweicloud.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
--2023-01-06 09:30:20--  https://repo.huaweicloud.com/docker-ce/linux/centos/docker-ce.repo
Resolving repo.huaweicloud.com (repo.huaweicloud.com)... 123.125.16.221, 123.125.16.224, 123.125.16.226, ...
Connecting to repo.huaweicloud.com (repo.huaweicloud.com)|123.125.16.221|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1919 (1.9K) [application/octet-stream]
Saving to: ‘/etc/yum.repos.d/docker-ce.repo’

100%[===================================================================================================>] 1,919       --.-K/s   in 0s      

2023-01-06 09:30:20 (423 MB/s) - ‘/etc/yum.repos.d/docker-ce.repo’ saved [1919/1919]

[root@master yum.repos.d]# cd /etc/yum.repos.d/
[root@master yum.repos.d]# ls
CentOS-Base.repo  CentOS-Debuginfo.repo  CentOS-Media.repo    CentOS-Vault.repo  epel.repo         epel-testing.repo
CentOS-CR.repo    CentOS-fasttrack.repo  CentOS-Sources.repo  docker-ce.repo     epel.repo.rpmnew

# 安装docker-ce
[root@master yum.repos.d]# yum -y install docker-ce
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
docker-ce-stable                                                                                                      | 3.5 kB  00:00:00     
(1/2): docker-ce-stable/7/x86_64/primary_db                                                                           |  91 kB  00:00:00     
(2/2): docker-ce-stable/7/x86_64/updateinfo                                                                           |   55 B  00:00:00     
Resolving Dependencies
--> Running transaction check
---> Package docker-ce.x86_64 3:20.10.22-3.el7 will be installed
--> Processing Dependency: container-selinux >= 2:2.74 for package: 3:docker-ce-20.10.22-3.el7.x86_64
--> Processing Dependency: containerd.io >= 1.4.1 for package: 3:docker-ce-20.10.22-3.el7.x86_64
--> Processing Dependency: docker-ce-cli for package: 3:docker-ce-20.10.22-3.el7.x86_64
--> Processing Dependency: docker-ce-rootless-extras for package: 3:docker-ce-20.10.22-3.el7.x86_64
--> Processing Dependency: libcgroup for package: 3:docker-ce-20.10.22-3.el7.x86_64
--> Running transaction check
---> Package container-selinux.noarch 2:2.119.2-1.911c772.el7_8 will be installed
--> Processing Dependency: policycoreutils-python for package: 2:container-selinux-2.119.2-1.911c772.el7_8.noarch
---> Package containerd.io.x86_64 0:1.6.14-3.1.el7 will be installed
---> Package docker-ce-cli.x86_64 1:20.10.22-3.el7 will be installed
--> Processing Dependency: docker-scan-plugin(x86-64) for package: 1:docker-ce-cli-20.10.22-3.el7.x86_64
---> Package docker-ce-rootless-extras.x86_64 0:20.10.22-3.el7 will be installed
--> Processing Dependency: fuse-overlayfs >= 0.7 for package: docker-ce-rootless-extras-20.10.22-3.el7.x86_64
--> Processing Dependency: slirp4netns >= 0.4 for package: docker-ce-rootless-extras-20.10.22-3.el7.x86_64
---> Package libcgroup.x86_64 0:0.41-21.el7 will be installed
--> Running transaction check
---> Package docker-scan-plugin.x86_64 0:0.23.0-3.el7 will be installed
---> Package fuse-overlayfs.x86_64 0:0.7.2-6.el7_8 will be installed
--> Processing Dependency: libfuse3.so.3(FUSE_3.2)(64bit) for package: fuse-overlayfs-0.7.2-6.el7_8.x86_64
--> Processing Dependency: libfuse3.so.3(FUSE_3.0)(64bit) for package: fuse-overlayfs-0.7.2-6.el7_8.x86_64
--> Processing Dependency: libfuse3.so.3()(64bit) for package: fuse-overlayfs-0.7.2-6.el7_8.x86_64
---> Package policycoreutils-python.x86_64 0:2.5-34.el7 will be installed
--> Processing Dependency: setools-libs >= 3.3.8-4 for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: libsemanage-python >= 2.5-14 for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: audit-libs-python >= 2.1.3-4 for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: python-IPy for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: libqpol.so.1(VERS_1.4)(64bit) for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: libqpol.so.1(VERS_1.2)(64bit) for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: libapol.so.4(VERS_4.0)(64bit) for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: checkpolicy for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: libqpol.so.1()(64bit) for package: policycoreutils-python-2.5-34.el7.x86_64
--> Processing Dependency: libapol.so.4()(64bit) for package: policycoreutils-python-2.5-34.el7.x86_64
---> Package slirp4netns.x86_64 0:0.4.3-4.el7_8 will be installed
--> Running transaction check
---> Package audit-libs-python.x86_64 0:2.8.5-4.el7 will be installed
---> Package checkpolicy.x86_64 0:2.5-8.el7 will be installed
---> Package fuse3-libs.x86_64 0:3.6.1-4.el7 will be installed
---> Package libsemanage-python.x86_64 0:2.5-14.el7 will be installed
---> Package python-IPy.noarch 0:0.75-6.el7 will be installed
---> Package setools-libs.x86_64 0:3.3.8-4.el7 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

=============================================================================================================================================
 Package                                 Arch                 Version                                   Repository                      Size
=============================================================================================================================================
Installing:
 docker-ce                               x86_64               3:20.10.22-3.el7                          docker-ce-stable                22 M
Installing for dependencies:
 audit-libs-python                       x86_64               2.8.5-4.el7                               base                            76 k
 checkpolicy                             x86_64               2.5-8.el7                                 base                           295 k
 container-selinux                       noarch               2:2.119.2-1.911c772.el7_8                 extras                          40 k
 containerd.io                           x86_64               1.6.14-3.1.el7                            docker-ce-stable                33 M
 docker-ce-cli                           x86_64               1:20.10.22-3.el7                          docker-ce-stable                30 M
 docker-ce-rootless-extras               x86_64               20.10.22-3.el7                            docker-ce-stable               8.5 M
 docker-scan-plugin                      x86_64               0.23.0-3.el7                              docker-ce-stable               3.8 M
 fuse-overlayfs                          x86_64               0.7.2-6.el7_8                             extras                          54 k
 fuse3-libs                              x86_64               3.6.1-4.el7                               extras                          82 k
 libcgroup                               x86_64               0.41-21.el7                               base                            66 k
 libsemanage-python                      x86_64               2.5-14.el7                                base                           113 k
 policycoreutils-python                  x86_64               2.5-34.el7                                base                           457 k
 python-IPy                              noarch               0.75-6.el7                                base                            32 k
 setools-libs                            x86_64               3.3.8-4.el7                               base                           620 k
 slirp4netns                             x86_64               0.4.3-4.el7_8                             extras                          81 k

Transaction Summary
=============================================================================================================================================
Install  1 Package (+15 Dependent packages)

Total download size: 98 M
Installed size: 361 M
Downloading packages:
(1/16): audit-libs-python-2.8.5-4.el7.x86_64.rpm                                                                      |  76 kB  00:00:00     
(2/16): checkpolicy-2.5-8.el7.x86_64.rpm                                                                              | 295 kB  00:00:00     
(3/16): container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm                                                          |  40 kB  00:00:00     
warning: /var/cache/yum/x86_64/7/docker-ce-stable/packages/containerd.io-1.6.14-3.1.el7.x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID 621e9f35: NOKEY
Public key for containerd.io-1.6.14-3.1.el7.x86_64.rpm is not installed
(4/16): containerd.io-1.6.14-3.1.el7.x86_64.rpm                                                                       |  33 MB  00:00:13     
(5/16): docker-ce-cli-20.10.22-3.el7.x86_64.rpm                                                                       |  30 MB  00:00:27     
(6/16): docker-ce-rootless-extras-20.10.22-3.el7.x86_64.rpm                                                           | 8.5 MB  00:00:07     
(7/16): fuse-overlayfs-0.7.2-6.el7_8.x86_64.rpm                                                                       |  54 kB  00:00:00     
(8/16): libsemanage-python-2.5-14.el7.x86_64.rpm                                                                      | 113 kB  00:00:00     
(9/16): policycoreutils-python-2.5-34.el7.x86_64.rpm                                                                  | 457 kB  00:00:00     
(10/16): python-IPy-0.75-6.el7.noarch.rpm                                                                             |  32 kB  00:00:00     
(11/16): setools-libs-3.3.8-4.el7.x86_64.rpm                                                                          | 620 kB  00:00:00     
(12/16): slirp4netns-0.4.3-4.el7_8.x86_64.rpm                                                                         |  81 kB  00:00:00     
(13/16): libcgroup-0.41-21.el7.x86_64.rpm                                                                             |  66 kB  00:00:00     
(14/16): fuse3-libs-3.6.1-4.el7.x86_64.rpm                                                                            |  82 kB  00:00:00     
(15/16): docker-scan-plugin-0.23.0-3.el7.x86_64.rpm                                                                   | 3.8 MB  00:00:04     
(16/16): docker-ce-20.10.22-3.el7.x86_64.rpm                                                                          |  22 MB  00:00:59     
---------------------------------------------------------------------------------------------------------------------------------------------
Total                                                                                                        1.6 MB/s |  98 MB  00:00:59     
Retrieving key from https://download.docker.com/linux/centos/gpg
Importing GPG key 0x621E9F35:
 Userid     : "Docker Release (CE rpm) <docker@docker.com>"
 Fingerprint: 060a 61c5 1b55 8a7f 742b 77aa c52f eb6b 621e 9f35
 From       : https://download.docker.com/linux/centos/gpg
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
  Installing : libcgroup-0.41-21.el7.x86_64                                                                                             1/16 
  Installing : setools-libs-3.3.8-4.el7.x86_64                                                                                          2/16 
  Installing : audit-libs-python-2.8.5-4.el7.x86_64                                                                                     3/16 
  Installing : fuse3-libs-3.6.1-4.el7.x86_64                                                                                            4/16 
  Installing : fuse-overlayfs-0.7.2-6.el7_8.x86_64                                                                                      5/16 
  Installing : checkpolicy-2.5-8.el7.x86_64                                                                                             6/16 
  Installing : python-IPy-0.75-6.el7.noarch                                                                                             7/16 
  Installing : slirp4netns-0.4.3-4.el7_8.x86_64                                                                                         8/16 
  Installing : libsemanage-python-2.5-14.el7.x86_64                                                                                     9/16 
  Installing : policycoreutils-python-2.5-34.el7.x86_64                                                                                10/16 
  Installing : 2:container-selinux-2.119.2-1.911c772.el7_8.noarch                                                                      11/16 
setsebool:  SELinux is disabled.
  Installing : containerd.io-1.6.14-3.1.el7.x86_64                                                                                     12/16 
  Installing : docker-scan-plugin-0.23.0-3.el7.x86_64                                                                                  13/16 
  Installing : 1:docker-ce-cli-20.10.22-3.el7.x86_64                                                                                   14/16 
  Installing : docker-ce-rootless-extras-20.10.22-3.el7.x86_64                                                                         15/16 
  Installing : 3:docker-ce-20.10.22-3.el7.x86_64                                                                                       16/16 
  Verifying  : 3:docker-ce-20.10.22-3.el7.x86_64                                                                                        1/16 
  Verifying  : docker-scan-plugin-0.23.0-3.el7.x86_64                                                                                   2/16 
  Verifying  : docker-ce-rootless-extras-20.10.22-3.el7.x86_64                                                                          3/16 
  Verifying  : fuse-overlayfs-0.7.2-6.el7_8.x86_64                                                                                      4/16 
  Verifying  : libsemanage-python-2.5-14.el7.x86_64                                                                                     5/16 
  Verifying  : slirp4netns-0.4.3-4.el7_8.x86_64                                                                                         6/16 
  Verifying  : 2:container-selinux-2.119.2-1.911c772.el7_8.noarch                                                                       7/16 
  Verifying  : 1:docker-ce-cli-20.10.22-3.el7.x86_64                                                                                    8/16 
  Verifying  : python-IPy-0.75-6.el7.noarch                                                                                             9/16 
  Verifying  : checkpolicy-2.5-8.el7.x86_64                                                                                            10/16 
  Verifying  : containerd.io-1.6.14-3.1.el7.x86_64                                                                                     11/16 
  Verifying  : policycoreutils-python-2.5-34.el7.x86_64                                                                                12/16 
  Verifying  : fuse3-libs-3.6.1-4.el7.x86_64                                                                                           13/16 
  Verifying  : audit-libs-python-2.8.5-4.el7.x86_64                                                                                    14/16 
  Verifying  : setools-libs-3.3.8-4.el7.x86_64                                                                                         15/16 
  Verifying  : libcgroup-0.41-21.el7.x86_64                                                                                            16/16 

Installed:
  docker-ce.x86_64 3:20.10.22-3.el7                                                                                                          

Dependency Installed:
  audit-libs-python.x86_64 0:2.8.5-4.el7      checkpolicy.x86_64 0:2.5-8.el7            container-selinux.noarch 2:2.119.2-1.911c772.el7_8   
  containerd.io.x86_64 0:1.6.14-3.1.el7       docker-ce-cli.x86_64 1:20.10.22-3.el7     docker-ce-rootless-extras.x86_64 0:20.10.22-3.el7    
  docker-scan-plugin.x86_64 0:0.23.0-3.el7    fuse-overlayfs.x86_64 0:0.7.2-6.el7_8     fuse3-libs.x86_64 0:3.6.1-4.el7                      
  libcgroup.x86_64 0:0.41-21.el7              libsemanage-python.x86_64 0:2.5-14.el7    policycoreutils-python.x86_64 0:2.5-34.el7           
  python-IPy.noarch 0:0.75-6.el7              setools-libs.x86_64 0:3.3.8-4.el7         slirp4netns.x86_64 0:0.4.3-4.el7_8                   

Complete!

# 配置镜像加速器
[root@master yum.repos.d]# mkdir /etc/docker
[root@master yum.repos.d]# cat <<EOF> /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://kn0t2bca.mirror.aliyuncs.com"]
}
EOF

# 启动docker
[root@master yum.repos.d]# systemctl enable --now docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.

11 安装Kubernetes组件

# 配置kubernetes yum源(这里实验就17版本)
[root@master ~]# vi /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://repo.huaweicloud.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgchech=0
repo_gpgcheck=0
gpgkey=https://repo.huaweicloud.com/kubernetes/yum/doc/yum-key.gpg
                        https://repo.huaweicloud.com/kubernetes/yum/doc/rpm-package-key.gpg

# 安装kubernetes
[root@master ~]# yum install -y kubeadm-1.17.4-0 kubelet-1.17.4-0 kubectl-1.17.4-0
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
kubernetes                                                                                                            | 1.4 kB  00:00:00     
kubernetes/primary                                                                                                    | 102 kB  00:00:00     
kubernetes                                                                                                                           751/751
Resolving Dependencies
--> Running transaction check
---> Package kubeadm.x86_64 0:1.17.4-0 will be installed
--> Processing Dependency: kubernetes-cni >= 0.7.5 for package: kubeadm-1.17.4-0.x86_64
--> Processing Dependency: cri-tools >= 1.13.0 for package: kubeadm-1.17.4-0.x86_64
---> Package kubectl.x86_64 0:1.17.4-0 will be installed
---> Package kubelet.x86_64 0:1.17.4-0 will be installed
--> Processing Dependency: socat for package: kubelet-1.17.4-0.x86_64
--> Processing Dependency: conntrack for package: kubelet-1.17.4-0.x86_64
--> Running transaction check
---> Package conntrack-tools.x86_64 0:1.4.4-7.el7 will be installed
--> Processing Dependency: libnetfilter_cttimeout.so.1(LIBNETFILTER_CTTIMEOUT_1.1)(64bit) for package: conntrack-tools-1.4.4-7.el7.x86_64
--> Processing Dependency: libnetfilter_cttimeout.so.1(LIBNETFILTER_CTTIMEOUT_1.0)(64bit) for package: conntrack-tools-1.4.4-7.el7.x86_64
--> Processing Dependency: libnetfilter_cthelper.so.0(LIBNETFILTER_CTHELPER_1.0)(64bit) for package: conntrack-tools-1.4.4-7.el7.x86_64
--> Processing Dependency: libnetfilter_queue.so.1()(64bit) for package: conntrack-tools-1.4.4-7.el7.x86_64
--> Processing Dependency: libnetfilter_cttimeout.so.1()(64bit) for package: conntrack-tools-1.4.4-7.el7.x86_64
--> Processing Dependency: libnetfilter_cthelper.so.0()(64bit) for package: conntrack-tools-1.4.4-7.el7.x86_64
---> Package cri-tools.x86_64 0:1.19.0-0 will be installed
---> Package kubernetes-cni.x86_64 0:0.8.7-0 will be installed
---> Package socat.x86_64 0:1.7.3.2-2.el7 will be installed
--> Running transaction check
---> Package libnetfilter_cthelper.x86_64 0:1.0.0-11.el7 will be installed
---> Package libnetfilter_cttimeout.x86_64 0:1.0.0-7.el7 will be installed
---> Package libnetfilter_queue.x86_64 0:1.0.2-2.el7_2 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

=============================================================================================================================================
 Package                                   Arch                      Version                             Repository                     Size
=============================================================================================================================================
Installing:
 kubeadm                                   x86_64                    1.17.4-0                            kubernetes                    8.7 M
 kubectl                                   x86_64                    1.17.4-0                            kubernetes                    9.4 M
 kubelet                                   x86_64                    1.17.4-0                            kubernetes                     20 M
Installing for dependencies:
 conntrack-tools                           x86_64                    1.4.4-7.el7                         base                          187 k
 cri-tools                                 x86_64                    1.19.0-0                            kubernetes                    5.7 M
 kubernetes-cni                            x86_64                    0.8.7-0                             kubernetes                     19 M
 libnetfilter_cthelper                     x86_64                    1.0.0-11.el7                        base                           18 k
 libnetfilter_cttimeout                    x86_64                    1.0.0-7.el7                         base                           18 k
 libnetfilter_queue                        x86_64                    1.0.2-2.el7_2                       base                           23 k
 socat                                     x86_64                    1.7.3.2-2.el7                       base                          290 k

Transaction Summary
=============================================================================================================================================
Install  3 Packages (+7 Dependent packages)

Total download size: 63 M
Installed size: 266 M
Downloading packages:
(1/10): conntrack-tools-1.4.4-7.el7.x86_64.rpm                                                                        | 187 kB  00:00:00     
warning: /var/cache/yum/x86_64/7/kubernetes/packages/67ffa375b03cea72703fe446ff00963919e8fce913fbc4bb86f06d1475a6bdf9-cri-tools-1.19.0-0.x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID 3e1ba8d5: NOKEY
Public key for 67ffa375b03cea72703fe446ff00963919e8fce913fbc4bb86f06d1475a6bdf9-cri-tools-1.19.0-0.x86_64.rpm is not installed
(2/10): 67ffa375b03cea72703fe446ff00963919e8fce913fbc4bb86f06d1475a6bdf9-cri-tools-1.19.0-0.x86_64.rpm                | 5.7 MB  00:00:00     
(3/10): 0767753f85f415bbdf1df0e974eafccb653bee06149600c3ee05b903bdc897ba-kubeadm-1.17.4-0.x86_64.rpm                  | 8.7 MB  00:00:00     
(4/10): 06400b25ef3577561502f9a7a126bf4975c03b30aca0fb19bb636f870ab93876-kubectl-1.17.4-0.x86_64.rpm                  | 9.4 MB  00:00:00     
(5/10): libnetfilter_cttimeout-1.0.0-7.el7.x86_64.rpm                                                                 |  18 kB  00:00:00     
(6/10): 0c45baca5fcc05bb75f1e953ecaf85844efac01bf9c1ef3c219f2b41eade3168-kubelet-1.17.4-0.x86_64.rpm                  |  20 MB  00:00:00     
(7/10): libnetfilter_queue-1.0.2-2.el7_2.x86_64.rpm                                                                   |  23 kB  00:00:00     
(8/10): socat-1.7.3.2-2.el7.x86_64.rpm                                                                                | 290 kB  00:00:00     
(9/10): libnetfilter_cthelper-1.0.0-11.el7.x86_64.rpm                                                                 |  18 kB  00:00:00     
(10/10): db7cb5cb0b3f6875f54d10f02e625573988e3e91fd4fc5eef0b1876bb18604ad-kubernetes-cni-0.8.7-0.x86_64.rpm           |  19 MB  00:00:10     
---------------------------------------------------------------------------------------------------------------------------------------------
Total                                                                                                        5.9 MB/s |  63 MB  00:00:10     
Retrieving key from https://repo.huaweicloud.com/kubernetes/yum/doc/yum-key.gpg
Importing GPG key 0x307EA071:
 Userid     : "Rapture Automatic Signing Key (cloud-rapture-signing-key-2021-03-01-08_01_09.pub)"
 Fingerprint: 7f92 e05b 3109 3bef 5a3c 2d38 feea 9169 307e a071
 From       : https://repo.huaweicloud.com/kubernetes/yum/doc/yum-key.gpg
Importing GPG key 0x836F4BEB:
 Userid     : "gLinux Rapture Automatic Signing Key (//depot/google3/production/borg/cloud-rapture/keys/cloud-rapture-pubkeys/cloud-rapture-signing-key-2020-12-03-16_08_05.pub) <glinux-team@google.com>"
 Fingerprint: 59fe 0256 8272 69dc 8157 8f92 8b57 c5c2 836f 4beb
 From       : https://repo.huaweicloud.com/kubernetes/yum/doc/yum-key.gpg
Retrieving key from https://repo.huaweicloud.com/kubernetes/yum/doc/rpm-package-key.gpg
Importing GPG key 0x3E1BA8D5:
 Userid     : "Google Cloud Packages RPM Signing Key <gc-team@google.com>"
 Fingerprint: 3749 e1ba 95a8 6ce0 5454 6ed2 f09c 394c 3e1b a8d5
 From       : https://repo.huaweicloud.com/kubernetes/yum/doc/rpm-package-key.gpg
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
  Installing : libnetfilter_cthelper-1.0.0-11.el7.x86_64                                                                                1/10 
  Installing : socat-1.7.3.2-2.el7.x86_64                                                                                               2/10 
  Installing : libnetfilter_cttimeout-1.0.0-7.el7.x86_64                                                                                3/10 
  Installing : libnetfilter_queue-1.0.2-2.el7_2.x86_64                                                                                  4/10 
  Installing : conntrack-tools-1.4.4-7.el7.x86_64                                                                                       5/10 
  Installing : kubernetes-cni-0.8.7-0.x86_64                                                                                            6/10 
  Installing : kubelet-1.17.4-0.x86_64                                                                                                  7/10 
  Installing : kubectl-1.17.4-0.x86_64                                                                                                  8/10 
  Installing : cri-tools-1.19.0-0.x86_64                                                                                                9/10 
  Installing : kubeadm-1.17.4-0.x86_64                                                                                                 10/10 
  Verifying  : cri-tools-1.19.0-0.x86_64                                                                                                1/10 
  Verifying  : conntrack-tools-1.4.4-7.el7.x86_64                                                                                       2/10 
  Verifying  : kubernetes-cni-0.8.7-0.x86_64                                                                                            3/10 
  Verifying  : kubeadm-1.17.4-0.x86_64                                                                                                  4/10 
  Verifying  : kubectl-1.17.4-0.x86_64                                                                                                  5/10 
  Verifying  : libnetfilter_queue-1.0.2-2.el7_2.x86_64                                                                                  6/10 
  Verifying  : libnetfilter_cttimeout-1.0.0-7.el7.x86_64                                                                                7/10 
  Verifying  : socat-1.7.3.2-2.el7.x86_64                                                                                               8/10 
  Verifying  : kubelet-1.17.4-0.x86_64                                                                                                  9/10 
  Verifying  : libnetfilter_cthelper-1.0.0-11.el7.x86_64                                                                               10/10 

Installed:
  kubeadm.x86_64 0:1.17.4-0                     kubectl.x86_64 0:1.17.4-0                     kubelet.x86_64 0:1.17.4-0                    

Dependency Installed:
  conntrack-tools.x86_64 0:1.4.4-7.el7           cri-tools.x86_64 0:1.19.0-0                    kubernetes-cni.x86_64 0:0.8.7-0             
  libnetfilter_cthelper.x86_64 0:1.0.0-11.el7    libnetfilter_cttimeout.x86_64 0:1.0.0-7.el7    libnetfilter_queue.x86_64 0:1.0.2-2.el7_2   
  socat.x86_64 0:1.7.3.2-2.el7                  

Complete!

# 配置kubelet的cgroup
[root@master ~]# vi /etc/sysconfig/kubelet
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
KUBE_PROXY_MODE="ipvs"

# 设置开机自启
[root@master ~]# systemctl enable kubelet
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.

12 查看集群所需镜像

[root@master ~]# kubeadm config images list
I0106 09:35:08.828094    1981 version.go:251] remote version is much newer: v1.26.0; falling back to: stable-1.17
W0106 09:35:09.558699    1981 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0106 09:35:09.558714    1981 validation.go:28] Cannot validate kubelet config - no validator is available
k8s.gcr.io/kube-apiserver:v1.17.17
k8s.gcr.io/kube-controller-manager:v1.17.17
k8s.gcr.io/kube-scheduler:v1.17.17
k8s.gcr.io/kube-proxy:v1.17.17
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.4.3-0
k8s.gcr.io/coredns:1.6.5

所有主机都安装完

[root@master ~]# kubelet --version
Kubernetes v1.17.4
[root@node01 ~]# kubelet --version
Kubernetes v1.17.4
[root@node02 ~]# kubelet --version
Kubernetes v1.17.4

三、集群部署

1 集群初始化

master节点

#创建集群
[root@master ~]# kubeadm init \
   --apiserver-advertise-address=192.168.0.10 \
   --image-repository registry.aliyuncs.com/google_containers \
   --kubernetes-version v1.18.0 \
   --service-cidr=10.96.0.0/12 \
   --pod-network-cidr=10.244.0.0/16
W0106 09:44:58.157182    2522 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0106 09:44:58.157221    2522 validation.go:28] Cannot validate kubelet config - no validator is available
[init] Using Kubernetes version: v1.18.0
[preflight] Running pre-flight checks
	[WARNING KubernetesVersion]: Kubernetes version is greater than kubeadm version. Please consider to upgrade kubeadm. Kubernetes version: 1.18.0. Kubeadm version: 1.17.x
	[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.22. Latest validated version: 19.03
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.0.10]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [master localhost] and IPs [192.168.0.10 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [master localhost] and IPs [192.168.0.10 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
W0106 09:47:43.270684    2522 manifests.go:214] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[control-plane] Creating static Pod manifest for "kube-scheduler"
W0106 09:47:43.271723    2522 manifests.go:214] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 14.501687 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node master as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: a3um0n.m1pdmcn4l6ci3j35
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.0.10:6443 --token a3um0n.m1pdmcn4l6ci3j35 \
    --discovery-token-ca-cert-hash sha256:7103a2a4eabde7dcaba47ff5e3acf91a170229bd0d31e66a99d4e0df917ce8db 
# 创建配置文件
[root@master ~]# mkdir -p $HOME/.kube
[root@master ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

node节点

[root@node01 ~]# kubeadm join 192.168.0.10:6443 --token a3um0n.m1pdmcn4l6ci3j35 \
>     --discovery-token-ca-cert-hash sha256:7103a2a4eabde7dcaba47ff5e3acf91a170229bd0d31e66a99d4e0df917ce8db 
W0106 09:49:49.764498    2491 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
	[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.22. Latest validated version: 19.03
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
[root@node02 ~]# kubeadm join 192.168.0.10:6443 --token a3um0n.m1pdmcn4l6ci3j35 \
>     --discovery-token-ca-cert-hash sha256:7103a2a4eabde7dcaba47ff5e3acf91a170229bd0d31e66a99d4e0df917ce8db 
W0106 09:49:58.874119    2434 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
	[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.22. Latest validated version: 19.03
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

查看节点信息

[root@master ~]# kubectl get nodes
NAME     STATUS     ROLES    AGE     VERSION
master   NotReady   master   4m27s   v1.17.4
node01   NotReady   <none>   2m18s   v1.17.4
node02   NotReady   <none>   2m23s   v1.17.4

# 默认的token有效期为2小时,可设置永久不过期的token
[root@master ~]# kubeadm token create --ttl 0 --print-join-command
W0106 10:03:20.495178    7227 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0106 10:03:20.495209    7227 validation.go:28] Cannot validate kubelet config - no validator is available
kubeadm join 192.168.0.10:6443 --token ovprzv.ejx1hfx5xqq7q1w4     --discovery-token-ca-cert-hash sha256:7103a2a4eabde7dcaba47ff5e3acf91a170229bd0d31e66a99d4e0df917ce8db

2 安装网络插件

master节点

[root@master ~]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
--2023-01-06 10:08:06--  https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.110.133, 185.199.108.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 4621 (4.5K) [text/plain]
Saving to: ‘kube-flannel.yml’

100%[===================================================================================================>] 4,621       --.-K/s   in 0s      

2023-01-06 10:08:07 (78.7 MB/s) - ‘kube-flannel.yml’ saved [4621/4621]

3 启动网络插件

master节点

[root@master ~]# kubectl apply -f kube-flannel.yml
namespace/kube-flannel created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created

4 查看状态

master节点

# 查看网络插件状态
[root@master ~]# kubectl get pods -n kube-system
NAME                             READY   STATUS    RESTARTS   AGE
coredns-5dbbf58dbf-j4vsr         1/1     Running   0          2m37s
coredns-5dbbf58dbf-xrhfr         1/1     Running   0          2m37s
etcd-master                      1/1     Running   0          2m54s
kube-apiserver-master            1/1     Running   0          2m54s
kube-controller-manager-master   1/1     Running   0          2m54s
kube-proxy-75wjv                 1/1     Running   0          2m37s
kube-proxy-bdxtl                 1/1     Running   0          2m23s
kube-proxy-pzrt2                 1/1     Running   0          2m19s
kube-scheduler-master            1/1     Running   0          2m54s

# 查看节点状态
[root@master ~]# kubectl get nodes
NAME     STATUS   ROLES    AGE     VERSION
master   Ready    master   3m5s    v1.17.4
node01   Ready    <none>   2m32s   v1.17.4
node02   Ready    <none>   2m28s   v1.17.4

# 查看集群健康状况
[root@master ~]# kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
controller-manager   Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
[root@master ~]# kubectl cluster-info
Kubernetes master is running at https://192.168.0.10:6443
KubeDNS is running at https://192.168.0.10:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

四、集群测试

# 创建nginx服务
[root@master ~]# kubectl create deployment nginx  --image=nginx:1.14-alpine
deployment.apps/nginx created

# 暴露端口
[root@master ~]# kubectl expose deploy nginx  --port=80 --target-port=80  --type=NodePort
service/nginx exposed

# 查看服务
[root@master ~]# kubectl get pod,svc
NAME                         READY   STATUS    RESTARTS   AGE
pod/nginx-55f8fd7cfc-fc2pk   1/1     Running   0          42s

NAME                 TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
service/kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        7m3s
service/nginx        NodePort    10.100.97.193   <none>        80:31429/TCP   19s

# 查看pod
[root@master ~]# kubectl get pod
NAME                     READY   STATUS    RESTARTS   AGE
nginx-55f8fd7cfc-fc2pk   1/1     Running   0          117s