Kubernetes服务优化重习

原创已于 2026-05-23 22:46:52 修改 · 345 阅读

8 ·

本内容遵循CC 4.0 BY-SA版权协议

GEO检测

标签

#kubernetes #容器 #云原生 #云计算 #笔记

于 2026-05-22 18:19:36 首次发布

存储docker registry的认证信息

建立harbor私有仓库

# docker login reg.timinglee.org

# vim pod-auth.yml

apiVersion: v1

kind: Pod

metadata:

  name: test-noauth

spec:

  containers:

  - name: nginxtest

    image: reg.timinglee.org/test/nginx:latest   # 无认证

# kubectl create secret docker-registry docker-auth --docker-server=reg.timinglee.org --docker-username=admin --docker-password=lee

# kubectl get secrets 查看结果：

# vim pod-auth.yml

apiVersion: v1

kind: Pod

metadata:

  name: test-auth

spec:

  containers:

  - name: nginxtest

    image: reg.timinglee.org/test/nginx:latest

  imagePullSecrets:                              #不设定docker认证时无法下载镜像

  - name: docker-auth

volumes配置管理

  容器中文件在磁盘上是临时存放的，这给容器中运行的特殊应用程序带来一些问题

    当容器崩溃时，kubelet将重新启动容器，文件会丢失，因为容器会以干净的状态重建。

    当在一个 Pod 中同时运行多个容器时，常常需要在这些容器之间共享文件。

    Kubernetes 卷具有明确的生命周期与使用它的 Pod 相同卷比 Pod 中运行的任何容器的存活期都长，在容器重新启动时数据也会得到保留

    当一个 Pod 不再存在时，卷也将不再存在。

    Kubernetes 可以支持许多类型的卷，Pod 也能同时使用任意数量的卷。

    卷不能挂载到其他卷，也不能与其他卷有硬链接。 Pod 中的每个容器必须独立地指定每个卷的挂载位置。

emptyDir卷

master ~]# mkdir -p volumes

# cd volumes/

# vim empty.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: empty

  name: empty

spec:

  containers:

  - image: busybox

    name: busybox

    command:

    - /bin/sh

    - -c

    - sleep 100000

    volumeMounts:

    - mountPath: /cache

      name: cache-vol

  - image: nginx

    name: nginx

    volumeMounts:

    - mountPath: /usr/share/nginx/html

      name: cache-vol

  volumes:

  - name: cache-vol

    emptyDir:

      medium: Memory

      sizeLimit: 100Mi

# kubectl apply -f empty.yml

# kubectl get pods -o wide

# curl 10.244.2.44

# kubectl exec -it pods/empty -c busybox -- /bin/sh

/ # ls

/ # cd cache/

/cache # dd if=/dev/zero of=bigfile bs=1M count=99

# curl 10.244.2.44

hostPath

# kubectl run hostpath --image nginx --dry-run=client -o yaml > hostpath.yml

# vim hostpath.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: hostpath

  name: hostpath

spec:

  containers:

  - image: nginx

    name: hostpath

    volumeMounts:

    - mountPath: /usr/share/nginx/html

      name: timinglee

  volumes:

  - name: timinglee

    hostPath:

      path: /data

      type: DirectoryOrCreate    #当/data目录不存在时自动建立

# kubectl get pods -o wide

# curl 10.244.2.45

ssh -l root node1

node1 ~]# ll /data/

master ~]# echo hello timinglee > /data/index.html

nfs卷

建立nfs共享存储

node3 ~]# mkdir /share

# dnf install nfs-utils -y

# systemctl enable --now nfs-server.service

# echo "/share *(sync,rw,no_root_squash)" > /etc/exports

# exportfs -rv

# showmount -e

在所有work节点安装nfs-utils

master ~]# for i in 10 20; do ssh -l root 172.25.254.$i dnf install nfs-utils -y; done

# for i in 10 20; do ssh -l root 172.25.254.$i showmount -e 172.25.254.30 ; done

建立nfs卷

master ~]# kubectl run web --image nginx --dry-run=client -o yaml >> nfs.yml

# vim nfs.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: web1

  name: web1

spec:

  nodeName: node1

  containers:

  - image: nginx

    name: web1

    volumeMounts:

    - mountPath: /usr/share/nginx/html

      name: cache-vol

  volumes:

  - name: cache-vol

    nfs:

      server: 172.25.254.30

      path: /share

# kubectl apply -f nfs.yml

# kubectl get pods -o wide

node3 ~]# echo hello timinglee > /share/index.html

master ~]# kubectl delete -f nfs.yml

# vim nfs.yml

# kubectl get pods -o wide

# curl 10.244.2.46

PersistentVolume的静态持久卷

建立存储目录

node3 ~]# mkdir /share/pv{1..3} -p

# dnf install nfs-utils -y

# systemctl enable --now nfs-server

# echo "/share *(sync,rw,no_root_squash)" > /etc/exports

# exportfs -rv

编写pv建立的yaml文件

master volumes]# vim pv.yml

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv1

spec:

  capacity:

    storage: 5Gi

  volumeMode: Filesystem

  accessModes:

  - ReadWriteOnce

  persistentVolumeReclaimPolicy: Retain

  storageClassName: nfs

  nfs:

     path: /share/pv1

     server: 172.25.254.30

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv2

spec:

  capacity:

    storage: 10Gi

  volumeMode: Filesystem

  accessModes:

  - ReadWriteMany

  persistentVolumeReclaimPolicy: Retain

  storageClassName: nfs

  nfs:

     path: /share/pv2

     server: 172.25.254.30

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv3

spec:

  capacity:

    storage: 15Gi

  volumeMode: Filesystem

  accessModes:

  - ReadOnlyMany

  persistentVolumeReclaimPolicy: Retain

  storageClassName: nfs

  nfs:

     path: /share/pv3

     server: 172.25.254.30

# kubectl apply -f pv.yml

# kubectl get pv

一次性指定多个pvc

master ~]# vim pv.yml

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: pvc1

spec:

  storageClassName: nfs

  accessModes:

    - ReadWriteOnce   #写什么取决于上面的 name 写的什么

  resources:

    requests:

      storage: 1Gi

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: pvc2

spec:

  storageClassName: nfs

  accessModes:

    - ReadWriteMany

  resources:

    requests:

      storage: 5Gi

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: pvc3

spec:

  storageClassName: nfs

  accessModes:

    - ReadOnlyMany

  resources:

    requests:

      storage: 10Gi

# kubectl get pvc 查看结果：

# kubectl -n kube-system get pvc

master ~]# vim pvc-test1.yml

apiVersion: v1

kind: Pod

metadata:

  name: pvc-test1

spec:

  containers:

  - name: nginx

    image: nginx:latest

    volumeMounts:

    - name: pv-storage

      mountPath: /usr/share/nginx/html

  volumes:

  - name: pv-storage

    persistentVolumeClaim:

      claimName: pvc1

# kubectl exec -it pod-pvc1-test1 -- /bin/bash

/ # echo "pvc-test1" > /usr/share/nginx/html/test1.txt

docker ]# cat pv1/test1.txt  查看结果：

master ~]# kubectl exec -it pods/pvc-test1 /bin/bash

/# cd /usr/share/nginx/html

html# ls

# vim checkpvpod.yml

apiVersion: v1

kind: Pod

metadata:

  name: timinglee

spec:

  containers:

  - image: nginx

    name: nginx

    volumeMounts:

    - mountPath: /usr/share/nginx/html

      name: vol1

  volumes:

  - name: vol1

    persistentVolumeClaim:

      claimName: pvc1

# kubectl apply -f checkpvpod.yml

# kubectl get pods

# curl 10.244.4.12

node3 pv1]# echo hello timinglee > index.html

删除pvc和pv

master ~]# kubectl get pv

# kubectl delete pvc pvc2

# kubectl get pv 查看结果：

# kubectl edit pv pv2

动态持久卷

上传所需镜像

volumes]# docker load -i /root/nfs-subdir-external-provisioner-4.0.2.tar

# docker tag registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2 reg.timinglee.org/sig-storage/nfs-subdir-external-provisioner:v4.0.2

# docker push reg.timinglee.org/sig-storage/nfs-subdir-external-provisioner:v4.0.2

# kubectl delete pvc pvc1;kubectl delete pvc pvc2;kubectl delete pvc pvc3

# kubectl delete pv pv1;kubectl delete pv pv2;kubectl delete pv pv3

建立授权

# vim storagesa.yml

apiVersion: v1

kind: Namespace

metadata:

  name: nfs-client-provisioner

---

apiVersion: v1

kind: ServiceAccount

metadata:

  name: nfs-client-provisioner

  namespace: nfs-client-provisioner

---

kind: ClusterRole

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: nfs-client-provisioner-runner

rules:

  - apiGroups: [""]

    resources: ["nodes"]

    verbs: ["get", "list", "watch"]

  - apiGroups: [""]

    resources: ["persistentvolumes"]

    verbs: ["get", "list", "watch", "create", "delete"]

  - apiGroups: [""]

    resources: ["persistentvolumeclaims"]

    verbs: ["get", "list", "watch", "update"]

  - apiGroups: ["storage.k8s.io"]

    resources: ["storageclasses"]

    verbs: ["get", "list", "watch"]

  - apiGroups: [""]

    resources: ["events"]

    verbs: ["create", "update", "patch"]

---

kind: ClusterRoleBinding

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: run-nfs-client-provisioner

subjects:

  - kind: ServiceAccount

    name: nfs-client-provisioner

    namespace: nfs-client-provisioner

roleRef:

  kind: ClusterRole

  name: nfs-client-provisioner-runner

  apiGroup: rbac.authorization.k8s.io

---

kind: Role

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: leader-locking-nfs-client-provisioner

  namespace: nfs-client-provisioner

rules:

  - apiGroups: [""]

    resources: ["endpoints"]

    verbs: ["get", "list", "watch", "create", "update", "patch"]

---

kind: RoleBinding

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: leader-locking-nfs-client-provisioner

  namespace: nfs-client-provisioner

subjects:

  - kind: ServiceAccount

    name: nfs-client-provisioner

    namespace: nfs-client-provisioner

roleRef:

  kind: Role

  name: leader-locking-nfs-client-provisioner

  apiGroup: rbac.authorization.k8s.io

vim storagesa.yml

apiVersion: v1

kind: Namespace

metadata:

  name: nfs-client-provisioner

---

apiVersion: v1

kind: ServiceAccount

metadata:

  name: nfs-client-provisioner

  namespace: nfs-client-provisioner

---

kind: ClusterRole

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: nfs-client-provisioner-runner

rules:

  - apiGroups: [""]

    resources: ["nodes"]

    verbs: ["get", "list", "watch"]

  - apiGroups: [""]

    resources: ["persistentvolumes"]

    verbs: ["get", "list", "watch", "create", "delete"]

  - apiGroups: [""]

    resources: ["persistentvolumeclaims"]

    verbs: ["get", "list", "watch", "update"]

  - apiGroups: ["storage.k8s.io"]

    resources: ["storageclasses"]

    verbs: ["get", "list", "watch"]

  - apiGroups: [""]

    resources: ["events"]

    verbs: ["create", "update", "patch"]

---

kind: ClusterRoleBinding

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: run-nfs-client-provisioner

subjects:

  - kind: ServiceAccount

    name: nfs-client-provisioner

    namespace: nfs-client-provisioner

roleRef:

  kind: ClusterRole

  name: nfs-client-provisioner-runner

  apiGroup: rbac.authorization.k8s.io

---

kind: Role

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: leader-locking-nfs-client-provisioner

  namespace: nfs-client-provisioner

rules:

  - apiGroups: [""]

    resources: ["endpoints"]

    verbs: ["get", "list", "watch", "create", "update", "patch"]

---

kind: RoleBinding

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: leader-locking-nfs-client-provisioner

  namespace: nfs-client-provisioner

subjects:

  - kind: ServiceAccount

    name: nfs-client-provisioner

    namespace: nfs-client-provisioner

roleRef:

  kind: Role

  name: leader-locking-nfs-client-provisioner

  apiGroup: rbac.authorization.k8s.io

# kubectl -n nfs-client-provisioner get sa

建立控制器

# vim storageclassdep.yml

spec:

  replicas: 1

  strategy:

    type: Recreate

  selector:

    matchLabels:

      app: nfs-client-provisioner

  template:

    metadata:

      labels:

        app: nfs-client-provisioner

    spec:

      serviceAccountName: nfs-client-provisioner

      containers:

        - name: nfs-client-provisioner

          image: sig-storage/nfs-subdir-external-provisioner:v4.0.2

          volumeMounts:

            - name: nfs-client-root

              mountPath: /persistentvolumes

          env:

            - name: PROVISIONER_NAME

              value: k8s-sigs.io/nfs-subdir-external-provisioner

            - name: NFS_SERVER

              value: 172.25.254.30

            - name: NFS_PATH

              value: /share

      volumes:

        - name: nfs-client-root

          nfs:

            server: 172.25.254.30

            path: /share

# kubectl apply -f storageclassdep.yml

# kubectl -n nfs-client-provisioner get pods

建立存储类

# vim storageclass.yml

apiVersion: storage.k8s.io/v1

kind: StorageClass

metadata:

name: nfs-client

provisioner: k8s-sigs.io/nfs-subdir-external-provisioner

parameters:

archiveOnDelete: "false"

# kubectl apply -f storageclass.yml

# kubectl get storageclasses.storage.k8s.io

建立pvc

# vim pvc.yml

kind: PersistentVolumeClaim

apiVersion: v1

metadata:

  name: test-claim

spec:

  storageClassName: nfs-client

  accessModes:

    - ReadWriteMany

  resources:

    requests:

      storage: 1G

# kubectl get pvc

# kubectl delete -f pvc.yml

设定默认存储类

# kubectl edit sc nfs-client

# kubectl get sc

# vim pvc.yml

kind: PersistentVolumeClaim

apiVersion: v1

metadata:

  name: test-claim

spec:

  #storageClassName: nfs-client

  accessModes:

    - ReadWriteMany

  resources:

    requests:

      storage: 1G

# kubectl apply -f pvc.yml

# kubectl get pvc

statfulset控制器整合动态卷

# kubectl create service clusterip timinglee --tcp 80:80 --clusterip="None" --dry-run=client -o yaml > headless.yml    statfulset控制器整合动态卷

# vim headless.yml

apiVersion: v1

kind: Service

metadata:

  labels:

    app: timinglee

  name: timinglee

spec:

  clusterIP: None

  ports:

  - name: webport

    port: 80

    protocol: TCP

    targetPort: 80

  selector:

    app: webserver

  type: ClusterIP

# kubectl create deployment webserver --image nginx --replicas 1 --dry-run=client -o yaml > statefulset.yml    创建statefulset

# vim statefulset.yml

apiVersion: apps/v1

kind: StatefulSet

metadata:

  labels:

    app: webserver

  name: webserver

spec:

  serviceName: "timinglee"

  replicas: 1

  selector:

    matchLabels:

      app: webserver

  template:

    metadata:

      labels:

        app: webserver

    spec:

      containers:

      - image: nginx

        name: nginx

        volumeMounts:

          - name: www

            mountPath: /usr/share/nginx/html

  volumeClaimTemplates:

  - metadata:

      name: www

    spec:

      storageClassName: nfs-client

      accessModes:

       - ReadWriteOnce

      resources:

        requests:

          storage: 1Gi

# kubectl apply -f statefulset.yml

# kubectl get statefulsets.apps

# kubectl get pods

# kubectl get pvc

# kubectl scale statefulset webserver --replicas 2

# kubectl get pods

# kubectl get pvc

# kubectl scale statefulset webserver --replicas 3

node3 ~]# cd /share/

# echo webserver1 > default-www-webserver-0-pvc-***/index.html

# echo webserver2 > default-www-webserver-1-pvc-***/index.html

# echo webserver3 > default-www-webserver-2-pvc-***/index.html

master ~]# kubectl run -it testpod --image busyboxplus

$ curl webserver-0.timinglee

# kubectl delete -f statefulset.yml

# kubectl apply -f statefulset.yml

fannel插件转换为calico插件

部署calico

# kubectl delete -f /root/kube-flannel.yml   【删除flannel插件】

# rm -rf /etc/cni/net.d/10-flannel.conflist 【删除所有节点上flannel配置文件,避免冲突】

# ssh -l root 172.25.254.10 rm -rf /etc/cni/net.d/10-flannel.conflist

# ssh -l root 172.25.254.20 rm -rf /etc/cni/net.d/10-flannel.conflist

# mkdir -p /root/network

# cd /root/network/

# vim calico.yaml      【修改文件】

# docker load -i calico-3.28.1.tar

# docker tag calico/cni:v3.28.1 reg.timinglee.org/calico/cni:v3.28.1

# docker push reg.timinglee.org/calico/cni:v3.28.1

# docker tag calico/node:v3.28.1 reg.timinglee.org/calico/node:v3.28.1

# docker push reg.timinglee.org/calico/node:v3.28.1

# docker tag calico/kube-controllers:v3.28.1 reg.timinglee.org/calico/kube-controllers:v3.28.1

# docker push reg.timinglee.org/calico/kube-controllers:v3.28.1

# docker tag calico/typha:v3.28.1 reg.timinglee.org/calico/typha:v3.28.1

# docker push reg.timinglee.org/calico/typha:v3.28.1

# kubectl apply -f calico.yaml --validate=false   【运行文件】

# kubectl -n kube-system get pods | grep calico

calico测试

# kubectl run testpod --image nginx

# kubectl get pods

# kubectl get pods -o wide

# curl 10.244.166.128

调度器

nodeName调度

# watch -n 1 kubectl get pods -o wide

# mkdir -p Scheduler

# cd Scheduler/

# kubectl run nginx --image nginx --dry-run=client -o yaml > nginx.yml

# vim nginx.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: nginx

  name: nginx

spec:# nodeName: #未指定调度阶段

  containers:

  - image: nginx

    name: nginx

# kubectl apply -f nginx.yml

# kubectl delete -f nginx.yml

# kubectl apply -f nginx.yml

nodeselector调度

# vim nginx.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: nginx

  name: nginx

spec:

  nodeSelector:

    app: timinglee

  containers:

  - image: nginx

    name: nginx

# kubectl get nodes --show-labels

# kubectl label nodes

# kubectl label nodes node2 app=timinglee

# kubectl get nodes --show-labels

# kubectl label nodes node2 app-

节点亲和

倾向满足

# vim nginx.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: nginx

  name: nginx

spec:

  containers:

  - image: nginx

    name: nginx

  affinity:

    nodeAffinity:

      preferredDuringSchedulingIgnoredDuringExecution:

      - preference:

         matchExpressions:

          - key: disk

            operator: In

            values:

            - ssd

            - iscsi

        weight: 50

# kubectl delete -f nginx.yml

# kubectl label nodes node1 disk=ssd

# kubectl apply -f nginx.yml

# vim nginx.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: nginx

  name: nginx

spec:

  containers:

  - image: nginx

    name: nginx

  affinity:

    nodeAffinity:

      requiredDuringSchedulingIgnoredDuringExecution:

        nodeSelectorTerms:

        - matchExpressions:

          - key: disk

            operator: In

            values:

            - ssd

            - iscsi

# kubectl apply -f nginx.yml

节点上必须存在标签 disk，并且该标签的值是 ssd 或 iscsi，如果集群中没有节点带有 disk=ssd 或 disk=iscsi 标签，该 Pod 将永远无法调度（状态为 Pending）。

必须满足

# vim nginx.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: nginx

  name: nginx

spec:

  containers:

  - image: nginx

    name: nginx

  affinity:

    nodeAffinity:

      requiredDuringSchedulingIgnoredDuringExecution:

        nodeSelectorTerms:

        - matchExpressions:

          - key: disk

            operator: In

            values:

            - ssd

            - iscsi

# kubectl apply -f nginx.yml

# vim nginx.yml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: nginx

  name: nginx

spec:

  containers:

  - image: nginx

    name: nginx

  affinity:

    nodeAffinity:

      requiredDuringSchedulingIgnoredDuringExecution:

        nodeSelectorTerms:

        - matchExpressions:

          - key: disk

            operator: NotIn #反向选择

            values:

            - ssd

            - iscsi

POD亲和

# kubectl create deployment webcluster --image nginx --replicas 2 --dry-run=client -o yaml > webcluster.yml

# vim webcluster.yml

apiVersion: apps/v1

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 2

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

      affinity:

        podAffinity:

          requiredDuringSchedulingIgnoredDuringExecution:

          - labelSelector:

              matchExpressions:

              - key: app

                operator: In

                values:

                - webcluster

            topologyKey: "kubernetes.io/hostname"

# kubectl apply -f webcluster.yml

POD反亲和

# kubectl create deployment webcluster --image nginx --replicas 2 --dry-run=client -o yaml > webcluster.yml

# vim webcluster.yml

apiVersion: apps/v1

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 3

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

      affinity:

        podAntiAffinity:

          requiredDuringSchedulingIgnoredDuringExecution:

          - labelSelector:

              matchExpressions:

              - key: app

                operator: In

                values:

                - webcluster

            topologyKey: "kubernetes.io/hostname"

节点的污点设定

开启监控并启动该一个deployment控制器

# watch -n 1 kubectl get pods -o wide

# kubectl create deployment webcluster --image nginx --replicas 2 --dry-run=client -o yaml > dep.yml

# vim dep.yml

apiVersion: apps/v1

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 2

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

# kubectl apply -f dep.yml

设定污点并购观察

NoExecute

# kubectl taint node node1 nodetype=badnode:NoExecute

# kubectl taint node node1 nodetype-

NoSchedule

# kubectl taint node node2 nodetype=badnode:NoSchedule

# kubectl delete -f dep.yml; kubectl apply -f dep.yml

PreferNoSchedule

# kubectl delete -f dep.yml

# kubectl taint node node2 nodetype=badnode:PreferNoSchedule

# kubectl apply -f dep.yml

# vim webcluster.yml #pod反亲和

apiVersion: apps/v1

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 3

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

      affinity:

        podAntiAffinity:

          requiredDuringSchedulingIgnoredDuringExecution:

          - labelSelector:

              matchExpressions:

              - key: app

                operator: In

                values:

                - webcluster

            topologyKey: "kubernetes.io/hostname"

# kubectl apply -f webcluster.yml

污点容忍

设置节点不同类型给的污点

# kubectl taint node node1 name=lee:NoSchedule

# kubectl taint node node2 nodetype=badnode:NoSchedule

运行deployment控制器

# kubectl apply -f dep.yml

# kubectl delete -f dep.yml

精确容忍指定污点

# vim dep.yml

apiVersion: apps/v1

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 2

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

      tolerations: #污点容忍

      - operator: Equal

        key: nodetype

        value: badnode

        effect: NoSchedule

容忍所有标签的NoSchedule污点模式

# vim dep.yml

apiVersion: apps/v1

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 3

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

      tolerations:

      - operator: Exists

        effect: NoSchedule

容忍所有污点

# vim dep.yml

kind: Deployment

metadata:

  labels:

    app: webcluster

  name: webcluster

spec:

  replicas: 3

  selector:

    matchLabels:

      app: webcluster

  template:

    metadata:

      labels:

        app: webcluster

    spec:

      containers:

      - image: nginx

        name: nginx

      tolerations:

      - operator: Exists

# kubectl apply -f dep.yml

ServiceAccount

现在私有仓库时遇到的授权问题

# kubectl run testpod --image nginx --dry-run=client -o yaml > testpod.yaml

# vim testpod.yaml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: testpod

  name: testpod

spec:

  containers:

  - image: nginx

    name: testpod

# kubectl apply -f testpod.yaml

# kubectl describe pods testpod | grep Service

# vim testpod.yaml   指定使用过私有仓库镜像

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: testpod

  name: testpod

spec:

  containers:

  - image: reg.timinglee.org/timinglee/myapp:v1 #私有仓库经香港

    name: testpod

imagePullPolicy: Always

解决授权问题

# kubectl create serviceaccount timinglee

# kubectl create secret docker-registry docker-login --docker-username admin --docker-password lee --docker-server reg.timinglee.org --docker-email lee@timinglee.org

# kubectl describe sa timinglee

# kubectl edit sa timinglee

# kubectl describe sa timinglee

# vim testpod.yaml

apiVersion: v1

kind: Pod

metadata:

  labels:

    run: testpod

  name: testpod

spec:

  serviceAccountName: timinglee

  containers:

  - image: reg.timinglee.org/timinglee/myapp:v1

    name: testpod

    imagePullPolicy: Always

# kubectl apply -f testpod.yaml

建立集群用户

建立用户证书

# cd /etc/kubernetes/pki/

# openssl genrsa -out timinglee.key 2048

# openssl req -new -key timinglee.key -out timinglee.csr -subj "/CN=timinglee"

# openssl x509 -req -in timinglee.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out timinglee.crt -days 365

# openssl x509 -in timinglee.crt -text -noout 【查看证书】

建立用户

# kubectl config set-credentials timinglee --client-certificate /etc/kubernetes/pki/timinglee.crt --client-key /etc/kubernetes/pki/timinglee.key --embed-certs=true

# kubectl config view

为用户创建集群的安全上下文

# kubectl config set-context timinglee@kubernetes --cluster kubernetes --user timinglee

# kubectl config view 查看结果：

切换用户

# kubectl config use-context timinglee@kubernetes

# kubectl get pods 【用户在集群中只有用户身份没有授权】

切换回集群管理

# kubectl config use-context kubernetes-admin@kubernetes

如果需要删除用户

# kubectl config delete-user timinglee

授权

role和rolebinding

# kubectl config use-context kubernetes-admin@kubernetes   切换回管理员

建立role授权

# kubectl create role timingleerole --dry-run=client --verb=get --resource pods -o yaml > timingleerole.yaml

# vim timingleerole.yaml

apiVersion: rbac.authorization.k8s.io/v1

kind: Role

metadata:

  name: timingleerole

rules:

- apiGroups:

  - ""

  resources:

  - pods

  verbs:

  - get

  - watch

  - list

  - create

  - update

  - path

  - delete

- apiGroups:

  - "apps"

  resources:

  - deployments

  verbs:

  - get

  - list

  - watch

  - create

创建role

# kubectl apply -f myrole.yml

# kubectl describe role myrole

# kubectl create rolebinding timinglee --role myrole --namespace default --user timinglee --dry-run=client -o yaml > rolebinding-myrole.yml

# vim rolebinding-myrole.yml

apiVersion: rbac.authorization.k8s.io/v1

kind: RoleBinding

metadata:

  name: timinglee

  namespace: default #角色绑定必须指定namespace

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: Role

  name: myrole

subjects:

- apiGroup: rbac.authorization.k8s.io

  kind: User

  name: timinglee

# kubectl apply -f rolebinding-myrole.yml

# kubectl get rolebindings.rbac.authorization.k8s.io timinglee

切换用户测试授权

# kubectl config use-context timinglee@kubernetes

# kubectl get pods

# kubectl get svc

切换回管理员

# kubectl config use-context kubernetes-admin@kubernetes

clusterrole和clusterrolebind

建立clusterrole

# kubectl create clusterrole timingleeclusterrole --resource=deployment --verb get --dry-run=client -o yaml > timingleeclusterrole.yml

# vim timingleeclusterrole.yml

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRole

metadata:

  name: timingleeclusterrole

rules:

- apiGroups:

  - ""

  resources:

  - pods

  verbs:

  - get

  - watch

  - list

  - create

  - update

  - path

  - delete

- apiGroups:

  - "apps"

  resources:

  - deployments

  verbs:

  - get

  - watch

  - list

  - create

- apiGroups:

  - ""

  resources:

  - services

  verbs:

  - get

  - watch

  - list

  - create

# kubectl apply -f timingleeclusterrole.yml

# kubectl describe clusterrole myclusterrole

# vim timingleeclusterrole.yml

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRoleBinding

metadata:

  name: clusterrolebind-myclusterrole

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  name: myclusterrole

subjects:

- apiGroup: rbac.authorization.k8s.io

  kind: User

  name: timinglee

# kubectl get rolebindings.rbac.authorization.k8s.io timinglee

# kubectl describe clusterrolebindings.rbac.authorization.k8s.io clusterrolebind-myclusterrole

权限测试

# kubectl config use-context timinglee@kubernetes

# kubectl get pods -A        【可以查看pod，也可以建立运行pod】

# kubectl get deployments.apps -A     【集群角色已经授权，可以查看】

# kubectl get svc -A

# kubectl exec -it test-pod -- sh

# kubectl config use-context kubernetes-admin@kubernetes   【切换回管理员】

部署helm

helm简介

    Helm是Kubernetes 应用的包管理工具，主要用来管理 Charts，类似Linux系统的yum。

    Helm Chart是用来封装Kubernetes原生应用程序的一系列YAML文件。可以在你部署应用的时候自定义应用程序的一些 Metadata，以便于应用程序的分发。

  对于应用发布者而言：通过Helm打包应用、管理应用依赖关系、管理应用版本并发布应用到软件仓库。

  对于使用者而言：使用Helm后能以简单的方式在Kubernetes上查找、安装、升级、回滚、卸载应用程序

安装helm

# tar zxf /root/helm/helm-v3.15.4-linux-amd64.tar.gz

# cd /root/helm/linux-amd64/

# ls

# cp -p helm /usr/local/bin/

配置helm命令补齐

# echo "source <(helm completion bash)" >> ~/.bashrc

# source ~/.bashrc

# helm version 查看结果：

helm常用操作

  create       创建一个 chart 并指定名字

  dependency   管理 chart 依赖

  get          下载一个 release。可用子命令：all、hooks、manifest、notes、values

  history      获取 release 历史

  install      安装一个 chart

  list         列出 release

  package      将 chart 目录打包到 chart 存档文件中

  pull         从远程仓库中下载 chart 并解压到本地 # helm pull stable/mysql -- untar

  repo         添加，列出，移除，更新和索引 chart 仓库。可用子命令：add、index、 list、remove、update

  rollback     从之前版本回滚

  search       根据关键字搜索 chart。可用子命令：hub、repo

  show         查看 chart 详细信息。可用子命令：all、chart、readme、values

  status       显示已命名版本的状态

  template     本地呈现模板

  uninstall    卸载一个 release

  upgrade      更新一个 release

  version      查看 helm 客户端版本

查询官方应用中心

# helm search hub nginx        #在官方仓库中搜索

# helm search repo nginx          #在本地仓库中搜索

管理第三方repo源

# helm repo add aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts 【添加阿里云】

# helm repo add bitnami https://charts.bitnami.com/bitnami   【添加bitnami仓库】

# helm repo add miscro http://mirror.azure.cn/kubernetes/charts/   【微软仓库】

# helm repo list   【查看仓库信息】

# helm search repo aliyun    【查看仓库存储helm清单】

# helm repo remove aliyun   【删除第三方存储库】

helm的使用方法

# helm search repo nginx    【查找chart】

# helm show chart bitnami/nginx    【查看chart信息】

安装chart 包

# helm install mariadb miscro/mariadb

# helm list 查看结果：

# kubectl get pods

# helm status mariadb    【查看项目的发布状态】

# helm uninstall mariadb      【卸载项目】

# helm list 查看结果：

安装项目前预定义项目选项

# helm pull bitnami/nginx

# docker load -i /root/nginx-1.27.1-debian-12-r2.tar

# docker tag bitnami/nginx:1.27.1-debian-12-r2 reg.timinglee.org/bitnami/nginx:1.27.1-debian-12-r2

# docker push reg.timinglee.org/bitnami/nginx:1.27.1-debian-12-r2

# cd ~

# tar zxf nginx-18.1.11.tgz

# cd nginx/

# vim values.yaml 【项目变量文件】

# helm install timinglee /root/nginx

# kubectl get svc

# kubectl get pods 查看结果：

# vim values.yaml 【更新变量文件】

# helm upgrade timinglee .

# kubectl get svc 查看结果：

# kubectl get ingress

# vim /etc/hosts

# curl www.timinglee.org

# helm history timinglee

# helm uninstall timinglee    【删除项目】

# helm list 查看结果：

构建helm中的chart包

Helm Chart目录结构

# helm create timinglee    【简历chart项目】

# tree timinglee/

构建方法

# cd /root/helm/linux-amd64/timinglee/

# vim Chart.yaml

# vim values.yaml

#更改内容

image:

  repository: myapp

  pullPolicy: IfNotPresent

  tag: "v1"

ingress:

  enabled: true

  className: "nginx"

  annotations: {}

    # kubernetes.io/ingress.class: nginx

    # kubernetes.io/tls-acme: "true"

  hosts:

    - host: www.timinglee.org

      paths:

        - path: /

          pathType: ImplementationSpecific

# helm lint . 检测查看：

# helm install timinglee .

# kubectl get deployments.apps

项目打包

# cd /root/helm/linux-amd64

# helm package timinglee/

# ls

项目可以通过各种分享方式发方为任何人后部署即可

# helm install timinglee timinglee-0.1.0.tgz

构建helm仓库

在harbor仓库中构建一个公开的项目

安装helm push插件

如果网络没问题情况下直接安装即可

# cd ~

# dnf install git -y

# helm plugin install https://github.com/chartmuseum/helm-push

离线安装

# mkdir -p ~/.local/share/helm/plugins/helm-push

# tar zxf helm-push_0.10.4_linux_amd64.tar.gz -C ~/.local/share/helm/plugins/helm-push

# ls ~/.local/share/helm/plugins/helm-push

查看helm调用命令是否成功

# helm cm-push --help 查看结果：

上传项目到仓库

# helm repo add timinglee https://reg.timinglee.org/chartrepo/timinglee 【添加仓库】

为helm添加证书

# cp /etc/docker/certs.d/reg.timinglee.org/ca.crt /etc/pki/ca-trust/source/anchors/

# update-ca-trust 【更新本地ca认证库】

再次添加仓库

# helm repo add timinglee https://reg.timinglee.org/chartrepo/timinglee

上传本地项目

命令执行格式：helm cm-push <项目名称> <仓库名称> -u admin -p lee

# helm cm-push timinglee-0.1.0.tgz timinglee -u admin -p lee

查看项目上传情况

# helm search repo timinglee 【上传后数据未更新】

# helm repo update timinglee 【更新仓库】

# helm search repo timinglee 查看结果：

安装项目

# helm install timinglee timinglee/timinglee 【安装】

# curl www.timinglee.org 查看运行：

helm的版本迭代

从新构建新版本项目

# vim /root/helm/linux-amd64/timinglee/Chart.yaml

# vim /root/helm/linux-amd64/timinglee/values.yaml

# helm package timinglee

上传项目到helm仓库中

# helm cm-push timinglee-0.2.0.tgz timinglee -u admin -p lee

# helm repo update timinglee 【更新仓库】

# helm search repo timinglee -l 查看结果：

更新应用

# helm upgrade timinglee timinglee/timinglee

# curl http://www.timinglee.org/

显示项目版本

# helm history timinglee

应用回滚

# helm rollback timinglee

# helm history timinglee

# curl www.timinglee.org 查看运行：

Prometheus

Prometheus架构

在k8s中部署Prometheus

# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

在helm中添加Prometheus仓库

# mkdir -p prometheus

# cd prometheus/

# tar zxf kube-prometheus-stack-62.6.0.tgz

# ls kube-prometheus-stack/

# tar zxf prometheus-adapter-4.11.0.tgz

# ls prometheus-adapter/

根据所有项目中的values.yaml中指定的image路径下载容器镜像并上传至harbor仓库

容器镜像prometheus推送

# docker load -i prometheus-62.6.0.tar

# docker tag quay.io/prometheus/prometheus:v2.54.1 reg.timinglee.org/prometheus/prometheus:v2.54.1

# docker tag quay.io/thanos/thanos:v0.36.1 reg.timinglee.org/thanos/thanos:v0.36.1

# docker tag quay.io/prometheus/alertmanager:v0.27.0 reg.timinglee.org/prometheus/alertmanager:v0.27.0

# docker tag quay.io/prometheus-operator/admission-webhook:v0.76.1 reg.timinglee.org/prometheus-operator/admission-webhook:v0.76.1

# docker tag quay.io/prometheus-operator/prometheus-operator:v0.76.1 reg.timinglee.org/prometheus-operator/prometheus-operator:v0.76.1

# docker tag registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6 reg.timinglee.org/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6

# docker tag quay.io/prometheus-operator/prometheus-config-reloader:v0.76.1 reg.timinglee.org/prometheus-operator/prometheus-config-reloader:v0.76.1

# docker push reg.timinglee.org/prometheus/prometheus:v2.54.1

# docker push reg.timinglee.org/thanos/thanos:v0.36.1

# docker push reg.timinglee.org/prometheus/alertmanager:v0.27.0

# docker push reg.timinglee.org/prometheus-operator/admission-webhook:v0.76.1

# docker push reg.timinglee.org/prometheus-operator/prometheus-operator:v0.76.1

# docker push reg.timinglee.org/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6

# docker push reg.timinglee.org/prometheus-operator/prometheus-config-reloader:v0.76.1

# docker load -i node-exporter-v1.8.2.tar

# docker tag reg.timinglee.org/prometheus/node-exporter:v1.8.2 reg.timinglee.org/prometheus/node-exporter:v1.8.2

# docker push reg.timinglee.org/prometheus/node-exporter:v1.8.2

容器镜像grafana推送

# docker load -i grafana-11.2.0.tar

# docker tag grafana/grafana:11.2.0 reg.timinglee.org/grafana/grafana:11.2.0

# docker tag grafana/grafana-image-renderer:latest reg.timinglee.org/grafana/grafana-image-renderer:latest

# docker tag quay.io/kiwigrid/k8s-sidecar:1.27.4 reg.timinglee.org/kiwigrid/k8s-sidecar:1.27.4

# docker tag bats/bats:v1.4.1 reg.timinglee.org/bats/bats:v1.4.1

# docker push reg.timinglee.org/grafana/grafana:11.2.0

# docker push reg.timinglee.org/grafana/grafana-image-renderer:latest

# docker push reg.timinglee.org/kiwigrid/k8s-sidecar:1.27.4

# docker push reg.timinglee.org/bats/bats:v1.4.1

容器镜像nginx-exporter推送

# docker load -i nginx-exporter-1.3.0-debian-12-r2.tar

# docker tag bitnami/nginx-exporter:1.3.0-debian-12-r2 reg.timinglee.org/bitnami/nginx-exporter:1.3.0-debian-12-r2

# docker push reg.timinglee.org/bitnami/nginx-exporter:1.3.0-debian-12-r2

容器镜像kube-state-metrics推送

# docker load -i kube-state-metrics-2.13.0.tar

# docker tag registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0 reg.timinglee.org/kube-state-metrics/kube-state-metrics:v2.13.0

# docker tag quay.io/brancz/kube-rbac-proxy:v0.18.0 reg.timinglee.org/brancz/kube-rbac-proxy:v0.18.0

# docker push reg.timinglee.org/kube-state-metrics/kube-state-metrics:v2.13.0

# docker push reg.timinglee.org/brancz/kube-rbac-proxy:v0.18.0

利用helm安装Prometheus

卸载命令（如需要重新安装）: helm -n kube-prometheus-stack uninstall kube-prometheus-stack

# kubectl create namespace kube-prometheus-stack

# cd /root/prometheus/kube-prometheus-stack     【在这个目录执行】

# helm -n kube-prometheus-stack install kube-prometheus-stack .

# kubectl --namespace kube-prometheus-stack get pods 查看结果：

# kubectl -n kube-prometheus-stack get svc

# kubectl -n kube-prometheus-stack edit svc kube-prometheus-stack-grafana  【修改暴漏方式】

各个svc的作用

    alertmanager-operated 告警管理

    kube-prometheus-stack-grafana 展示prometheus采集到的指标

    kube-prometheus-stack-prometheus-node-exporter     收集节点级别的指标的工具

    kube-prometheus-stack-prometheus 主程序

登陆grafana

查看grafana密码

# kubectl -n kube-prometheus-stack get secrets kube-prometheus-stack-grafana -o yaml

# echo "YWRtaW4=" | base64 -d

# echo -n "cHJvbS1vcGVyYXRvcg==" | base64 -d

prometheus监控

下载prometheus

官网：https://github.com/prometheus-operator/prometheus-operator

登录使用

设置语言

监控使用示例

建立监控项目

# helm pull bitnami/nginx --version 18.1.11   【下载示例所需helm项目】

# vim values.yaml   【修改项目开启监控】

# kubectl -n kube-prometheus-stack edit svc kube-prometheus-stack-prometheus

# kubectl -n kube-prometheus-stack get svc kube-prometheus-stack-prometheus

查看监控

# kubectl -n kube-prometheus-stack get servicemonitors.monitoring.coreos.com --show-labels

# helm install timinglee .     【安装项目，在安装之前一定要上传镜像到仓库中】

# kubectl get svc 查看结果：

# ab -c 5 -n 100 http://172.25.254.53/index.html    【压力测试】