Multi-Master Cluster Deployment

info

Minimum node requirements: 3 nodes.
Master node redundancy is supported; if one Master fails, the cluster can still operate and run workloads normally.
For both Standard and Professional clusters, a multi-Master deployment is used when there are three microservice nodes.

This document details the deployment of a Kubernetes cluster based on the CentOS 7.9 / Debian 12 operating system.

Server IP	Host Role
192.168.10.20	Kubernetes 01 (Master, Node)
192.168.10.21	Kubernetes 02 (Master, Node)
192.168.10.22	Kubernetes 03 (Master, Node)

Server Requirements

No network policy restrictions between the cluster servers
Hostnames between cluster servers must not duplicate
Primary network card MAC addresses must not duplicate [Check using ip link]
product_id must not duplicate [Check using cat /sys/class/dmi/id/product_uuid]
Port 6443 for kubelet must not be occupied [Check using nc -vz 127.0.0.1 6443]
Disable swap memory [Execute the swapoff -a command to disable, and disable swap partition mounting in /etc/fstab]

Configure HOSTS

Add the following hosts information to each node in the Kubernetes cluster, pointing k8s-master to the three master nodes.

cat >> /etc/hosts << EOF
192.168.10.20 k8s-master
192.168.10.21 k8s-master
192.168.10.22 k8s-master
EOF

Note: This hosts information must be added to every node in the Kubernetes cluster, including any nodes added later.

Install the CRI Container Runtime Environment

Operations must be carried out on each node of the Kubernetes cluster.

Download the Docker installation package

Server has internet access
Server does not support internet access

wget https://pdpublic.mingdao.com/private-deployment/offline/common/docker-27.3.1.tgz

# Docker installation package download link; download and upload to the target server
https://pdpublic.mingdao.com/private-deployment/offline/common/docker-27.3.1.tgz

Install Docker

tar -zxvf docker-27.3.1.tgz
mv -f docker/* /usr/local/bin/

Create directories for Docker and Containerd configuration files
```
mkdir /etc/docker
mkdir /etc/containerd
```

Create the daemon.json file for Docker

cat > /etc/docker/daemon.json <<\EOF
{
"registry-mirrors": ["https://uvlkeb6d.mirror.aliyuncs.com"],
"data-root": "/data/docker",
"max-concurrent-downloads": 10,
"exec-opts": ["native.cgroupdriver=cgroupfs"],
"storage-driver": "overlay2",
"default-address-pools":[{"base":"172.80.0.0/16","size":24}],
"insecure-registries": ["127.0.0.1:5000"]
}
EOF

Create the config.toml file for Containerd

cat > /etc/containerd/config.toml <<\EOF
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/data/containerd"
state = "/run/containerd"
temp = ""
version = 2

[cgroup]
path = ""

[debug]
address = ""
format = ""
gid = 0
level = ""
uid = 0

[grpc]
address = "/var/run/containerd/containerd.sock"
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
tcp_address = ""
tcp_tls_ca = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0

[metrics]
address = ""
grpc_histogram = false

[plugins]

[plugins."io.containerd.gc.v1.scheduler"]
    deletion_threshold = 0
    mutation_threshold = 100
    pause_threshold = 0.02
    schedule_delay = "0s"
    startup_delay = "100ms"

[plugins."io.containerd.grpc.v1.cri"]
    device_ownership_from_security_context = false
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = true
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    enable_unprivileged_icmp = false
    enable_unprivileged_ports = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 16384
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "127.0.0.1:5000/pause:3.8"
    selinux_category_range = 1024
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = true
    unset_seccomp_profile = ""

    [plugins."io.containerd.grpc.v1.cri".cni]
    bin_dir = "/usr/local/kubernetes/cni/bin"
    conf_dir = "/etc/cni/net.d"
    conf_template = ""
    ip_pref = ""
    max_conf_num = 1

    [plugins."io.containerd.grpc.v1.cri".containerd]
    default_runtime_name = "runc"
    disable_snapshot_annotations = true
    discard_unpacked_layers = false
    ignore_rdt_not_enabled_errors = false
    no_pivot = false
    snapshotter = "overlayfs"

    [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = "io.containerd.runc.v2"

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            BinaryName = ""
            CriuImagePath = ""
            CriuPath = ""
            CriuWorkPath = ""
            IoGid = 0
            IoUid = 0
            NoNewKeyring = false
            NoPivotRoot = false
            Root = ""
            ShimCgroup = ""
            SystemdCgroup = true

    [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".image_decryption]
    key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]
    config_path = ""

    [plugins."io.containerd.grpc.v1.cri".registry.auths]

    [plugins."io.containerd.grpc.v1.cri".registry.configs]

    [plugins."io.containerd.grpc.v1.cri".registry.headers]

    [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
    tls_cert_file = ""
    tls_key_file = ""

[plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"

[plugins."io.containerd.internal.v1.restart"]
    interval = "10s"

[plugins."io.containerd.internal.v1.tracing"]
    sampling_ratio = 1.0
    service_name = "containerd"

[plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"

[plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false

[plugins."io.containerd.runtime.v1.linux"]
    no_shim = false
    runtime = "runc"
    runtime_root = ""
    shim = "containerd-shim"
    shim_debug = false

[plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
    sched_core = false

[plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]

[plugins."io.containerd.service.v1.tasks-service"]
    rdt_config_file = ""

[plugins."io.containerd.snapshotter.v1.aufs"]
    root_path = ""

[plugins."io.containerd.snapshotter.v1.btrfs"]
    root_path = ""

[plugins."io.containerd.snapshotter.v1.devmapper"]
    async_remove = false
    base_image_size = ""
    discard_blocks = false
    fs_options = ""
    fs_type = ""
    pool_name = ""
    root_path = ""

[plugins."io.containerd.snapshotter.v1.native"]
    root_path = ""

[plugins."io.containerd.snapshotter.v1.overlayfs"]
    root_path = ""
    upperdir_label = false

[plugins."io.containerd.snapshotter.v1.zfs"]
    root_path = ""

[plugins."io.containerd.tracing.processor.v1.otlp"]
    endpoint = ""
    insecure = false
    protocol = ""

[proxy_plugins]

[stream_processors]

[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar"

[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
"io.containerd.timeout.bolt.open" = "0s"
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"

[ttrpc]
address = ""
gid = 0
uid = 0
EOF

6. Configure the systemd file for docker

cat > /etc/systemd/system/docker.service <<EOF
[Unit]
Description=Docker
After=network-online.target
Wants=network-online.target
Requires=containerd.service
[Service]
Type=notify
ExecStart=/usr/local/bin/dockerd --containerd /var/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=1024000
LimitNPROC=infinity
LimitCORE=0
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF

7. Configure the systemd file for containerd

cat > /etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/local/bin/containerd --config /etc/containerd/config.toml
LimitNOFILE=1024000
LimitNPROC=infinity
LimitCORE=0
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF

8. Start containerd and docker and enable them to start at boot

systemctl daemon-reload && systemctl restart containerd && systemctl enable containerd
systemctl daemon-reload && systemctl restart docker && systemctl enable docker

Install CNI Plugins

This operation is required on all nodes of the Kubernetes cluster

Download the CNI plugin files

Server with Internet Access
Server without Internet Access

wget https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/cni-plugins-linux-amd64-v1.1.1.tgz

# CNI plugin package download link, download and upload to target server
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/cni-plugins-linux-amd64-v1.1.1.tgz

Create the CNI file installation directory

mkdir -p /usr/local/kubernetes/cni/bin

Extract the CNI plugin to the installation directory

tar -zxvf cni-plugins-linux-amd64-v1.1.1.tgz -C /usr/local/kubernetes/cni/bin

Install K8S Cluster Commands

Install crictl/kubeadm/kubelet/kubectl commands, this operation is required on all nodes of the Kubernetes cluster

Create the command installation directory

mkdir -p /usr/local/kubernetes/bin

Download command files to the installation directory

Server with Internet Access
Server without Internet Access

wget https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/crictl-v1.25.0-linux-amd64.tar.gz
tar -zxvf crictl-v1.25.0-linux-amd64.tar.gz -C /usr/local/kubernetes/bin
curl -o /usr/local/kubernetes/bin/kubeadm https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubeadm
curl -o /usr/local/kubernetes/bin/kubelet https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubelet
curl -o /usr/local/kubernetes/bin/kubectl https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubectl

# crictl file download link, download and upload to target server, then extract to /usr/local/kubernetes/bin
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/crictl-v1.25.0-linux-amd64.tar.gz
tar -zxvf crictl-v1.25.0-linux-amd64.tar.gz -C /usr/local/kubernetes/bin

# kubeadm file download link, download and upload to target server /usr/local/kubernetes/bin/
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubeadm

# kubelet file download link, download and upload to target server /usr/local/kubernetes/bin/
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubelet

# kubectl file download link, download and upload to target server /usr/local/kubernetes/bin/
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubectl

Grant executable permissions to the command files

chmod +x /usr/local/kubernetes/bin/*
chown $(whoami):$(groups) /usr/local/kubernetes/bin/*

Configure systemd to manage kubelet

cat > /etc/systemd/system/kubelet.service <<\EOF
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/home/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/local/kubernetes/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

Configure systemd to manage kubeadm

mkdir -p /etc/systemd/system/kubelet.service.d

cat > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<\EOF
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/local/kubernetes/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
EOF

Start kubelet and enable it to start at boot

systemctl daemon-reload && systemctl restart kubelet && systemctl enable kubelet

There is no need to check the service status after restarting here, as the service will automatically start after subsequent kubeadm init and kubeadm join steps.

Set the directory for K8S commands and add to environment variables

CentOS
Debian

export PATH=/usr/local/kubernetes/bin/:$PATH
echo 'export PATH=/usr/local/kubernetes/bin/:$PATH' >> /etc/bashrc

export PATH=/usr/local/kubernetes/bin/:$PATH
echo 'export PATH=/usr/local/kubernetes/bin/:$PATH' >> /etc/bash.bashrc

Configure to prevent errors when pulling images with crictl later

crictl config runtime-endpoint unix:///run/containerd/containerd.sock

Install Environment Dependencies

This operation is required on all nodes of the Kubernetes cluster

Install environment dependencies socat/conntrack

Server with Internet Access
Server without Internet Access

# Use yum to install on CentOS/Red Hat
yum install -y socat conntrack-tools

# Use apt to install on Debian/Ubuntu
apt install -y socat conntrack

# Socat package download link, download and upload to target server (using CentOS 7.9 here, re-download if dependencies do not match)
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/socat-deps-centos7.tar.gz

# Extract and install
tar -zxvf socat-deps-centos7.tar.gz
rpm -Uvh --nodeps socat-deps-centos7/*.rpm

# Conntrack package download link, download and upload to target server (using CentOS 7.9 here, re-download if dependencies do not match)
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/conntrack-tools-deps-centos7.tar.gz

# Extract and install
tar -zxvf conntrack-tools-deps-centos7.tar.gz
rpm -Uvh --nodeps conntrack-tools-deps-centos7/*.rpm

Check if any command is missing

docker --version && dockerd --version && pgrep -f 'dockerd' && crictl --version && kubeadm version && kubelet --version && kubectl version --client=true && socat -V | grep 'socat version' && conntrack --version && echo ok || echo error

Outputting ok means everything is correct; if error is shown, investigate and resolve the missing commands.

Modify Kernel Configuration

This operation is required on all nodes of the Kubernetes cluster

Add kernel modules

cat > /etc/modules-load.d/kubernetes.conf <<EOF
overlay
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
EOF

Load the modules

modprobe overlay
modprobe br_netfilter
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh

Add kernel parameters

cat >> /etc/sysctl.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
vm.max_map_count = 262144

# MD Config
net.nf_conntrack_max = 524288
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 8192 87380 16777216
net.ipv4.tcp_wmem = 8192 65536 16777216
net.ipv4.tcp_max_syn_backlog = 32768
net.core.netdev_max_backlog =  32768
net.core.netdev_budget = 600
net.core.somaxconn = 32768
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 2
net.ipv4.tcp_mem = 8388608 12582912 16777216
net.ipv4.ip_local_port_range = 1024 65000
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_keepalive_intvl = 10
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_time = 600
vm.max_map_count = 262144
net.netfilter.nf_conntrack_tcp_be_liberal = 0
net.netfilter.nf_conntrack_tcp_max_retrans = 3
net.netfilter.nf_conntrack_tcp_timeout_max_retrans = 300
net.netfilter.nf_conntrack_tcp_timeout_established = 86400
fs.inotify.max_user_watches=10485760
fs.inotify.max_user_instances=10240
EOF

sysctl --system

Prepare K8S Environment Images

This operation is required on all nodes of the Kubernetes cluster

Load offline images

Server with Internet Access
Server without Internet Access

wget https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubeadm-1.25.4-images.tar.gz
docker load -i kubeadm-1.25.4-images.tar.gz

# Offline image package download link, download and upload to target server and load the image
https://pdpublic.mingdao.com/private-deployment/offline/common/kubernetes-1.25.4/kubeadm-1.25.4-images.tar.gz
docker load -i kubeadm-1.25.4-images.tar.gz

Start a local repository to tag the images

docker run -d -p 5000:5000 --restart always --name registry registry:2
for i in $(docker images | grep 'registry.k8s.io\|rancher' | awk 'NR!=0{print $1":"$2}');do docker tag $i $(echo $i | sed -e "s/registry.k8s.io/127.0.0.1:5000/" -e "s#coredns/##" -e "s/rancher/127.0.0.1:5000/");done
for i in $(docker images | grep :5000 | awk 'NR!=0{print $1":"$2}');do docker push $i;done
docker images | grep :5000

Initialize the First Master Node

Only operate on the Kubernetes 01 node

Initialize the master node

kubeadm init --control-plane-endpoint "k8s-master:6443" --upload-certs --cri-socket unix:///var/run/containerd/containerd.sock -v 5 --kubernetes-version=1.25.4 --image-repository=127.0.0.1:5000 --pod-network-cidr=10.244.0.0/16

The final output will look similar to:

...
You can now join any number of control-plane nodes by running the following command on each as root:
    kubeadm join k8s-master:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07

Please note that the certificate-key gives access to cluster-sensitive data, keep it secret!
As a safeguard, uploaded certificates will be deleted in two hours; if necessary, you can use kubeadm init phase upload-certs to reload certs afterward.

Then, you can join any number of worker nodes by running the following on each as root:
    kubeadm join k8s-master:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866

Copy this output to a text file. You will need it later to join master and node nodes to the cluster.

Modify the nodePort usable port range

sed -i '/- kube-apiserver/a\ \ \ \ - --service-node-port-range=1024-32767' /etc/kubernetes/manifests/kube-apiserver.yaml

Set the configuration path

CentOS
Debian

export KUBECONFIG=/etc/kubernetes/admin.conf
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/bashrc

export KUBECONFIG=/etc/kubernetes/admin.conf
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/bash.bashrc

Adjust the current node Pod limit

echo "maxPods: 300" >> /var/lib/kubelet/config.yaml
systemctl restart kubelet

Allow the master to participate in scheduling
- Wait about 1-2 minutes after initializing the master node before executing the following command
- Before executing, check the status of the kubelet service with systemctl status kubelet, to see if it is running
```
kubectl taint node $(kubectl get node | grep control-plane | awk '{print $1}') node-role.kubernetes.io/control-plane:NoSchedule-
```
- After executing this command, the correct output should be: "xxxx untainted". If the output does not match, wait a bit and execute it again for confirmation.

Install the network plugin

cat > /usr/local/kubernetes/kube-flannel.yml <<EOF
---
kind: Namespace
apiVersion: v1
metadata:
  name: kube-flannel
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
       #image: flannelcni/flannel-cni-plugin:v1.1.0 for ppc64le and mips64le (dockerhub limitations may apply)
        image: 127.0.0.1:5000/mirrored-flannelcni-flannel-cni-plugin:v1.1.0
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
       #image: flannelcni/flannel:v0.20.1 for ppc64le and mips64le (dockerhub limitations may apply)
        image: 127.0.0.1:5000/mirrored-flannelcni-flannel:v0.20.1
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
       #image: flannelcni/flannel:v0.20.1 for ppc64le and mips64le (dockerhub limitations may apply)
        image: 127.0.0.1:5000/mirrored-flannelcni-flannel:v0.20.1
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /usr/local/kubernetes/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
EOF

kubectl apply -f /usr/local/kubernetes/kube-flannel.yml

Join Other Master Nodes to the Cluster

You need to perform operations on Kubernetes 02/03 nodes

Join the Kubernetes cluster

kubeadm join k8s-master:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07

This command is the output of executing kubeadm init successfully on the master node. This is just an example; each cluster is different.
If forgotten, refer to the following steps to regenerate on the first master node:
1. Regenerate the join command
```
kubeadm token create --print-join-command
```
2. Re-upload the certificates and generate a new decryption key
```
kubeadm init phase upload-certs --upload-certs
```
3. Concatenate the join command, adding --control-plane and --certificate-key parameters, and use the generated decryption key as the --certificate-key parameter value
```
kubeadm join k8s-master:6443 --token 1b6i9d.0qqufwsjrjpuhkwo --discovery-token-ca-cert-hash sha256:3d28faa49e9cac7dd96aded0bef33a6af1ced57e45f0b12c6190f3d4e1055456 --control-plane --certificate-key 57a0f0e9be1d9f1c74bab54a52faa143ee9fd9c26a60f1b3b816b17b93ecaf6f
```
  - At this point, you have obtained the join command to add master nodes to the cluster

Modify the nodePort usable port range

sed -i '/- kube-apiserver/a\ \ \ \ - --service-node-port-range=1024-32767' /etc/kubernetes/manifests/kube-apiserver.yaml

Set the configuration path

CentOS
Debian

export KUBECONFIG=/etc/kubernetes/admin.conf
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/bashrc

export KUBECONFIG=/etc/kubernetes/admin.conf
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/bash.bashrc

Adjust the current node Pod limit

echo "maxPods: 300" >> /var/lib/kubelet/config.yaml
systemctl restart kubelet

Allow the master node to participate in scheduling
- Wait about 1-2 minutes after finishing the current node initialization before executing the command below
- Before executing, check the status of the kubelet service with systemctl status kubelet, to see if it is running
```
kubectl taint node $(kubectl get node | grep control-plane | awk '{print $1}') node-role.kubernetes.io/control-plane:NoSchedule-
```
- After executing this command, the correct output should be: "xxxx untainted". If the output does not match, wait a bit and execute it again for confirmation

Add New Worker Nodes to the Cluster

For example, Flink nodes or subsequently added microservice nodes that join the current multi-master Kubernetes cluster as worker nodes

Join the Kubernetes cluster
```
kubeadm join 192.168.10.20:6443 --token 3nwjzw.pdod3r27lnqqhi0x \
    --discovery-token-ca-cert-hash sha256:a84445303a0f8249e7eae3059cb99d46038dc275b2dc2043a022de187a1175a2
```
- This command is the output of executing kubeadm init successfully on the master node, serving as an example. Each cluster is different.
- If forgotten, you can re-obtain it by executing kubeadm token create --print-join-command on the master node.

Adjust the current node Pod limit

echo "maxPods: 300" >> /var/lib/kubelet/config.yaml
systemctl restart kubelet

Check Cluster Status

Check node status

kubectl get pod -n kube-system    # The READY column needs to be "1/1"
kubectl get node                  # The STATUS column needs to be "Ready"

Download the image (each microservice node needs to operate)

Download and upload the centos:7.9.2009 image to each server in advance

Offline image download link: https://pdpublic.mingdao.com/private-deployment/offline/common/centos7.9.2009.tar.gz

Load offline image on each server:
```
gunzip -d centos7.9.2009.tar.gz
ctr -n k8s.io image import centos7.9.2009.tar
```

Only write configuration on microservices node 01 to start the test container

cat > /usr/local/kubernetes/test.yaml <<\EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: test
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: test
  template:
    metadata:
      labels:
        app: test
      annotations:
        md-update: '20200517104741'
    spec:
      containers:
      - name: test
        image: centos:7.9.2009
        command:
        - sh
        - -c
        - |
          echo $(hostname) > hostname.txt
          python -m SimpleHTTPServer
        resources:
          limits:
            memory: 512Mi
            cpu: 1
          requests:
            memory: 64Mi
            cpu: 0.01
        volumeMounts:
        - name: tz-config
          mountPath: /etc/localtime
      volumes:
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Etc/GMT-8

---

apiVersion: v1
kind: Service
metadata:
  name: test
  namespace: default
spec:
  selector:
    app: test
  ports:
  - name: external-test
    port: 8000
    targetPort: 8000
    nodePort: 8000
  type: NodePort
EOF

kubectl apply -f /usr/local/kubernetes/test.yaml

Check Pod status
```
kubectl get pod -o wide
```
Test access
```
curl 127.0.0.1:8000/hostname.txt
```
- Multiple curls should normally return the hostname of different Pods

If the curl reaches containers on other nodes, it may take about 1 second to return. In such cases, disable the hardware offload function of the flannel.1 network interface (needs configuration on every node in the Kubernetes cluster)

cat > /etc/systemd/system/disable-offload.service <<\EOF
[Unit]
Description=Disable offload for flannel.1
After=network-online.target flanneld.service

[Service]
Type=oneshot
ExecStartPre=/bin/bash -c 'while [ ! -d /sys/class/net/flannel.1 ]; do sleep 1; done'
ExecStart=/sbin/ethtool --offload flannel.1 rx off tx off

[Install]
WantedBy=multi-user.target
EOF

Reload the systemd configuration and start the service

systemctl daemon-reload
systemctl enable disable-offload
systemctl start disable-offload

Server Requirements​

Configure HOSTS​

Install the CRI Container Runtime Environment​

6. Configure the systemd file for docker​

7. Configure the systemd file for containerd​

8. Start containerd and docker and enable them to start at boot​

Install CNI Plugins​

Install K8S Cluster Commands​

Install Environment Dependencies​

Modify Kernel Configuration​

Prepare K8S Environment Images​

Initialize the First Master Node​

Join Other Master Nodes to the Cluster​

Add New Worker Nodes to the Cluster​

Check Cluster Status​

Server Requirements

Configure HOSTS

Install the CRI Container Runtime Environment

6. Configure the systemd file for docker

7. Configure the systemd file for containerd

8. Start containerd and docker and enable them to start at boot

Install CNI Plugins

Install K8S Cluster Commands

Install Environment Dependencies

Modify Kernel Configuration

Prepare K8S Environment Images

Initialize the First Master Node

Join Other Master Nodes to the Cluster

Add New Worker Nodes to the Cluster

Check Cluster Status