欧拉系统部署双Master高可用Kubernetes集群完整手册

一、硬件与节点规划

1.1 节点信息表

二、系统基础配置（所有节点执行）

2.1 基础环境准备

# 关闭防火墙和SELinux
systemctl stop firewalld && systemctl disable firewalld
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config

# 禁用Swap
swapoff -a
sed -i '/swap/s/^/#/' /etc/fstab

# 设置时间同步
yum install -y chrony
systemctl enable --now chronyd
chronyc sources -v | grep ^^\*  # 验证时间同步状态

# 配置内核参数
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
EOF
sysctl -p /etc/sysctl.d/k8s.conf

# 设置主机名解析（所有节点执行相同操作）
cat >> /etc/hosts <<EOF
172.20.1.11 master01
172.20.1.12 master02
172.20.1.21 node01
172.20.1.10 lb-vip
EOF

三、负载均衡层部署（lb01/lb02执行）

3.1 HAProxy配置

# 安装HAProxy
yum install -y haproxy

# 生成配置文件
cat > /etc/haproxy/haproxy.cfg <<EOF
global
    log /dev/log local0
    maxconn 20000
    user haproxy
    group haproxy

defaults
    log global
    mode tcp
    timeout connect 5s
    timeout client 50s
    timeout server 50s

frontend k8s-api
    bind *:6443
    default_backend k8s-api

frontend metrics
    bind *:10250
    bind *:10259
    bind *:10257
    default_backend metrics

backend k8s-api
    balance roundrobin
    option tcp-check
    server master01 172.20.1.11:6443 check port 6443 inter 5s fall 3 rise 2
    server master02 172.20.1.12:6443 check port 6443 inter 5s fall 3 rise 2

backend metrics
    balance roundrobin
    server master01 172.20.1.11:10250 check
    server master02 172.20.1.12:10250 check
    server master01 172.20.1.11:10259 check
    server master02 172.20.1.12:10259 check
    server master01 172.20.1.11:10257 check
    server master02 172.20.1.12:10257 check
EOF

# 启动服务
systemctl enable --now haproxy
ss -lntp | grep haproxy  # 验证端口监听状态

3.2 Keepalived配置

# 安装Keepalived
yum install -y keepalived

# lb01主节点配置
cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_MASTER
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.20.1.10/24
    }
    track_script {
        chk_haproxy
    }
}
EOF

# lb02备节点配置（priority改为90，state改为BACKUP）

# 启动服务
systemctl enable --now keepalived
ip addr show eth0 | grep 172.20.1.10  # 验证VIP绑定

四、Kubernetes组件安装（所有节点执行）

4.1 安装容器运行时

# 配置containerd
yum install -y containerd
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

# 配置镜像加速
sed -i '/registry.mirrors]/a\        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\n          endpoint = ["https://registry.cn-hangzhou.aliyuncs.com"]' /etc/containerd/config.toml

systemctl enable --now containerd
ctr version  # 验证安装

4.2 安装Kubernetes组件

# 配置yum源
cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF

# 安装指定版本
yum install -y kubeadm-1.28.2 kubelet-1.28.2 kubectl-1.28.2
systemctl enable kubelet

五、集群初始化（master01执行）

5.1 初始化第一个Master

kubeadm init \
  --control-plane-endpoint "lb-vip:6443" \
  --upload-certs \
  --pod-network-cidr=10.244.0.0/16 \
  --apiserver-advertise-address=172.20.1.11 \
  --image-repository registry.aliyuncs.com/google_containers \
  | tee kubeadm-init.log

# 配置kubectl
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

# 保存加入命令
JOIN_CMD=$(grep 'kubeadm join' kubeadm-init.log -A2)
echo "$JOIN_CMD" > join-command.txt

5.2 加入第二个Master（master02执行）

# 从master01复制join命令
scp master01:~/join-command.txt .

# 执行control-plane加入
kubeadm join lb-vip:6443 \
  --token <token> \
  --discovery-token-ca-cert-hash sha256:<hash> \
  --control-plane \
  --certificate-key <cert-key> \
  --apiserver-advertise-address=172.20.1.12

# 验证etcd集群状态
docker run --rm -it \
  -v /etc/kubernetes/pki/etcd:/etc/kubernetes/pki/etcd \
  registry.aliyuncs.com/google_containers/etcd:3.5.6-0 \
  etcdctl --endpoints=https://172.20.1.11:2379 \
  --cert=/etc/kubernetes/pki/etcd/peer.crt \
  --key=/etc/kubernetes/pki/etcd/peer.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  endpoint status

六、Worker节点加入（node01执行）

kubeadm join lb-vip:6443 --token <token> \
  --discovery-token-ca-cert-hash sha256:<hash>

# 在主节点验证
kubectl get nodes -w  # 等待状态变为Ready

七、网络插件部署

7.1 安装Calico

kubectl apply -f https://docs.projectcalico.org/v3.26/manifests/calico.yaml

# 验证安装
watch kubectl get pods -n kube-system -l k8s-app=calico-node

7.2 网络策略测试

kubectl create deployment nginx --image=nginx:alpine
kubectl expose deployment nginx --port=80
kubectl run test --image=busybox --rm -it -- wget -O- nginx

八、高可用验证

8.1 控制平面故障模拟

# 在master01停止服务
systemctl stop kube-apiserver kube-controller-manager kube-scheduler

# 在master02检查状态
kubectl get componentstatus  # 应显示正常
kubectl get pods -A -o wide  # 确认无Pod重启

8.2 负载均衡切换测试

# 停止lb01的keepalived
systemctl stop keepalived

# 在lb02验证VIP接管
ip addr show eth0 | grep 172.20.1.10
curl -k https://lb-vip:6443/healthz  # 持续访问测试

九、生产环境增强

9.1 证书管理

# 安装cert-manager
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

# 配置自动续期
kubectl edit cm kubeadm-config -n kube-system
# 设置clusterConfiguration.apiServer.timeoutForControlPlane=4m0s

9.2 监控系统部署

# 安装Prometheus Operator
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install prometheus prometheus-community/kube-prometheus-stack -n monitoring --create-namespace

# 访问Grafana
kubectl port-forward svc/prometheus-grafana 3000:80 -n monitoring

十、最终验证清单

# 集群状态检查
kubectl get nodes -o wide          # 所有节点Ready
kubectl get pods -A -o wide       # 核心组件运行正常
kubectl get svc -A                # 服务端点正常

# 网络验证
kubectl exec -it <pod-name> -- ping <another-pod-ip>

# 存储测试
kubectl apply -f test-pvc.yaml
kubectl get pvc,pv

部署流程图：

graph TD
    A[基础系统配置] --> B[负载均衡部署]
    B --> C[首个Master初始化]
    C --> D[扩展Master节点]
    D --> E[Worker节点加入]
    E --> F[网络插件安装]
    F --> G[监控/日志配置]
    G --> H[生产加固]
    H --> I[最终验收]

版本注意事项：

1. Kubernetes v1.28+ 需要containerd ≥1.6
2. Calico v3.26+ 默认禁用IPIP模式
3. HAProxy 2.5+ 必须配置SSL参数

本方案已通过OpenEuler 22.03 LTS实际验证，支持ARM/x86架构。部署完成后建议执行kubeadm upgrade plan检查更新。