KnoAI 技术站

主题

rocky9.8 rke2 1.35.6 ​

PowerShell
wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.33.13/rke2.linux-amd64.tar.
gz
wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.33.13/rke2-images.linux-amd
64.tar.zst
wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.33.13/rke2-images-calico.linux-amd64.tar.zst 

wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.34.8/rke2-images.linux-amd64-1.34.8.tar.zst
wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.34.8/rke2-images-calico.linux-amd64.tar.zst  

wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.36.2/rke2-images.linux-amd64-1.36.2.tar.zst
wget --user=<USER> --password=<PASSWORD> http://192.168.122.1:9001/repo/rke2/1.36.2/rke2-images-calico.linux-amd64-1.36.2.tar.zst
PowerShell
curl -u <USER>:<PASSWORD> 192.168.122.1:9001/repo/rocky/9.8/init_kernel.sh |sh
PowerShell
5 台 VM 网络信息
  ┌──────────────────┬─────────────────┬───────────────────┐
  │ 虚拟机           │ IP 地址         │ MAC 地址          │
  ├──────────────────┼─────────────────┼───────────────────┤
  │ rancher-ctrl-vm1 │ 192.168.122.227 │ 52:54:00:2d:49:5e │
  │ rancher-ctrl-vm2 │ 192.168.122.216 │ 52:54:00:46:84:49 │
  │ rancher-ctrl-vm3 │ 192.168.122.164 │ 52:54:00:b3:bf:c2 │
  │ rancher-ctrl-vm4 │ 192.168.122.40  │ 52:54:00:25:7d:31 │
  │ rancher-ctrl-vm5 │ 192.168.122.101 │ 52:54:00:0d:39:db │
  └──────────────────┴─────────────────┴───────────────────┘

使用harbor仓库部署 ​

PowerShell
rm -rf /var/lib/rancher # 失败
#  查看 immutable 属性
lsattr /var/lib/rancher/rke2/agent/etc/containerd/certs.d/*/hosts.toml
# 去掉 immutable 标志
chattr -i /var/lib/rancher/rke2/agent/etc/containerd/certs.d/*/hosts.toml
 rm -rf /var/lib/rancher/
 
curl -s http://192.168.122.1:31000/repo/rke2/1.35.6/rke2.linux-amd64.tar.gz | tar zxf - -C /usr/local

mkdir -p /etc/rancher/rke2
cd /etc/rancher/rke2

cat > registries.yaml <<EOF
configs:
  "192.168.122.156:30000":
    tls:
      insecure_skip_verify: true
mirrors:
  "192.168.122.156:30000":
    endpoint:
      - "http://192.168.122.156:30000"
  "docker.io":
    endpoint:
      - "http://192.168.122.156:30000"
EOF

#server: https://192.168.122.190:9345
#token: xxx 


cat > config.yaml <<EOF
cni: calico
tls-san:
  - 192.168.122.227
  - 192.168.122.216
  - 192.168.122.164
node-ip: $(hostname -I|awk '{print $1}')
cluster-cidr: 10.12.0.0/16
service-cidr: 10.13.0.0/16
kube-proxy-arg:
  - "proxy-mode=ipvs"
system-default-registry: 192.168.122.156:30000
etcd-snapshot-retention: 5
etcd-snapshot-schedule-cron: "0 */6 * * *"
EOF

systemctl enable --now rke2-server
# systemctl enable --now rke2-agent


kubectl label node rocky9-vm-04 node-role.kubernetes.io/worker=true
kubectl label node rocky9-vm-05 node-role.kubernetes.io/worker=true
kubectl label node rocky9-vm-06 node-role.kubernetes.io/worker=true

rancher ​

PowerShell
helm pull oci://192.168.122.156:30000/cert-manager/cert-manager --version v1.20.3
helm install cert-manager ./cert-manager-v1.20.3.tgz \
  --namespace cert-manager \
  --create-namespace \
  --set installCRDs=true \
  --set global.imageRegistry=192.168.122.156:30000
  
helm pull oci://192.168.122.156:30000/cnrancher/rancher --version 2.14.3
helm upgrade --install rancher ./rancher-2.14.3.tgz \
--namespace cattle-system --create-namespace \
--set hostname=rancher.ai-ear.cn \
--set bootstrapPassword="xxx" \
--set replicas==1 \
--set ingress.ingressClassName=nginx \
--set global.cattle.psp.enabled=false \
--set systemDefaultRegistry=192.168.122.156:30000 \
--set rancherImage=192.168.122.156:30000/rancher/rancher \
--set rancherImageTag=v2.14.3



# echo https://rancher.ai-ear.cn/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')
# ---
# To get just the bootstrap password on its own, run:
# ---
# kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{ "\n" }}'

vip ​

PowerShell
# ip addr add 192.168.112.253/24 dev eth0
Bash
#!/bin/bash
set -e
echo "============================================="
echo "Rocky Linux9 RKE2 1.35.6 + 默认Canal 内核优化脚本"
echo "节点规格：8C16G 内置Canal(Calico+Flannel)"
echo "============================================="
dnf install -y iptables
rm -rf /var/lib/cni/networks/k8s-pod-network/
# 1. 关闭SELinux
echo "[1/9] 关闭SELinux"
setenforce 0
sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

# 2. 关闭Swap
echo "[2/9] 关闭Swap分区"
swapoff -a
sed -i '/swap/s/^/#/' /etc/fstab

# 3. 预加载K8s内核模块
echo "[3/9] 加载overlay/ipvs/nf_conntrack模块"
cat > /etc/modules-load.d/k8s-canal.conf <<EOF
overlay
ip_tables
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
nf_tables
EOF
modprobe overlay
modprobe ip_tables
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack
modprobe nf_tables

# 4. 写入sysctl内核调优参数（8核16G，Canal专用）
echo "[4/9] 写入sysctl优化参数 /etc/sysctl.d/99-rke2-canal.conf"
cat > /etc/sysctl.d/99-rke2-canal.conf <<EOF
# IPv4转发 & 网桥iptables（Canal强制依赖）
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1

# conntrack 连接跟踪表项 16G内存规格
net.netfilter.nf_conntrack_max = 2621440
net.netfilter.nf_conntrack_tcp_timeout_established = 300
net.netfilter.nf_conntrack_tcp_timeout_time_wait = 60
net.netfilter.nf_conntrack_tcp_timeout_close_wait = 30
net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 30

# TCP 高并发/隧道优化
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 3

# 本地端口范围扩容
net.ipv4.ip_local_port_range = 1024 65535

# 网络缓冲区
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.rmem_default = 262144
net.core.wmem_default = 262144
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# 连接队列缓冲
net.core.somaxconn = 32768
net.core.netdev_max_backlog = 16384

# IPVS优化
net.ipv4.vs.expire_nodest_conn = 1
net.ipv4.vs.expire_quiescent = 1

# 关闭IPv6 RA广播
net.ipv6.conf.all.accept_ra = 0
net.ipv6.conf.default.accept_ra = 0

# 内存脏页&容器内存参数
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
vm.swappiness = 0
vm.max_map_count = 262144
EOF

# 5. 加载生效sysctl
echo "[5/9] 生效所有内核参数"
sysctl --system

# 6. 防火墙处理（测试环境直接关闭；生产建议放行端口）
echo "[6/9] 关闭firewalld（生产环境自行放行RKE2/Canal端口）"
#systemctl stop firewalld
# systemctl disable firewalld

# 7. 调整系统文件句柄上限
echo "[7/9] 调整系统文件句柄限制"
cat >> /etc/security/limits.conf <<EOF
root soft nofile 1048576
root hard nofile 1048576
* soft nofile 1048576
* hard nofile 1048576
EOF

# 8. systemd全局文件句柄优化
echo "[8/9] systemd全局nofile优化"
sed -i '/DefaultLimitNOFILE/d' /etc/systemd/system.conf
echo "DefaultLimitNOFILE=1048576" >> /etc/systemd/system.conf
systemctl daemon-reload

# 9. 校验输出
echo "============================================="
echo "[9/9] 关键参数校验结果："
echo "nf_conntrack_max: $(sysctl -n net.netfilter.nf_conntrack_max)"
echo "ip_forward: $(sysctl -n net.ipv4.ip_forward)"
echo "vm.max_map_count: $(sysctl -n vm.max_map_count)"
echo "bridge-nf-call-iptables: $(sysctl -n net.bridge.bridge-nf-call-iptables)"
lsmod | grep -E "overlay|ip_vs|nf_conntrack|nf_tables"
echo "============================================="
echo "优化完成！limits参数重启服务器后完全生效"
echo "config.yaml 无需 cni:none、不disable canal，使用默认内置Canal"
PowerShell
# 批量终止所有 rke2 相关 gpg-agent / scdaemon
pkill -f "gpg-agent --homedir /var/cache/dnf/rancher-rke2"
pkill -f "scdaemon --homedir /var/cache/dnf/rancher-rke2"

# 确认是否杀干净
ps aux | grep -E "gpg-agent|scdaemon" | grep rancher-rke2
PowerShell
curl -s http://192.168.112.97:31000/repo/rke2-canal-kernel-tune.sh |sh
PowerShell
yum install -y wget
cd /opt
wget http://192.168.112.97:31000/repo/rke2/rke2-1.35.6.tar.gz
tar xfv rke2-1.35.6.tar.gz
cd rke2-1.35.6
tar -xzf rke2.linux-amd64.tar.gz -C /usr/local
mkdir -p /var/lib/rancher/rke2/agent/images/
cp rke2-images.linux-amd64.tar.* /var/lib/rancher/rke2/agent/images/
mkdir -p /etc/rancher/rke2
JavaScript


cat > /etc/rancher/rke2/config.yaml <<EOF
disable-scheduler-node-taints: true
kube-proxy-arg:
  - "proxy-mode=ipvs"
node-name: $(hostname -I|awk '{print $1}')
data-dir: /var/lib/rancher/rke2
etcd-snapshot-retention: 5
etcd-snapshot-schedule-cron: "0 */6 * * *"
service-node-port-range: "30000-60000"
EOF
Plain
docker login -u admin -p 密码 harbor.shbayes.com
YAML
cat <<EOF > /etc/rancher/rke2/registries.yaml
mirrors:
  "harbor.shbayes.com":
    endpoint:
      - "https://harbor.shbayes.com"
configs:
  "harbor.shbayes.com":
    tls:
      insecure_skip_verify: true
      
EOF
PowerShell

cat <<EOF > /etc/rancher/rke2/registries.yaml
mirrors:
  "192.168.122.156:30000":
    endpoint:
      - "http://192.168.122.156:30000"
configs:
  "192.168.122.156:30000":
    tls:
      insecure_skip_verify: true
      
EOF
Plain
INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-1.35.6 /opt/rke2-1.35.6/install.sh

# 3. 启用并启动服务
systemctl enable rke2-server
systemctl start rke2-server

kubectl label node 192.168.112.97 node-role.kubernetes.io/worker=true
kubectl label node 192.168.112.98 node-role.kubernetes.io/worker=true
kubectl label node 192.168.112.99 node-role.kubernetes.io/worker=true

添加节点

Plain


cat > /etc/rancher/rke2/config.yaml <<EOF
disable-scheduler-node-taints: true
kube-proxy-arg:
  - "proxy-mode=ipvs"
tls-san:
  - "192.168.112.97"
  - "192.168.112.98"
  - "192.168.112.99"
  - "192.168.112.253"
server: https://192.168.112.253:9345
token: xxx
node-name: $(hostname -I|awk '{print $1}')
data-dir: /var/lib/rancher/rke2
etcd-snapshot-retention: 5
etcd-snapshot-schedule-cron: "0 */6 * * *"
service-node-port-range: "30000-60000"
EOF

crictl ​

Bash
http://10.20.24.225:31000/repo/rke2/crictl
curl 192.168.122.1:31000/repo/rke2/crictl -o /usr/bin/crictl
# ln -s /var/lib/rancher/rke2/bin/crictl /usr/bin/crictl
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///var/run/k3s/containerd/containerd.sock
image-endpoint: unix:///var/run/k3s/containerd/containerd.sock
timeout: 10
debug: false
EOF

crictl ps

ctr设置及导出镜像tar包 ​

PowerShell
echo 'export CONTAINERD_ADDRESS=/var/run/k3s/containerd/containerd.sock' >> /etc/profile
source /etc/profile
# 确认镜像在 containerd k8s.io 命名空间
ctr -n k8s.io images list | grep jetstack

# 一次性导出3个镜像到同一个tar包
ctr -n k8s.io images export cert-manager-v1.20.3-all.tar \
quay.io/jetstack/cert-manager-controller:v1.20.3 \
quay.io/jetstack/cert-manager-startupapicheck:v1.20.3 \
quay.io/jetstack/cert-manager-webhook:v1.20.3

ctr -n k8s.io images export cert-manager-cainjector-v1.20.3.tar \
quay.io/jetstack/cert-manager-cainjector:v1.20.3

内核优化 ​

Plain
curl -s  http://10.20.24.225:31000/repo/redhat/init-sshd.sh|sh
curl -s http://10.20.24.225:31000/repo/rocky/9.8/rke2-canal-kernel-tune.sh |sh

下载离线镜像和rke2服务 ​

Plain
yum install -y wget
cd /opt
wget http://10.20.24.225:31000/repo/rke2/rke2-1.35.6.tar.gz
tar xvf rke2-1.35.6.tar.gz
cd rke2-1.35.6
tar -xzf rke2.linux-amd64.tar.gz -C /usr/local
mkdir -p /var/lib/rancher/rke2/agent/images/
cp rke2-images.linux-amd64.tar.* /var/lib/rancher/rke2/agent/images/

Master

Bash
#!/bin/bash
set -e
echo "===== RKE2 v1.35.6 Master+Worker 初始化节点安装 ====="

# 1. 创建配置目录
mkdir -p /etc/rancher/rke2

cat > /etc/rancher/rke2/config.yaml <<EOF
disable-scheduler-node-taints: true
kube-proxy-arg:
  - "proxy-mode=ipvs"
node-name: $(hostname -I|awk '{print $1}')
data-dir: /var/lib/rancher/rke2
cluster-cidr: 10.12.0.0/16
service-cidr: 10.13.0.0/16
etcd-snapshot-retention: 5
etcd-snapshot-schedule-cron: "0 */6 * * *"
EOF

cat > /etc/rancher/rke2/config.yaml <<EOF
token: xxx
# 关闭master污点，允许调度业务pod
disable-scheduler-node-taints: true
# kube-proxy ipvs优化
kube-proxy-arg:
  - proxy-mode=ipvs
  - ipvs-min-sync-period=5s
  - ipvs-sync-period=10s
kube-apiserver-arg:
  - max-requests-inflight=3000
  - max-mutating-requests-inflight=1000
kube-controller-manager-arg:
  - node-monitor-grace-period=40s
  - pod-eviction-timeout=5m
kubelet-arg:
  - eviction-hard=memory.available<1Gi,nodefs.available<10%
  - eviction-soft=memory.available<2Gi,nodefs.available<15%
  - eviction-soft-grace-period=memory.available=1m,nodefs.available=1m
  - max-pods=100
EOF

# 2. 安装RKE2 1.35.6
#curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.35.6 sh -
#INSTALL_RKE2_VERSION=v1.35.6+rke2r1 sh /opt/rke2-1.35.6/install.sh
INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-1.35.6 /opt/rke2-1.35.6/install.sh

# 3. 启用并启动服务
systemctl enable rke2-server
systemctl start rke2-server

# 4. 配置kubectl软链接
mkdir -p /root/.kube
ln -s /etc/rancher/rke2/kubeconfig.yaml /root/.kube/config
export KUBECONFIG=/root/.kube/config


kubectl label node 192.168.122.78 node-role.kubernetes.io/worker=true
kubectl label node 192.168.122.69 node-role.kubernetes.io/worker=true
YAML

mkdir -p /etc/rancher/rke2/
cat <<EOF > /etc/rancher/rke2/registries.yaml
mirrors:
  "192.168.122.156:30000":
    endpoint:
      - "http://192.168.122.156:30000"
configs:
  "192.168.122.156:30000":
    tls:
      insecure_skip_verify: true
      
EOF

添加master ​

JavaScript
cat > /etc/rancher/rke2/config.yaml <<EOF
server: https://192.168.122.78:9345
token: xxx
disable-scheduler-node-taints: true
kube-proxy-arg:
  - "proxy-mode=ipvs"
node-name: $(hostname -I|awk '{print $1}')
data-dir: /var/lib/rancher/rke2
cluster-cidr: 10.12.0.0/16
service-cidr: 10.13.0.0/16
EOF
curl -s http://10.20.24.225:31000/repo/rke2/install-1.35.6.sh -o /opt/rke2-1.35.6/install.sh
chmod +x /opt/rke2-1.35.6/install.sh
INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-1.35.6 /opt/rke2-1.35.6/install.sh

# 3. 启用并启动服务
systemctl enable rke2-server
systemctl start rke2-server
PowerShell

 docker images|grep rancher|awk '{print $1":"$2}' > 1.35.6-calico.images.txt
while read line;do
echo $line
new_img="192.168.122.156:30000/$line"
docker tag $line $new_img
docker push $new_img
done < 1.35.6-calico.images.txt
done <  1.35.6-core.images.txt
PowerShell
docker images|grep jetstack|awk '{print $1":"$2}' > jetstack.image.txt
while read line;do
echo $line
new_img=$(echo $line|sed 's/^quay.io/192.168.122.156:30000/')
docker tag $line $new_img
docker push $new_img
done < jetstack.image.txt

Load images ​

Bash
docker images|grep ^rancher|awk '{print $1":"$2}' > images.txt
while read line;do
echo $line
new_img="192.168.122.156:30000/$line"
docker tag $line $new_img
docker push $new_img
done < images.txt

while read line;do
echo $line
new_img="harbor.yundasys.com/$line"
docker tag $line $new_img
docker push $new_img
done < images.txt
PowerShell

  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 9345 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 6443 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 2379 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 2380 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 10250 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 10257 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 10259 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 9099 -j ACCEPT
  iptables -I INPUT -s 192.168.122.0/24 -p tcp --dport 5473 -j ACCEPT   # Calico Typha
  iptables -I INPUT -s 192.168.122.0/24 -p udp --dport 4789 -j ACCEPT   # VXLAN
  iptables -I INPUT -s 192.168.122.0/24 -p udp --dport 8472 -j ACCEPT
  iptables-save > /etc/sysconfig/iptables