From 464119c3af52a5aa0a9c25e2dc6e40a79eaa0953 Mon Sep 17 00:00:00 2001
From: cby <46200206+cby-chen@users.noreply.github.com>
Date: Tue, 15 Feb 2022 16:25:39 +0800
Subject: [PATCH] Add files via upload
---
CoreDNS/coredns.yaml | 2 +-
README.md | 2095 +++++++++++++++++++++++++++++++++++++++++-
calico/calico.yaml | 2 +-
dashboard/admin.yaml | 24 +
4 files changed, 2120 insertions(+), 3 deletions(-)
create mode 100644 dashboard/admin.yaml
diff --git a/CoreDNS/coredns.yaml b/CoreDNS/coredns.yaml
index 7bca2fd..6a1c1d0 100644
--- a/CoreDNS/coredns.yaml
+++ b/CoreDNS/coredns.yaml
@@ -185,7 +185,7 @@ metadata:
spec:
selector:
k8s-app: kube-dns
- clusterIP: KUBEDNS_SERVICE_IP
+ clusterIP: 10.96.0.10
ports:
- name: dns
port: 53
diff --git a/README.md b/README.md
index ddf04c9..d2afb64 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,2094 @@
-# Kubernetes
\ No newline at end of file
+# 二进制安装Kubernetes（k8s） v1.23.3
+
+# 1.环境
+
+| 主机名称 | IP地址 | 说明 | 软件 |
+| -------- | ------------ | -------- | ---------------------------------------------------------------------------------------------- |
+| Master01 | 10.0.0.21 | master节点 | kube-apiserver、kube-controller-manager、kube-scheduler、etcd、
kubelet、kube-proxy、nfs-client |
+| Master02 | 10.0.0.22 | master节点 | kube-apiserver、kube-controller-manager、kube-scheduler、etcd、
kubelet、kube-proxy、nfs-client |
+| Master03 | 10.0.0.23 | master节点 | kube-apiserver、kube-controller-manager、kube-scheduler、etcd、
kubelet、kube-proxy、nfs-client |
+| Node01 | 10.0.0.24 | node节点 | kubelet、kube-proxy、nfs-client |
+| Node02 | 10.0.0.25 | node节点 | kubelet、kube-proxy、nfs-client |
+| Lb01 | 192.168.1.76 | Lb01节点 | haproxy、keepalived |
+| Lb02 | 192.168.1.77 | Lb02节点 | haproxy、keepalived |
+| | 192.168.1.78 | VIP | |
+| | | | |
+
+| 软件 | 版本 |
+|:------------------------------------------------------------------------ |:-------------------------- |
+| 内核 | 5.16.7-1.el8.elrepo.x86_64 |
+| CentOS 8 | v8 |
+| kube-apiserver、kube-controller-manager、kube-scheduler、kubelet、kube-proxy | v1.23.3 |
+| etcd | v3.5.2 |
+| docker-ce | v20.10.9 |
+| containerd | v1.6.0 |
+| cfssl | v1.6.1 |
+| cni | v1.6.0 |
+| crictl | v1.23.0 |
+| haproxy | v1.8.27 |
+| keepalived | v2.1.5 |
+
+网段
+
+物理主机：192.168.1.0/24
+
+service：10.96.0.0/12
+
+pod：172.16.0.0/12
+
+如果有条件建议k8s集群与etcd集群分开安装
+
+## 1.1.k8s基础系统环境配置
+
+### 1.2.配置IP
+
+```shell
+ssh root@192.168.1.100 "nmcli con mod ens18 ipv4.addresses 192.168.1.76/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+ssh root@192.168.1.125 "nmcli con mod ens18 ipv4.addresses 192.168.1.77/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+ssh root@192.168.1.116 "nmcli con mod ens18 ipv4.addresses 192.168.1.78/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+ssh root@192.168.1.108 "nmcli con mod ens18 ipv4.addresses 192.168.1.79/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+ssh root@192.168.1.106 "nmcli con mod ens18 ipv4.addresses 192.168.1.80/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+ssh root@192.168.1.154 "nmcli con mod ens18 ipv4.addresses 192.168.1.86/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+ssh root@192.168.1.161 "nmcli con mod ens18 ipv4.addresses 192.168.1.87/24; nmcli con mod ens18 ipv4.gateway 192.168.1.99; nmcli con mod ens18 ipv4.method manual; nmcli con mod ens18 ipv4.dns "8.8.8.8"; nmcli con up ens18"
+```
+
+### 1.3.设置主机名
+
+```shell
+hostnamectl set-hostname k8s-master01
+hostnamectl set-hostname k8s-master02
+hostnamectl set-hostname k8s-master03
+hostnamectl set-hostname k8s-node01
+hostnamectl set-hostname k8s-node02
+hostnamectl set-hostname lb01
+hostnamectl set-hostname lb02
+```
+
+### 1.4.配置yum源
+
+```shell
+sudo sed -e 's|^mirrorlist=|#mirrorlist=|g' \
+ -e 's|^#baseurl=http://mirror.centos.org/$contentdir|baseurl=http://192.168.1.123/centos|g' \
+ -i.bak \
+ /etc/yum.repos.d/CentOS-*.repo
+```
+
+### 1.5.安装一些必备工具
+
+```shell
+yum -y install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git network-scripts tar curl -y
+```
+
+### 1.6.安装docker工具
+
+```shell
+curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
+```
+
+### 1.7.关闭防火墙
+
+```shell
+systemctl disable --now firewalld
+```
+
+### 1.8.关闭SELinux
+
+```shell
+setenforce 0
+sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
+```
+
+### 1.9.关闭交换分区
+
+```shell
+sed -ri 's/.*swap.*/#&/' /etc/fstab
+swapoff -a && sysctl -w vm.swappiness=0
+cat /etc/fstab
+
+# /dev/mapper/centos-swap swap swap defaults 0 0
+```
+
+### 1.10.关闭NetworkManager 并启用 network
+
+```shell
+systemctl disable --now NetworkManager
+systemctl start network && systemctl enable network
+```
+
+### 1.11.进行时间同步
+
+```shell
+服务端
+
+yum install chrony -y
+vim /etc/chrony.conf
+cat /etc/chrony.conf | grep -v "^#" | grep -v "^$"
+pool ntp.aliyun.com iburst
+driftfile /var/lib/chrony/drift
+makestep 1.0 3
+rtcsync
+allow 192.168.1.0/24
+local stratum 10
+keyfile /etc/chrony.keys
+leapsectz right/UTC
+logdir /var/log/chrony
+
+systemctl restart chronyd
+systemctl enable chronyd
+
+客户端
+
+yum install chrony -y
+vim /etc/chrony.conf
+cat /etc/chrony.conf | grep -v "^#" | grep -v "^$"
+pool 10.0.0.21 iburst
+driftfile /var/lib/chrony/drift
+makestep 1.0 3
+rtcsync
+keyfile /etc/chrony.keys
+leapsectz right/UTC
+logdir /var/log/chrony
+
+systemctl restart chronyd ; systemctl enable chronyd
+
+
+yum install chrony -y ; sed -i "s#2.centos.pool.ntp.org#10.0.0.21#g" /etc/chrony.conf ; systemctl restart chronyd ; systemctl enable chronyd
+
+
+使用客户端进行验证
+
+chronyc sources -v
+```
+
+### 1.12.配置ulimit
+
+```shell
+ulimit -SHn 65535
+cat >> /etc/security/limits.conf <> /etc/modules-load.d/ipvs.conf < /etc/sysctl.d/k8s.conf
+net.ipv4.ip_forward = 1
+net.bridge.bridge-nf-call-iptables = 1
+fs.may_detach_mounts = 1
+vm.overcommit_memory=1
+vm.panic_on_oom=0
+fs.inotify.max_user_watches=89100
+fs.file-max=52706963
+fs.nr_open=52706963
+net.netfilter.nf_conntrack_max=2310720
+
+
+net.ipv4.tcp_keepalive_time = 600
+net.ipv4.tcp_keepalive_probes = 3
+net.ipv4.tcp_keepalive_intvl =15
+net.ipv4.tcp_max_tw_buckets = 36000
+net.ipv4.tcp_tw_reuse = 1
+net.ipv4.tcp_max_orphans = 327680
+net.ipv4.tcp_orphan_retries = 3
+net.ipv4.tcp_syncookies = 1
+net.ipv4.tcp_max_syn_backlog = 16384
+net.ipv4.ip_conntrack_max = 65536
+net.ipv4.tcp_max_syn_backlog = 16384
+net.ipv4.tcp_timestamps = 0
+net.core.somaxconn = 16384
+EOF
+
+sysctl --system
+```
+
+### 1.18.所有节点配置hosts本地解析
+
+```shell
+cat > /etc/hosts < /etc/crictl.yaml < /etc/etcd/etcd.config.yml << EOF
+name: 'k8s-master01'
+data-dir: /var/lib/etcd
+wal-dir: /var/lib/etcd/wal
+snapshot-count: 5000
+heartbeat-interval: 100
+election-timeout: 1000
+quota-backend-bytes: 0
+listen-peer-urls: 'https://192.168.1.76:2380'
+listen-client-urls: 'https://192.168.1.76:2379,http://127.0.0.1:2379'
+max-snapshots: 3
+max-wals: 5
+cors:
+initial-advertise-peer-urls: 'https://192.168.1.76:2380'
+advertise-client-urls: 'https://192.168.1.76:2379'
+discovery:
+discovery-fallback: 'proxy'
+discovery-proxy:
+discovery-srv:
+initial-cluster: 'k8s-master01=https://192.168.1.76:2380,k8s-master02=https://192.168.1.77:2380,k8s-master03=https://192.168.1.78:2380'
+initial-cluster-token: 'etcd-k8s-cluster'
+initial-cluster-state: 'new'
+strict-reconfig-check: false
+enable-v2: true
+enable-pprof: true
+proxy: 'off'
+proxy-failure-wait: 5000
+proxy-refresh-interval: 30000
+proxy-dial-timeout: 1000
+proxy-write-timeout: 5000
+proxy-read-timeout: 0
+client-transport-security:
+ cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
+ key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
+ client-cert-auth: true
+ trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
+ auto-tls: true
+peer-transport-security:
+ cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
+ key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
+ peer-client-cert-auth: true
+ trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
+ auto-tls: true
+debug: false
+log-package-levels:
+log-outputs: [default]
+force-new-cluster: false
+EOF
+```
+
+### 4.1.2master02配置
+
+```shell
+cat > /etc/etcd/etcd.config.yml << EOF
+name: 'k8s-master02'
+data-dir: /var/lib/etcd
+wal-dir: /var/lib/etcd/wal
+snapshot-count: 5000
+heartbeat-interval: 100
+election-timeout: 1000
+quota-backend-bytes: 0
+listen-peer-urls: 'https://192.168.1.77:2380'
+listen-client-urls: 'https://192.168.1.77:2379,http://127.0.0.1:2379'
+max-snapshots: 3
+max-wals: 5
+cors:
+initial-advertise-peer-urls: 'https://192.168.1.77:2380'
+advertise-client-urls: 'https://192.168.1.77:2379'
+discovery:
+discovery-fallback: 'proxy'
+discovery-proxy:
+discovery-srv:
+initial-cluster: 'k8s-master01=https://192.168.1.76:2380,k8s-master02=https://192.168.1.77:2380,k8s-master03=https://192.168.1.78:2380'
+initial-cluster-token: 'etcd-k8s-cluster'
+initial-cluster-state: 'new'
+strict-reconfig-check: false
+enable-v2: true
+enable-pprof: true
+proxy: 'off'
+proxy-failure-wait: 5000
+proxy-refresh-interval: 30000
+proxy-dial-timeout: 1000
+proxy-write-timeout: 5000
+proxy-read-timeout: 0
+client-transport-security:
+ cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
+ key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
+ client-cert-auth: true
+ trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
+ auto-tls: true
+peer-transport-security:
+ cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
+ key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
+ peer-client-cert-auth: true
+ trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
+ auto-tls: true
+debug: false
+log-package-levels:
+log-outputs: [default]
+force-new-cluster: false
+EOF
+```
+
+### 4.1.3master03配置
+
+```shell
+cat > /etc/etcd/etcd.config.yml << EOF
+name: 'k8s-master03'
+data-dir: /var/lib/etcd
+wal-dir: /var/lib/etcd/wal
+snapshot-count: 5000
+heartbeat-interval: 100
+election-timeout: 1000
+quota-backend-bytes: 0
+listen-peer-urls: 'https://192.168.1.78:2380'
+listen-client-urls: 'https://192.168.1.78:2379,http://127.0.0.1:2379'
+max-snapshots: 3
+max-wals: 5
+cors:
+initial-advertise-peer-urls: 'https://192.168.1.78:2380'
+advertise-client-urls: 'https://192.168.1.78:2379'
+discovery:
+discovery-fallback: 'proxy'
+discovery-proxy:
+discovery-srv:
+initial-cluster: 'k8s-master01=https://192.168.1.76:2380,k8s-master02=https://192.168.1.77:2380,k8s-master03=https://192.168.1.78:2380'
+initial-cluster-token: 'etcd-k8s-cluster'
+initial-cluster-state: 'new'
+strict-reconfig-check: false
+enable-v2: true
+enable-pprof: true
+proxy: 'off'
+proxy-failure-wait: 5000
+proxy-refresh-interval: 30000
+proxy-dial-timeout: 1000
+proxy-write-timeout: 5000
+proxy-read-timeout: 0
+client-transport-security:
+ cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
+ key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
+ client-cert-auth: true
+ trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
+ auto-tls: true
+peer-transport-security:
+ cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
+ key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
+ peer-client-cert-auth: true
+ trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
+ auto-tls: true
+debug: false
+log-package-levels:
+log-outputs: [default]
+force-new-cluster: false
+EOF
+```
+
+## 4.2.创建service（所有master节点操作）
+
+### 4.2.1创建etcd.service并启动
+
+```shell
+cat > /usr/lib/systemd/system/etcd.service << EOF
+
+[Unit]
+Description=Etcd Service
+Documentation=https://coreos.com/etcd/docs/latest/
+After=network.target
+
+[Service]
+Type=notify
+ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
+Restart=on-failure
+RestartSec=10
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target
+Alias=etcd3.service
+
+EOF
+```
+
+### 4.2.2创建etcd证书目录
+
+```shell
+mkdir /etc/kubernetes/pki/etcd
+ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
+systemctl daemon-reload
+systemctl enable --now etcd
+```
+
+### 4.2.3查看etcd状态
+
+```shell
+export ETCDCTL_API=3
+etcdctl --endpoints="192.168.1.78:2379,192.168.1.77:2379,192.168.1.76:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem endpoint status --write-out=table
++-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
+| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
++-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
+| 192.168.1.78:2379 | 56875ab4a12c94e8 | 3.5.1 | 25 kB | false | false | 2 | 8 | 8 | |
+| 192.168.1.77:2379 | 33df6a8fe708d3fd | 3.5.1 | 25 kB | true | false | 2 | 8 | 8 | |
+| 192.168.1.76:2379 | 58fbe5ec9743048f | 3.5.1 | 20 kB | false | false | 2 | 8 | 8 | |
++-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
+```
+
+# 5.高可用配置
+
+## 5.1在lb01和lb02两台服务器上操作
+
+### 5.1.1安装keepalived和haproxy服务
+
+```shell
+yum -y install keepalived haproxy
+```
+
+### 5.1.2修改haproxy配置文件（两台配置文件一样）
+
+```shell
+# cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak
+
+cat >/etc/haproxy/haproxy.cfg<<"EOF"
+global
+ maxconn 2000
+ ulimit-n 16384
+ log 127.0.0.1 local0 err
+ stats timeout 30s
+
+defaults
+ log global
+ mode http
+ option httplog
+ timeout connect 5000
+ timeout client 50000
+ timeout server 50000
+ timeout http-request 15s
+ timeout http-keep-alive 15s
+
+
+frontend monitor-in
+ bind *:33305
+ mode http
+ option httplog
+ monitor-uri /monitor
+
+frontend k8s-master
+ bind 0.0.0.0:8443
+ bind 127.0.0.1:8443
+ mode tcp
+ option tcplog
+ tcp-request inspect-delay 5s
+ default_backend k8s-master
+
+
+backend k8s-master
+ mode tcp
+ option tcplog
+ option tcp-check
+ balance roundrobin
+ default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
+ server master01 192.168.1.76:6443 check
+ server master02 192.168.1.77:6443 check
+ server master03 192.168.1.78:6443 check
+EOF
+```
+
+### 5.1.3lb01配置keepalived master节点
+
+```shell
+#cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
+
+cat > /etc/keepalived/keepalived.conf << EOF
+! Configuration File for keepalived
+
+global_defs {
+ router_id LVS_DEVEL
+}
+vrrp_script chk_apiserver {
+ script "/etc/keepalived/check_apiserver.sh"
+ interval 5
+ weight -5
+ fall 2
+ rise 1
+}
+vrrp_instance VI_1 {
+ state MASTER
+ interface ens18
+ mcast_src_ip 192.168.1.86
+ virtual_router_id 51
+ priority 100
+ nopreempt
+ advert_int 2
+ authentication {
+ auth_type PASS
+ auth_pass K8SHA_KA_AUTH
+ }
+ virtual_ipaddress {
+ 192.168.1.88
+ }
+ track_script {
+ chk_apiserver
+} }
+
+EOF
+```
+
+### 5.1.4lb02配置keepalived backup节点
+
+```shell
+# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
+
+cat > /etc/keepalived/keepalived.conf << EOF
+! Configuration File for keepalived
+
+global_defs {
+ router_id LVS_DEVEL
+}
+vrrp_script chk_apiserver {
+ script "/etc/keepalived/check_apiserver.sh"
+ interval 5
+ weight -5
+ fall 2
+ rise 1
+
+}
+vrrp_instance VI_1 {
+ state BACKUP
+ interface ens18
+ mcast_src_ip 192.168.1.87
+ virtual_router_id 51
+ priority 50
+ nopreempt
+ advert_int 2
+ authentication {
+ auth_type PASS
+ auth_pass K8SHA_KA_AUTH
+ }
+ virtual_ipaddress {
+ 192.168.1.88
+ }
+ track_script {
+ chk_apiserver
+} }
+
+EOF
+```
+
+### 5.1.5健康检查脚本配置（两台lb主机）
+
+```shell
+cat > /etc/keepalived/check_apiserver.sh << EOF
+#!/bin/bash
+
+err=0
+for k in \$(seq 1 3)
+do
+ check_code=\$(pgrep haproxy)
+ if [[ \$check_code == "" ]]; then
+ err=\$(expr \$err + 1)
+ sleep 1
+ continue
+ else
+ err=0
+ break
+ fi
+done
+
+if [[ \$err != "0" ]]; then
+ echo "systemctl stop keepalived"
+ /usr/bin/systemctl stop keepalived
+ exit 1
+else
+ exit 0
+fi
+EOF
+
+# 给脚本授权
+
+chmod +x /etc/keepalived/check_apiserver.sh
+```
+
+### 5.1.6启动服务
+
+```shell
+systemctl daemon-reload
+systemctl enable --now haproxy
+systemctl enable --now keepalived
+```
+
+### 5.1.7测试高可用
+
+```shell
+# 能ping同
+
+[root@k8s-node02 ~]# ping 192.168.1.88
+
+# 能telnet访问
+
+[root@k8s-node02 ~]# telnet 192.168.1.88 8443
+
+# 关闭主节点，看vip是否漂移到备节点
+```
+
+# 6.k8s组件配置（区别于第4点）
+
+所有k8s节点创建以下目录
+
+```shell
+mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes
+```
+
+## 6.1.创建apiserver（所有master节点）
+
+### 6.1.1master01节点配置
+
+```shell
+cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
+
+[Unit]
+Description=Kubernetes API Server
+Documentation=https://github.com/kubernetes/kubernetes
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/kube-apiserver \
+ --v=2 \
+ --logtostderr=true \
+ --allow-privileged=true \
+ --bind-address=0.0.0.0 \
+ --secure-port=6443 \
+ --insecure-port=0 \
+ --advertise-address=192.168.1.76 \
+ --service-cluster-ip-range=10.96.0.0/12 \
+ --service-node-port-range=30000-32767 \
+ --etcd-servers=https://192.168.1.76:2379,https://192.168.1.77:2379,https://192.168.1.78:2379 \
+ --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
+ --etcd-certfile=/etc/etcd/ssl/etcd.pem \
+ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
+ --client-ca-file=/etc/kubernetes/pki/ca.pem \
+ --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
+ --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
+ --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \
+ --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \
+ --service-account-key-file=/etc/kubernetes/pki/sa.pub \
+ --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
+ --service-account-issuer=https://kubernetes.default.svc.cluster.local \
+ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
+ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
+ --authorization-mode=Node,RBAC \
+ --enable-bootstrap-token-auth=true \
+ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
+ --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
+ --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
+ --requestheader-allowed-names=aggregator \
+ --requestheader-group-headers=X-Remote-Group \
+ --requestheader-extra-headers-prefix=X-Remote-Extra- \
+ --requestheader-username-headers=X-Remote-User
+ # --token-auth-file=/etc/kubernetes/token.csv
+
+Restart=on-failure
+RestartSec=10s
+LimitNOFILE=65535
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+### 6.1.2master02节点配置
+
+```shell
+cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
+[Unit]
+Description=Kubernetes API Server
+Documentation=https://github.com/kubernetes/kubernetes
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/kube-apiserver \
+ --v=2 \
+ --logtostderr=true \
+ --allow-privileged=true \
+ --bind-address=0.0.0.0 \
+ --secure-port=6443 \
+ --insecure-port=0 \
+ --advertise-address=192.168.1.77 \
+ --service-cluster-ip-range=10.96.0.0/12 \
+ --service-node-port-range=30000-32767 \
+ --etcd-servers=https://192.168.1.76:2379,https://192.168.1.77:2379,https://192.168.1.78:2379 \
+ --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
+ --etcd-certfile=/etc/etcd/ssl/etcd.pem \
+ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
+ --client-ca-file=/etc/kubernetes/pki/ca.pem \
+ --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
+ --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
+ --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \
+ --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \
+ --service-account-key-file=/etc/kubernetes/pki/sa.pub \
+ --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
+ --service-account-issuer=https://kubernetes.default.svc.cluster.local \
+ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
+ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
+ --authorization-mode=Node,RBAC \
+ --enable-bootstrap-token-auth=true \
+ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
+ --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
+ --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
+ --requestheader-allowed-names=aggregator \
+ --requestheader-group-headers=X-Remote-Group \
+ --requestheader-extra-headers-prefix=X-Remote-Extra- \
+ --requestheader-username-headers=X-Remote-User
+ # --token-auth-file=/etc/kubernetes/token.csv
+
+Restart=on-failure
+RestartSec=10s
+LimitNOFILE=65535
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+### 6.1.3master03节点配置
+
+```shell
+cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
+
+[Unit]
+Description=Kubernetes API Server
+Documentation=https://github.com/kubernetes/kubernetes
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/kube-apiserver \
+ --v=2 \
+ --logtostderr=true \
+ --allow-privileged=true \
+ --bind-address=0.0.0.0 \
+ --secure-port=6443 \
+ --insecure-port=0 \
+ --advertise-address=192.168.1.78 \
+ --service-cluster-ip-range=10.96.0.0/12 \
+ --service-node-port-range=30000-32767 \
+ --etcd-servers=https://192.168.1.76:2379,https://192.168.1.77:2379,https://192.168.1.78:2379 \
+ --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
+ --etcd-certfile=/etc/etcd/ssl/etcd.pem \
+ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
+ --client-ca-file=/etc/kubernetes/pki/ca.pem \
+ --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
+ --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
+ --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \
+ --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \
+ --service-account-key-file=/etc/kubernetes/pki/sa.pub \
+ --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
+ --service-account-issuer=https://kubernetes.default.svc.cluster.local \
+ --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
+ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
+ --authorization-mode=Node,RBAC \
+ --enable-bootstrap-token-auth=true \
+ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
+ --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
+ --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
+ --requestheader-allowed-names=aggregator \
+ --requestheader-group-headers=X-Remote-Group \
+ --requestheader-extra-headers-prefix=X-Remote-Extra- \
+ --requestheader-username-headers=X-Remote-User
+ # --token-auth-file=/etc/kubernetes/token.csv
+
+Restart=on-failure
+RestartSec=10s
+LimitNOFILE=65535
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+### 6.1.4启动apiserver（所有master节点）
+
+```shell
+systemctl daemon-reload && systemctl enable --now kube-apiserver
+
+# 注意查看状态是否启动正常
+
+systemctl status kube-apiserver
+```
+
+## 6.2.配置kube-controller-manager service
+
+```shell
+所有master节点配置，且配置相同
+172.16.0.0/12为pod网段，按需求设置你自己的网段
+
+cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
+
+[Unit]
+Description=Kubernetes Controller Manager
+Documentation=https://github.com/kubernetes/kubernetes
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/kube-controller-manager \
+ --v=2 \
+ --logtostderr=true \
+ --address=127.0.0.1 \
+ --root-ca-file=/etc/kubernetes/pki/ca.pem \
+ --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \
+ --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \
+ --service-account-private-key-file=/etc/kubernetes/pki/sa.key \
+ --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \
+ --leader-elect=true \
+ --use-service-account-credentials=true \
+ --node-monitor-grace-period=40s \
+ --node-monitor-period=5s \
+ --pod-eviction-timeout=2m0s \
+ --controllers=*,bootstrapsigner,tokencleaner \
+ --allocate-node-cidrs=true \
+ --cluster-cidr=172.16.0.0/12 \
+ --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
+ --node-cidr-mask-size=24
+
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+### 6.2.1启动kube-controller-manager，并查看状态
+
+```shell
+systemctl daemon-reload
+systemctl enable --now kube-controller-manager
+systemctl status kube-controller-manager
+```
+
+## 6.3.配置kube-scheduler service
+
+### 6.3.1所有master节点配置，且配置相同
+
+```shell
+cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
+
+[Unit]
+Description=Kubernetes Scheduler
+Documentation=https://github.com/kubernetes/kubernetes
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/kube-scheduler \
+ --v=2 \
+ --logtostderr=true \
+ --address=127.0.0.1 \
+ --leader-elect=true \
+ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
+
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+### 6.3.2启动并查看服务状态
+
+```shell
+systemctl daemon-reload
+systemctl enable --now kube-scheduler
+systemctl status kube-scheduler
+```
+
+# 7.TLS Bootstrapping配置
+
+## 7.1在master01上配置
+
+```shell
+cd /root/Kubernetes/bootstrap
+
+kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.1.88:8443 --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
+
+kubectl config set-credentials tls-bootstrap-token-user --token=c8ad9c.2e4d610cf3e7426e --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
+
+kubectl config set-context tls-bootstrap-token-user@kubernetes --cluster=kubernetes --user=tls-bootstrap-token-user --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
+
+kubectl config use-context tls-bootstrap-token-user@kubernetes --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
+
+# token的位置在bootstrap.secret.yaml，如果修改的话到这个文件修改
+
+mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config
+```
+
+## 7.2查看集群状态，没问题的话继续后续操作
+
+```shell
+Warning: v1 ComponentStatus is deprecated in v1.19+
+NAME STATUS MESSAGE ERROR
+controller-manager Healthy ok
+etcd-0 Healthy {"health":"true","reason":""}
+scheduler Healthy ok
+etcd-1 Healthy {"health":"true","reason":""}
+etcd-2 Healthy {"health":"true","reason":""}
+
+kubectl create -f bootstrap.secret.yaml
+```
+
+# 8.node节点配置
+
+## 8.1.在master01上将证书复制到node节点
+
+```shell
+cd /etc/kubernetes/
+
+for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do
+ ssh $NODE mkdir -p /etc/kubernetes/pki
+ for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig; do
+ scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}
+ done
+ done
+```
+
+## 8.2.kubelet配置
+
+### 8.2.1所有k8s节点创建相关目录
+
+```shell
+mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/
+
+
+
+所有k8s节点配置kubelet service
+cat > /usr/lib/systemd/system/kubelet.service << EOF
+
+[Unit]
+Description=Kubernetes Kubelet
+Documentation=https://github.com/kubernetes/kubernetes
+After=docker.service
+Requires=docker.service
+
+[Service]
+ExecStart=/usr/local/bin/kubelet
+
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+EOF
+```
+
+### 8.2.2所有k8s节点配置kubelet service的配置文件
+
+```shell
+cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf << EOF
+[Service]
+Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
+Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --cgroup-driver=systemd"
+Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml"
+Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
+ExecStart=
+ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS
+
+EOF
+```
+
+### 8.2.3所有k8s节点创建kubelet的配置文件
+
+```shell
+cat > /etc/kubernetes/kubelet-conf.yml < 18m v1.23.3
+k8s-master02 NotReady 18m v1.23.3
+k8s-master03 NotReady 18m v1.23.3
+k8s-node01 NotReady 18m v1.23.3
+k8s-node02 NotReady 18m v1.23.3
+```
+
+## 8.3.kube-proxy配置
+
+### 8.3.1此配置只在master01操作
+
+```shell
+cd /root/Kubernetes/
+kubectl -n kube-system create serviceaccount kube-proxy
+
+kubectl create clusterrolebinding system:kube-proxy --clusterrole system:node-proxier --serviceaccount kube-system:kube-proxy
+
+SECRET=$(kubectl -n kube-system get sa/kube-proxy \
+ --output=jsonpath='{.secrets[0].name}')
+
+JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET \
+--output=jsonpath='{.data.token}' | base64 -d)
+
+PKI_DIR=/etc/kubernetes/pki
+K8S_DIR=/etc/kubernetes
+
+kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.1.88:8443 --kubeconfig=${K8S_DIR}/kube-proxy.kubeconfig
+
+kubectl config set-credentials kubernetes --token=${JWT_TOKEN} --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
+
+kubectl config set-context kubernetes --cluster=kubernetes --user=kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
+
+kubectl config use-context kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
+```
+
+### 8.3.2将kubeconfig发送至其他节点
+
+```shell
+for NODE in k8s-master02 k8s-master03; do
+ scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
+ done
+
+for NODE in k8s-node01 k8s-node02; do
+ scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
+ done
+```
+
+### 8.3.3所有k8s节点添加kube-proxy的配置和service文件
+
+```shell
+cat > /usr/lib/systemd/system/kube-proxy.service << EOF
+[Unit]
+Description=Kubernetes Kube Proxy
+Documentation=https://github.com/kubernetes/kubernetes
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/kube-proxy \
+ --config=/etc/kubernetes/kube-proxy.yaml \
+ --v=2
+
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+```shell
+cat > /etc/kubernetes/kube-proxy.yaml << EOF
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+bindAddress: 0.0.0.0
+clientConnection:
+ acceptContentTypes: ""
+ burst: 10
+ contentType: application/vnd.kubernetes.protobuf
+ kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
+ qps: 5
+clusterCIDR: 172.16.0.0/12
+configSyncPeriod: 15m0s
+conntrack:
+ max: null
+ maxPerCore: 32768
+ min: 131072
+ tcpCloseWaitTimeout: 1h0m0s
+ tcpEstablishedTimeout: 24h0m0s
+enableProfiling: false
+healthzBindAddress: 0.0.0.0:10256
+hostnameOverride: ""
+iptables:
+ masqueradeAll: false
+ masqueradeBit: 14
+ minSyncPeriod: 0s
+ syncPeriod: 30s
+ipvs:
+ masqueradeAll: true
+ minSyncPeriod: 5s
+ scheduler: "rr"
+ syncPeriod: 30s
+kind: KubeProxyConfiguration
+metricsBindAddress: 127.0.0.1:10249
+mode: "ipvs"
+nodePortAddresses: null
+oomScoreAdj: -999
+portRange: ""
+udpIdleTimeout: 250ms
+
+EOF
+```
+
+### 8.3.4启动kube-proxy
+
+```shell
+ systemctl daemon-reload
+ systemctl enable --now kube-proxy
+```
+
+# 9.安装Calico
+
+## 9.1以下步骤只在master01操作
+
+### 9.1.1更改calico网段
+
+```shell
+cd /root/Kubernetes/calico/
+sed -i "s#POD_CIDR#172.16.0.0/12#g" calico.yaml
+grep "IPV4POOL_CIDR" calico.yaml -A 1
+ - name: CALICO_IPV4POOL_CIDR
+ value: "172.16.0.0/12"
+
+# 创建
+
+kubectl apply -f calico.yaml
+```
+
+### 9.1.2查看容器状态
+
+```shell
+kubectl get po -n kube-system
+NAME READY STATUS RESTARTS AGE
+calico-kube-controllers-5dffd5886b-4blh6 1/1 Running 0 4m24s
+calico-node-fvbdq 1/1 Running 1 (2m51s ago) 4m23s
+calico-node-g8nqd 1/1 Running 0 4m23s
+calico-node-mdps8 1/1 Running 0 4m24s
+calico-node-nf4nt 1/1 Running 0 4m24s
+calico-node-sq2ml 1/1 Running 0 4m24s
+calico-typha-8445487f56-mg6p8 1/1 Running 0 4m24s
+calico-typha-8445487f56-pxbpj 1/1 Running 0 4m24s
+calico-typha-8445487f56-tnssl 1/1 Running 0 4m24s
+```
+
+# 10.安装CoreDNS
+
+## 10.1以下步骤只在master01操作
+
+### 10.1.1修改文件
+
+```shell
+cd /root/Kubernetes/CoreDNS/
+sed -i "s#KUBEDNS_SERVICE_IP#10.96.0.10#g" coredns.yaml
+
+cat coredns.yaml | grep clusterIP:
+ clusterIP: 10.96.0.10
+```
+
+### 10.1.2安装
+
+```shell
+kubectl create -f coredns.yaml
+serviceaccount/coredns created
+clusterrole.rbac.authorization.k8s.io/system:coredns created
+clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
+configmap/coredns created
+deployment.apps/coredns created
+service/kube-dns created
+```
+
+# 11.安装Metrics Server
+
+## 11.1以下步骤只在master01操作
+
+### 11.1.1安装Metrics-server
+
+在新版的Kubernetes中系统资源的采集均使用Metrics-server，可以通过Metrics采集节点和Pod的内存、磁盘、CPU和网络的使用率
+
+```shell
+安装metrics server
+cd /root/Kubernetes/metrics-server/
+
+kubectl create -f .
+
+serviceaccount/metrics-server created
+clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
+clusterrole.rbac.authorization.k8s.io/system:metrics-server created
+rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
+clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
+clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
+service/metrics-server created
+deployment.apps/metrics-server created
+apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
+```
+
+### 11.1.2稍等片刻查看状态
+
+```shell
+kubectl top node
+NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
+k8s-master01 172m 2% 1307Mi 16%
+k8s-master02 157m 1% 1189Mi 15%
+k8s-master03 155m 1% 1105Mi 14%
+k8s-node01 99m 1% 710Mi 9%
+k8s-node02 79m 0% 585Mi 7%
+```
+
+# 12.集群验证
+
+## 12.1部署pod资源
+
+```shell
+cat< 443/TCP 17h
+
+
+kubectl exec busybox -n default -- nslookup kubernetes
+3Server: 10.96.0.10
+Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
+
+Name: kubernetes
+Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
+```
+
+## 12.3测试跨命名空间是否可以解析
+
+```shell
+kubectl exec busybox -n default -- nslookup kube-dns.kube-system
+Server: 10.96.0.10
+Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
+
+Name: kube-dns.kube-system
+Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
+```
+
+## 12.4每个节点都必须要能访问Kubernetes的kubernetes svc 443和kube-dns的service 53
+
+```shell
+telnet 10.96.0.1 443
+Trying 10.96.0.1...
+Connected to 10.96.0.1.
+Escape character is '^]'.
+
+ telnet 10.96.0.10 53
+Trying 10.96.0.10...
+Connected to 10.96.0.10.
+Escape character is '^]'.
+
+curl 10.96.0.10:53
+curl: (52) Empty reply from server
+```
+
+## 12.5Pod和Pod之前要能通
+
+```shell
+kubectl get po -owide
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+busybox 1/1 Running 0 17m 172.27.14.193 k8s-node02
+
+ kubectl get po -n kube-system -owide
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+calico-kube-controllers-5dffd5886b-4blh6 1/1 Running 0 77m 172.25.244.193 k8s-master01
+calico-node-fvbdq 1/1 Running 1 (75m ago) 77m 192.168.1.76 k8s-master01
+calico-node-g8nqd 1/1 Running 0 77m 192.168.1.79 k8s-node01
+calico-node-mdps8 1/1 Running 0 77m 192.168.1.80 k8s-node02
+calico-node-nf4nt 1/1 Running 0 77m 192.168.1.78 k8s-master03
+calico-node-sq2ml 1/1 Running 0 77m 192.168.1.77 k8s-master02
+calico-typha-8445487f56-mg6p8 1/1 Running 0 77m 192.168.1.80 k8s-node02
+calico-typha-8445487f56-pxbpj 1/1 Running 0 77m 192.168.1.76 k8s-master01
+calico-typha-8445487f56-tnssl 1/1 Running 0 77m 192.168.1.79 k8s-node01
+coredns-5db5696c7-67h79 1/1 Running 0 63m 172.25.92.65 k8s-master02
+metrics-server-6bf7dcd649-5fhrw 1/1 Running 0 61m 172.18.195.1 k8s-master03
+
+# 进入busybox ping其他节点上的pod
+
+kubectl exec -ti busybox -- sh
+/ # ping 192.168.1.79
+PING 192.168.1.79 (192.168.1.79): 56 data bytes
+64 bytes from 192.168.1.79: seq=0 ttl=63 time=0.358 ms
+64 bytes from 192.168.1.79: seq=1 ttl=63 time=0.668 ms
+64 bytes from 192.168.1.79: seq=2 ttl=63 time=0.637 ms
+64 bytes from 192.168.1.79: seq=3 ttl=63 time=0.624 ms
+64 bytes from 192.168.1.79: seq=4 ttl=63 time=0.907 ms
+
+# 可以连通证明这个pod是可以跨命名空间和跨主机通信的
+```
+
+## 12.6创建三个副本，可以看到3个副本分布在不同的节点上（用完可以删了）
+
+```shell
+cat > deployments.yaml << EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: nginx-deployment
+ labels:
+ app: nginx
+spec:
+ replicas: 3
+ selector:
+ matchLabels:
+ app: nginx
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:1.14.2
+ ports:
+ - containerPort: 80
+
+EOF
+
+
+kubectl apply -f deployments.yaml
+deployment.apps/nginx-deployment created
+
+kubectl get pod
+NAME READY STATUS RESTARTS AGE
+busybox 1/1 Running 0 6m25s
+nginx-deployment-9456bbbf9-4bmvk 1/1 Running 0 8s
+nginx-deployment-9456bbbf9-9rcdk 1/1 Running 0 8s
+nginx-deployment-9456bbbf9-dqv8s 1/1 Running 0 8s
+
+# 删除nginx
+
+[root@k8s-master01 ~]# kubectl delete -f deployments.yaml
+```
+
+# 13.安装dashboard
+
+```shell
+cd /root/Kubernetes/dashboard/
+
+kubectl create -f .
+serviceaccount/admin-user created
+clusterrolebinding.rbac.authorization.k8s.io/admin-user created
+namespace/kubernetes-dashboard created
+serviceaccount/kubernetes-dashboard created
+service/kubernetes-dashboard created
+secret/kubernetes-dashboard-certs created
+secret/kubernetes-dashboard-csrf created
+secret/kubernetes-dashboard-key-holder created
+configmap/kubernetes-dashboard-settings created
+role.rbac.authorization.k8s.io/kubernetes-dashboard created
+clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created
+rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
+clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
+deployment.apps/kubernetes-dashboard created
+service/dashboard-metrics-scraper created
+deployment.apps/dashboard-metrics-scraper created
+```
+
+## 13.1创建管理员用户
+
+```shell
+cat > admin.yaml << EOF
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: admin-user
+
+ namespace: kube-system
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: admin-user
+ annotations:
+ rbac.authorization.kubernetes.io/autoupdate: "true"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cluster-admin
+subjects:
+
+- kind: ServiceAccount
+ name: admin-user
+ namespace: kube-system
+
+EOF
+```
+
+## 13.2执行yaml文件
+
+```shell
+kubectl apply -f admin.yaml -n kube-system
+
+serviceaccount/admin-user created
+clusterrolebinding.rbac.authorization.k8s.io/admin-user created
+```
+
+## 13.3更改dashboard的svc为NodePort，如果已是请忽略
+
+```shell
+kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
+ type: NodePort
+```
+
+## 13.4查看端口号
+
+```shell
+kubectl get svc kubernetes-dashboard -n kubernetes-dashboard
+NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+kubernetes-dashboard NodePort 10.98.201.22 443:31245/TCP 10m
+```
+
+## 13.5查看token
+
+```shell
+kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')
+Name: admin-user-token-k545k
+Namespace: kube-system
+Labels:
+Annotations: kubernetes.io/service-account.name: admin-user
+ kubernetes.io/service-account.uid: c308071c-4cf5-4583-83a2-eaf7812512b4
+
+Type: kubernetes.io/service-account-token
+
+Data
+====
+token: eyJhbGciOiJSUzI1NiIsImtpZCI6InYzV2dzNnQzV3hHb2FQWnYzdnlOSmpudmtpVmNjQW5VM3daRi12SFM4dEEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWs1NDVrIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJjMzA4MDcxYy00Y2Y1LTQ1ODMtODNhMi1lYWY3ODEyNTEyYjQiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.pshvZPi9ZJkXUWuWilcYs1wawTpzV-nMKesgF3d_l7qyTPaK2N5ofzIThd0SjzU7BFNb4_rOm1dw1Be5kLeHjY_YW5lDnM5TAxVPXmZQ0HJ2pAQ0pjQqCHFnPD0bZFIYkeyz8pZx0Hmwcd3ZdC1yztr0ADpTAmMgI9NC2ZFIeoFFo4Ue9ZM_ulhqJQjmgoAlI_qbyjuKCNsWeEQBwM6HHHAsH1gOQIdVxqQ83OQZUuynDQRpqlHHFIndbK2zVRYFA3GgUnTu2-VRQ-DXBFRjvZR5qArnC1f383jmIjGT6VO7l04QJteG_LFetRbXa-T4mcnbsd8XutSgO0INqwKpjw
+ca.crt: 1363 bytes
+namespace: 11 bytes
+```
+
+## 13.6登录dashboard
+
+https://192.168.1.76:31245/
+
+eyJhbGciOiJSUzI1NiIsImtpZCI6InYzV2dzNnQzV3hHb2FQWnYzdnlOSmpudmtpVmNjQW5VM3daRi12SFM4dEEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWs1NDVrIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJjMzA4MDcxYy00Y2Y1LTQ1ODMtODNhMi1lYWY3ODEyNTEyYjQiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.pshvZPi9ZJkXUWuWilcYs1wawTpzV-nMKesgF3d_l7qyTPaK2N5ofzIThd0SjzU7BFNb4_rOm1dw1Be5kLeHjY_YW5lDnM5TAxVPXmZQ0HJ2pAQ0pjQqCHFnPD0bZFIYkeyz8pZx0Hmwcd3ZdC1yztr0ADpTAmMgI9NC2ZFIeoFFo4Ue9ZM_ulhqJQjmgoAlI_qbyjuKCNsWeEQBwM6HHHAsH1gOQIdVxqQ83OQZUuynDQRpqlHHFIndbK2zVRYFA3GgUnTu2-VRQ-DXBFRjvZR5qArnC1f383jmIjGT6VO7l04QJteG_LFetRbXa-T4mcnbsd8XutSgO0INqwKpjw
+
+# 14.安装命令行自动补全功能
+
+```
+yum install bash-completion -y
+source /usr/share/bash-completion/bash_completion
+source <(kubectl completion bash)
+echo "source <(kubectl completion bash)" >> ~/.bashrc
+```
+
+# 附录：
+
+配置kube-controller-manager有效期100年（能不能生效的先配上再说）
+
+```shell
+vim /usr/lib/systemd/system/kube-controller-manager.service
+
+# [Service]下找个地方加上
+
+--cluster-signing-duration=876000h0m0s \
+
+
+# 重启
+
+systemctl daemon-reload
+systemctl restart kube-controller-manager
+```
+
+防止漏洞扫描
+
+```shell
+vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf
+
+[Service]
+Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.kubeconfig --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig"
+Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
+Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6"
+Environment="KUBELET_EXTRA_ARGS=--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --image-pull-progress-deadline=30m"
+ExecStart=
+ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS
+```
+
+预留空间，按需分配
+
+```shell
+vim /etc/kubernetes/kubelet-conf.yml
+
+rotateServerCertificates: true
+allowedUnsafeSysctls:
+
+ - "net.core*"
+ - "net.ipv4.*"
+ kubeReserved:
+ cpu: "1"
+ memory: 1Gi
+ ephemeral-storage: 10Gi
+ systemReserved:
+ cpu: "1"
+ memory: 1Gi
+ ephemeral-storage: 10Gi
+```
+
+数据盘要与系统盘分开；etcd使用ssd磁盘
+
diff --git a/calico/calico.yaml b/calico/calico.yaml
index 436d7ad..7aff274 100644
--- a/calico/calico.yaml
+++ b/calico/calico.yaml
@@ -4361,7 +4361,7 @@ spec:
# chosen from this range. Changing this value after installation will have
# no effect. This should fall within `--cluster-cidr`.
- name: CALICO_IPV4POOL_CIDR
- value: "POD_CIDR"
+ value: "172.16.0.0/12"
# Disable file logging so `kubectl logs` works.
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
diff --git a/dashboard/admin.yaml b/dashboard/admin.yaml
new file mode 100644
index 0000000..a9e4fbf
--- /dev/null
+++ b/dashboard/admin.yaml
@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: admin-user
+
+ namespace: kube-system
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: admin-user
+ annotations:
+ rbac.authorization.kubernetes.io/autoupdate: "true"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cluster-admin
+subjects:
+
+- kind: ServiceAccount
+ name: admin-user
+ namespace: kube-system
+