master节点初始化
1、准备系统环境
1）系统安装。CentOS版本：CentOS Linux release 7.4.1708 (Core)，内核版本：3.10.0-693.el7.x86_64； 2）配置网卡名和MAC地址的绑定、配置网卡绑定； 3）配置主机名和网络； 4）挂载磁盘，创建文件系统：一个分区挂载根目录，另一个分区挂载到/data。
2、配置Ansible管理节点
1）/etc/hosts文件配置
[root@d3-d3-master-001 ~]# mkdir kubernetes && cd kubernetes && cat > env.sh <<EOF
#!/bin/sh
export MASTER01_HOSTNAME="d3-master-001"
export MASTER02_HOSTNAME="d3-master-002"
export MASTER03_HOSTNAME="d3-master-003"
export MASTER01_IP="10.24.10.74"
export MASTER02_IP="10.24.10.75"
export MASTER03_IP="10.24.10.76"
export APISERVER_VIP="10.24.10.114"
export IFACE="bond0"
export SSH_PORT=5837
EOF
[root@d3-d3-master-001 kubernetes]# source env.sh
[root@d3-d3-master-001 kubernetes]# cat > /etc/hosts <<EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
################ kubernetes ################
$APISERVER_VIP apiserver-vip
$MASTER01_IP $MASTER01_HOSTNAME
$MASTER02_IP $MASTER02_HOSTNAME
$MASTER03_IP $MASTER03_HOSTNAME
10.12.8.248 docker.v2.aispeech.com
############################################
EOF
2）配置SSH无密码连接
[root@d3-master-001 kubernetes]# ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa
[root@d3-master-001 kubernetes]# for node in $MASTER0{1..3}_HOSTNAME;do ssh-copy-id -i ~/.ssh/id_rsa.pub -p $SSH_PORT root@$node;done
[root@d3-master-001 kubernetes]# for node in $MASTER0{1..3}_HOSTNAME;do ssh -p $SSH_PORT root@$node hostname;done
d3-master-001
d3-master-002
d3-master-003
3）配置Ansible主机清单文件
[root@d3-master-001 kubernetes]# yum install ansible -y
[root@d3-master-001 kubernetes]# cat > /etc/ansible/hosts <<EOF
[k8s-all:children]
k8s-masters
k8s-workers
[k8s-masters]
$MASTER01_HOSTNAME ansible_host=$MASTER01_IP ansible_port=$SSH_PORT ansible_user=root
$MASTER02_HOSTNAME ansible_host=$MASTER02_IP ansible_port=$SSH_PORT ansible_user=root
$MASTER03_HOSTNAME ansible_host=$MASTER03_IP ansible_port=$SSH_PORT ansible_user=root
[k8s-workers]
EOF
[root@d3-master-001 kubernetes]# ansible k8s-masters --list
hosts (3):
d3-master-001
d3-master-002
d3-master-003
[root@d3-master-001 kubernetes]# ansible k8s-masters -m ping
[root@d3-master-001 kubernetes]# ansible k8s-masters -m copy -a "src=/etc/hosts dest=/etc/hosts"
3、配置Yum软件源
1）准备Yum软件源配置文件
//系统自带软件源
[root@d3-master-001 kubernetes]# cat > docker/CentOS-Base.repo <<"EOF"
[base]
name=CentOS-$releasever - Base
baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/
enabled=1
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
[updates]
name=CentOS-$releasever - Updates
baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/
enabled=1
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
[extras]
name=CentOS-$releasever - Extras
baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/
enabled=1
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
EOF
//epel软件源
[root@d3-master-001 kubernetes]# cat > docker/epel.repo <<"EOF"
[epel]
name=Extra Packages for Enterprise Linux 7 - $basearch
baseurl=http://mirrors.aliyun.com/epel/7/$basearch
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
EOF
//docker软件源
[root@d3-master-001 kubernetes]# cat > docker/docker.repo <<"EOF"
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
EOF
//kubernetes软件源
[root@d3-master-001 kubernetes]# cat > docker/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
2）配置Yum软件源
[root@d3-master-001 kubernetes]# cat > pb-master-yum.yaml <<EOF
- hosts: k8s-masters
remote_user: root
tasks:
# configure yum repository
- name: archive original repository
shell: mkdir -p /etc/yum.repos.d/bak && mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak
ignore_errors: yes
- name: copy yum repositories
copy:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
with_items:
- {src: 'docker/CentOS-Base.repo',dest: '/etc/yum.repos.d/CentOS-Base.repo'}
- {src: 'docker/epel.repo',dest: '/etc/yum.repos.d/epel.repo'}
#- {src: 'docker/docker.repo',dest: '/etc/yum.repos.d/docker.repo'}
- {src: 'docker/kubernetes.repo',dest: '/etc/yum.repos.d/kubernetes.repo'}
- name: install some useful packages
yum:
name: nfs-utils,pciutils,psmisc,tcpdump,net-tools,telnet,nmap-ncat,bash-completion,tree
state: present
EOF
[root@d3-master-001 kubernetes]# ansible-playbook pb-master-yum.yaml
3）检查软件源
[root@d3-master-001 kubernetes]# yum repolist
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
repo id repo name status
base/7/x86_64 CentOS-7 - Base 10,097
docker-ce-stable/x86_64 Docker CE Stable - x86_64 58
epel/x86_64 Extra Packages for Enterprise Linux 7 - x86_64 13,426
extras/7/x86_64 CentOS-7 - Extras 304
kubernetes Kubernetes 409
updates/7/x86_64 CentOS-7 - Updates 332
repolist: 24,626
4、安装docker
开发环境、测试环境和华北灾备环境均使用18.06版本docker-ce，但是这里部署18.06版本后，多个节点多次出现kernel:NMI watchdog: BUG: soft lockup - CPU#31 stuck for 22s! [runc:[1:CHILD]:66572]错误直接导致整个系统hang死，可能原因是内核bug、CPU供电不足导致电压不稳、CPU虚拟化问题等，由于这批服务器使用了较老的E5或者E5 v2版本CPU，怀疑是老版CPU和新版runc兼容性有问题导致的，故降级为17.03版本docker-ce，降级之后没再出现soft lockup故障。
1）准备docker软件包和配置文件
docker-ce-17.03软件包：
[root@d3-master-001 kubernetes]# ll docker/*.rpm
-rw-r--r-- 1 root root 19529520 Oct 19 05:56 docker/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm
-rw-r--r-- 1 root root 29108 Oct 19 05:56 docker/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm
docker.service配置文件：
[root@d3-master-001 kubernetes]# cat > docker/docker.conf <<EOF
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd -H=unix:///var/run/docker.sock --graph=/data/docker --insecure-registry docker.v2.aispeech.com --registry-mirror=https://woom60mt.mirror.aliyuncs.com --group=docker --max-concurrent-downloads=10 --max-concurrent-uploads=10 --live-restore
EOF
- 注意：17.03版本使用
--graph而不是--data-root参数指定docker的工作目录； - 其他用户可以添加到
--group参数指定的用户组，获取使用docker命令的权限。
针对非GPU节点的/etc/docker/daemon.json文件：
[root@d3-master-001 kubernetes]# cat > docker/docker-daemon-cpu.json <<EOF
{
"exec-opts": ["native.cgroupdriver=cgroupfs"],
"log-driver": "json-file",
"log-opts": {
"max-size": "1024m",
"max-file": "3"
},
"storage-driver": "overlay2",
"storage-opts": [
"overlay2.override_kernel_check=true"
]
}
EOF
- 指定Docker的Cgroup驱动使用Linux的cgroupfs而非Kubernetes 1.14推荐的CentOS7的systemd，因为使用systemd时kubelet大量报
container.go:409] Failed to create summary reader for "/libcontainer_7554_systemd_test_default.slice": none of the resources are being tracked.错误，参考 https://github.com/kubernetes/kubernetes/issues/76531#issuecomment-522286281 ； - 指定Docker的日志驱动使用json-file，同时开启日志滚动——Kubernetes推荐的实现日志滚动的方法，缺点是日志滚动之后
kubectl logs命令无法查看之前被滚动的日志，不过这些日志可以在kibana中查看；针对每个容器，宿主机上最多保存3个1GB的日志文件，确保宿主机的磁盘不会被日志占满； - 指定Docker的存储驱动使用overlay2而非淘汰的overlay或者性能低下的devicemapper。
2）安装docker-ce
[root@d3-master-001 kubernetes]# cat > pb-master-docker.yaml <<EOF
- hosts: k8s-masters
remote_user: root
tasks:
# install docker-ce
- name: copy packages of the new version of docker-ce
copy:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
with_items:
- { src: 'docker/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm', dest: '/tmp/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm' }
- { src: 'docker/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm', dest: '/tmp/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm' }
- name: install new version of docker-ce
yum:
name: /tmp/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm,/tmp/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm
- name: delete packages of the new version of docker-ce
file:
path: '{{ item.name }}'
state: absent
with_items:
- { name: '/tmp/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm' }
- { name: '/tmp/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm' }
# configure docker
- name: mkdir /data/docker/
file: name=/data/docker/ state=directory
- name: mkdir /etc/docker/
file: path=/etc/docker/ state=directory
- name: mkdir /etc/systemd/system/docker.service.d/
file: path=/etc/systemd/system/docker.service.d/ state=directory
- name: copy docker.conf
copy:
src: docker/docker.conf
dest: /etc/systemd/system/docker.service.d/docker.conf
- name: copy daemon.json
copy:
src: docker/docker-daemon-cpu.json
dest: /etc/docker/daemon.json
# start docker.service
- name: start docker.service
systemd: name=docker state=restarted enabled=yes daemon_reload=yes
EOF
[root@d3-master-001 kubernetes]# ansible-playbook pb-master-docker.yaml
3）docker info信息
[root@d3-master-001 kubernetes]# docker info
Containers: 0
Running: 0
Paused: 0
Stopped: 0
Images: 0
Server Version: 17.03.2-ce
Storage Driver: overlay2
Backing Filesystem: extfs
Supports d_type: true
Native Overlay Diff: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host macvlan null overlay
Swarm: inactive
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: 4ab9917febca54791c5f071a9d1f404867857fcc
runc version: 54296cf40ad8143b62dbcaa1d90e520a2136ddfe
init version: 949e6fa
Security Options:
seccomp
Profile: default
Kernel Version: 3.10.0-693.el7.x86_64
Operating System: CentOS Linux 7 (Core)
OSType: linux
Architecture: x86_64
CPUs: 12
Total Memory: 31.15 GiB
Name: d3-master-001
ID: TUB7:GPAY:GTVC:K2CH:J7RX:RZQY:T42J:267T:WUVF:GC4C:HUV7:Q6HY
Docker Root Dir: /data/docker
Debug Mode (client): false
Debug Mode (server): false
File Descriptors: 95
Goroutines: 86
System Time: 2019-11-11T16:36:33.953867132+08:00
EventsListeners: 0
Registry: https://index.docker.io/v1/
Experimental: false
Insecure Registries:
docker.v2.aispeech.com
127.0.0.0/8
Registry Mirrors:
https://woom60mt.mirror.aliyuncs.com/
Live Restore Enabled: true
5、安装Kubernetes
1)准备配置文件
[root@d3-master-001 kubernetes]# cat > docker/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
EOF
2)安装Kubernetes
[root@d3-master-001 kubernetes]# cat > pb-master-kubernetes.yaml <<EOF
- hosts: k8s-masters
remote_user: root
tasks:
# prepare system
- name: disable swap
shell: swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
- name: disable firewalld
systemd: name=firewalld state=stopped enabled=no
- name: config iptables
shell: iptables -P INPUT ACCEPT && iptables -P FORWARD ACCEPT && iptables -P OUTPUT ACCEPT
- name: disable selinux
selinux: state=disabled
- name: add harbor's address to /etc/hosts
lineinfile:
path: /etc/hosts
regexp: 'docker\.v2\.aispeech\.com'
line: '{{harborIP}} docker.v2.aispeech.com'
# modify dns
- shell: nmcli con mod bond0 ipv4.dns "223.5.5.5 180.76.76.76 119.29.29.29"
name: modify bond0's dns servers
- shell: nmcli con mod bond0 ipv4.dns-options "options rotate options timeout:1 attempts:1"
name: modify bond0's dns options
- shell: nmcli con reload
name: reload bond0
# configure ntp
- name: install chrony
yum: name=chrony.x86_64 state=present
- name: enable chronyd.service
systemd: name=chronyd state=started enabled=yes
- name: configure ntp
shell: timedatectl set-timezone Asia/Shanghai && timedatectl set-ntp yes && timedatectl set-local-rtc 0
# install kubernetes
- name: install kubernetes v1.14.6
yum:
name: kubeadm-1.14.6-0.x86_64,kubelet-1.14.6-0.x86_64,kubectl-1.14.6-0.x86_64,kubernetes-cni-0.7.5-0.x86_64,cri-tools-1.13.0-0.x86_64
state: present
- name: enable kubelet.service
systemd: name=kubelet enabled=yes daemon_reload=yes
# configure kubernetes
- name: copy k8s.conf
copy:
src: docker/k8s.conf
dest: /etc/sysctl.d/k8s.conf
- name: modify sysctl.conf
shell: sed -i '/vm.swappiness=1/d' /etc/sysctl.conf
- name: make k8s.conf effective
shell: sysctl --system
# install ipvs required kernel modules and packages for kube-proxy
- name: install ipvs required packages
yum:
name: conntrack-tools,ipvsadm,ipset
state: present
- name: load ipvs required kernel modules
lineinfile:
path: /usr/lib/systemd/system/kubelet.service
regexp: 'ExecStartPre'
insertafter: '\[Service\]'
line: 'ExecStartPre=/usr/sbin/modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4'
EOF
[root@d3-master-001 kubernetes]# ansible-playbook pb-master-kubernetes.yaml
master节点配置
1、部署kubernetes使用的etcd集群kube-etcd
1）生成etcd-ca 安装cfssl工具集：
[root@d3-master-001 kubernetes]# curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o cfssl
[root@d3-master-001 kubernetes]# curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o cfssljson
[root@d3-master-001 kubernetes]# curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o cfssl-certinfo
[root@d3-master-001 kubernetes]# chmod +x cfssl* && mv cfssl* /usr/local/bin/
生成ca配置文件：
[root@d3-master-001 kubernetes]# mkdir cert && cat > cert/ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
证书签名请求文件：
[root@d3-master-001 kubernetes]# cat > cert/etcd-ca-csr.json <<EOF
{
"CN": "etcd-ca",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
],
"ca": {
"expiry": "262800h"
}
}
EOF
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert -initca cert/etcd-ca-csr.json|cfssljson -bare cert/etcd-ca
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/etcd-ca.pem |head -n 11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
11:cf:f3:cf:ea:69:8d:c5:5e:fc:55:5c:4b:ee:dd:79:ce:16:61:77
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=etcd-ca
Validity
Not Before: Oct 10 01:45:00 2019 GMT
Not After : Oct 2 01:45:00 2049 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=etcd-ca
2）创建etcd使用的server证书和私钥 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/kube-etcd-csr.json <<EOF
{
"CN": "kube-etcd",
"hosts": [
"localhost",
"127.0.0.1",
"$MASTER01_IP",
"$MASTER02_IP",
"$MASTER03_IP"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
]
}
EOF
- hosts 字段指定授权使用该证书的etcd节点IP或域名列表，这里将etcd集群的三个节点IP都列在其中。
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/etcd-ca.pem \
-ca-key=cert/etcd-ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/kube-etcd-csr.json | cfssljson -bare cert/kube-etcd
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/kube-etcd.pem |head -n 11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
78:ef:94:af:fd:ec:12:28:70:ae:bd:90:05:10:0f:19:57:d5:b7:45
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=etcd-ca
Validity
Not Before: Oct 10 01:46:00 2019 GMT
Not After : Oct 7 01:46:00 2029 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kube-etcd
3）下载etcd二进制文件
[root@d3-master-001 kubernetes]# wget https://github.com/etcd-io/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz
[root@d3-master-001 kubernetes]# tar -zxf etcd-v3.3.13-linux-amd64.tar.gz
4）创建etcd的systemd unit模板文件
[root@d5-dui-d3-master-001 kubernetes]# cat > kube-etcd.service.j2 <<EOF
[Unit]
Description=Etcd Server for Kubernetes
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/kube-etcd/
ExecStart=/usr/local/bin/kube-etcd \\
--data-dir=/var/lib/kube-etcd/ \\
--name={{ ansible_hostname }} \\
--client-cert-auth \\
--cert-file=/etc/kubernetes/pki/etcd/server.crt \\
--key-file=/etc/kubernetes/pki/etcd/server.key \\
--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt \\
--peer-client-cert-auth \\
--peer-cert-file=/etc/kubernetes/pki/etcd/server.crt \\
--peer-key-file=/etc/kubernetes/pki/etcd/server.key \\
--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt \\
--listen-peer-urls=https://{{ ansible_$IFACE.ipv4.address }}:2380 \\
--initial-advertise-peer-urls=https://{{ ansible_$IFACE.ipv4.address }}:2380 \\
--listen-client-urls=https://{{ ansible_$IFACE.ipv4.address }}:2379,https://127.0.0.1:2379 \\
--advertise-client-urls=https://{{ ansible_$IFACE.ipv4.address }}:2379 \\
--initial-cluster-token=kube-etcd-cluster \\
--initial-cluster=d3-master-001=https://$MASTER01_IP:2380,d3-master-002=https://$MASTER02_IP:2380,d3-master-003=https://$MASTER03_IP:2380 \\
--initial-cluster-state=new \\
--auto-compaction-mode=periodic \\
--auto-compaction-retention=1 \\
--max-request-bytes=10485760 \\
--quota-backend-bytes=6442450944 \\
--heartbeat-interval=250 \\
--election-timeout=2000
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
5）分发etcd相关证书、二进制文件和etcd.service文件
[root@d3-master-001 kubernetes]# cat > pb-kube-etcd.yaml <<EOF
- hosts: k8s-masters
remote_user: root
tasks:
- name: mkdir /etc/kubernetes/pki/etcd
file: path=/etc/kubernetes/pki/etcd state=directory
- name: copy etcd certs
copy:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
with_items:
- {src: 'cert/etcd-ca.pem',dest: '/etc/kubernetes/pki/etcd/ca.crt'}
- {src: 'cert/etcd-ca-key.pem',dest: '/etc/kubernetes/pki/etcd/ca.key'}
- {src: 'cert/kube-etcd.pem',dest: '/etc/kubernetes/pki/etcd/server.crt'}
- {src: 'cert/kube-etcd-key.pem',dest: '/etc/kubernetes/pki/etcd/server.key'}
- name: copy etcd binary file
copy:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
mode: '{{ item.mode }}'
with_items:
- {src: 'etcd-v3.3.13-linux-amd64/etcd',dest: '/usr/local/bin/kube-etcd',mode: 'u+x'}
- {src: 'etcd-v3.3.13-linux-amd64/etcdctl',dest: '/usr/local/bin/',mode: 'u+x'}
- name: copy kube-etcd.service
template:
src: kube-etcd.service.j2
dest: /usr/lib/systemd/system/kube-etcd.service
- name: create the data directory for kube-etcd
file: path=/var/lib/kube-etcd/ state=directory
- name: enable and start etcd.service
systemd: name=kube-etcd state=restarted enabled=yes daemon_reload=yes
EOF
[root@d3-master-001 kubernetes]# ansible-playbook pb-kube-etcd.yaml
[root@d3-master-001 kubernetes]# ansible k8s-masters -m shell -a "systemctl status kube-etcd.service|grep -e Loaded -e Active"
- 重新部署kube-etcd：
[root@d3-master-001 kubernetes]# ansible k8s-masters -m systemd -a "name=kube-etcd state=stopped"
[root@d3-master-001 kubernetes]# ansible k8s-masters -m file -a "name=/var/lib/kube-etcd/ state=absent"
[root@d3-master-001 kubernetes]# ansible-playbook pb-kube-etcd.yaml
6）验证etcd集群状态
[root@d3-master-001 kubernetes]# netstat -lnptu|grep etcd
tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 35441/kube-etcd
tcp 0 0 10.24.10.74:2379 0.0.0.0:* LISTEN 35441/kube-etcd
tcp 0 0 10.24.10.74:2380 0.0.0.0:* LISTEN 35441/kube-etcd
[root@d3-master-001 kubernetes]# ETCDCTL_API=3 etcdctl \
--endpoints=https://$MASTER01_IP:2379,https://$MASTER02_IP:2379,https://$MASTER03_IP:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
endpoint health
https://10.24.10.74:2379 is healthy: successfully committed proposal: took = 2.200039ms
https://10.24.10.75:2379 is healthy: successfully committed proposal: took = 2.329094ms
https://10.24.10.76:2379 is healthy: successfully committed proposal: took = 1.515048ms
查看集群成员：
[root@d3-master-001 kubernetes]# ETCDCTL_API=3 etcdctl \
--endpoints=https://$MASTER01_IP:2379,https://$MASTER02_IP:2379,https://$MASTER03_IP:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
member list -w table
+------------------+---------+---------------+--------------------------+--------------------------+
| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS |
+------------------+---------+---------------+--------------------------+--------------------------+
| 2903ce97206d7a99 | started | d3-master-001 | https://10.24.10.74:2380 | https://10.24.10.74:2379 |
| 35e16f7250799a1a | started | d3-master-003 | https://10.24.10.76:2380 | https://10.24.10.76:2379 |
| 7eb1e991b93d004c | started | d3-master-002 | https://10.24.10.75:2380 | https://10.24.10.75:2379 |
+------------------+---------+---------------+--------------------------+--------------------------+
查看当前的leader节点：
[root@d3-master-001 kubernetes]# ETCDCTL_API=3 etcdctl \
--endpoints=https://$MASTER01_IP:2379,https://$MASTER02_IP:2379,https://$MASTER03_IP:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
endpoint status -w table
+--------------------------+------------------+---------+---------+-----------+-----------+------------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+--------------------------+------------------+---------+---------+-----------+-----------+------------+
| https://10.24.10.74:2379 | 2903ce97206d7a99 | 3.3.13 | 20 kB | false | 2 | 8 |
| https://10.24.10.75:2379 | 7eb1e991b93d004c | 3.3.13 | 20 kB | false | 2 | 8 |
| https://10.24.10.76:2379 | 35e16f7250799a1a | 3.3.13 | 20 kB | true | 2 | 8 |
+--------------------------+------------------+---------+---------+-----------+-----------+------------+
2、部署apiserver的负载均衡器
这里使用keepalived和haproxy实现kube-apiserver的高可用，其中：
- keepalived提供kube-apiserver对外服务的VIP($APISERVER_VIP)；
- haproxy监听VIP(tcp/8443)，后端连接所有kube-apiserver实例(tcp/6443)，提供健康检查和负载均衡功能。
keepalived在运行过程中周期检查本机的haproxy进程状态，如果检测到haproxy进程异常，则触发重新选主的过程，VIP将飘移到新选出来的主节点，从而实现VIP的高可用。 所有组件（如kubectl、apiserver、controller-manager、scheduler等）都通过VIP和haproxy监听的8443端口访问kube-apiserver服务。
1）安装Keepalived和Haproxy Keepalived：
[root@d3-master-001 kubernetes]# ansible k8s-masters -m yum -a "name=keepalived,psmisc state=latest"
[root@d3-master-001 kubernetes]# rpm -q keepalived
keepalived-1.3.5-16.el7.x86_64
- 其中psmisc软件包提供killall命令。
Haproxy： Yum源提供的haproxy版本为“haproxy-1.5.18-9.el7.x86_64”，不支持某些功能，这里直接使用
[root@d3-master-001 kubernetes]# wget https://www.haproxy.org/download/2.0/src/haproxy-2.0.7.tar.gz
[root@d3-master-001 kubernetes]# tar -zxf haproxy-2.0.7.tar.gz
[root@d3-master-001 kubernetes]# cd haproxy-2.0.7/
[root@d3-master-001 haproxy-2.0.7]# make clean
[root@d3-master-001 haproxy-2.0.7]# yum install pcre-devel openssl-devel zlib-devel -y
[root@d3-master-001 haproxy-2.0.7]# make -j $(nproc) TARGET=linux-glibc \
USE_OPENSSL=1 USE_ZLIB=1 USE_PCRE=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_SYSTEMD=1 \
EXTRA_OBJS="contrib/prometheus-exporter/service-prometheus.o"
[root@d3-master-001 haproxy-2.0.7]# ll haproxy
-rwxr-xr-x 1 root root 12473768 Oct 17 10:55 haproxy
[root@d3-master-001 haproxy-2.0.7]# cd ..
2）haproxy配置文件 haproxy.service：
[root@d3-master-001 kubernetes]# cat > haproxy.service <<"EOF"
[Unit]
Description=HAProxy Load Balancer for Kubernetes Apiserver
After=syslog.target network.target
[Service]
EnvironmentFile=-/etc/sysconfig/haproxy
ExecStartPre=/usr/local/bin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -c -q
ExecStart=/usr/local/bin/haproxy -W -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid $OPTIONS
ExecReload=/bin/kill -USR2 $MAINPID
KillMode=mixed
Restart=always
Type=forking
[Install]
WantedBy=multi-user.target
EOF
haproxy.cfg：
[root@d3-master-001 kubernetes]# openssl rand -base64 15
fVB6UUYpa2y8OPZzrysA
[root@d3-master-001 kubernetes]# cat > haproxy.cfg <<EOF
global
log /dev/log local0
log /dev/log local1 notice
chroot /var/lib/haproxy # chroot增加系统安全性
stats socket /var/run/haproxy-admin.sock mode 660 level admin
stats timeout 30s
user haproxy
group haproxy
daemon
nbproc 1
defaults
log global
timeout connect 5000
timeout client 10m
timeout server 10m
listen admin-status
bind 0.0.0.0:8441
mode http
log 127.0.0.1 local0 err
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /stats
stats refresh 30s
stats realm welcome login\ Haproxy
stats auth admin:fVB6UUYpa2y8OPZzrysA
stats admin if FALSE
listen daoker-etcd
bind 0.0.0.0:8442
mode tcp
option tcplog
balance roundrobin
# option ssl-hello-chk
server d3-master-001 10.24.10.74:2381 check inter 2000 fall 3 rise 3 weight 1
server d3-master-002 10.24.10.75:2381 check inter 2000 fall 3 rise 3 weight 1
server d3-master-003 10.24.10.76:2381 check inter 2000 fall 3 rise 3 weight 1
listen kube-apiserver
bind 0.0.0.0:8443
mode tcp
option tcplog
balance roundrobin
# option ssl-hello-chk
server d3-master-001 10.24.10.74:6443 check inter 2000 fall 3 rise 3 weight 1
server d3-master-002 10.24.10.75:6443 check inter 2000 fall 3 rise 3 weight 1
server d3-master-003 10.24.10.76:6443 check inter 2000 fall 3 rise 3 weight 1
EOF
3）keepalived配置文件 keepalived是一主（master）多备（backup）运行模式，规划如下： master：$MASTER01_IP backup：$MASTER02_IP、$MASTER03_IP vip：$APISERVER_VIP
d3-master-001（MASTER）：
[root@d3-master-001 kubernetes]# cat > keepalived-$MASTER01_HOSTNAME.conf <<EOF
global_defs {
router_id $MASTER01_HOSTNAME
vrrp_mcast_group4 224.0.0.19
default_interface $IFACE
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -50
}
vrrp_instance VI-kube-apiserver {
# state MASTER
state BACKUP
priority 100
nopreempt
interface $IFACE
dont_track_primary
virtual_router_id 114
advert_int 3
authentication {
auth_type PASS
auth_pass kubeapis
}
virtual_ipaddress {
$APISERVER_VIP dev $IFACE label $IFACE:0
}
track_script {
check-haproxy
}
}
EOF
killall -0 haproxy命令检查所在节点的 haproxy 进程是否正常，如果异常则将权重减少50，从而触发重新选主过程；- virtual_router_id 用于标识属于该 HA 的 keepalived 实例，如果有多套 keepalived HA，则必须各不相同；
- router_id 用于标识不同的机器，默认为主机名，必须各不相同；
- 设置keepalived为master故障恢复后不重新抢回VIP，需要配置所有节点为BACKUP且带nopreempt参数。
d3-master-002（BACKUP）：
[root@d3-master-001 kubernetes]# cat > keepalived-$MASTER02_HOSTNAME.conf <<EOF
global_defs {
router_id $MASTER02_HOSTNAME
vrrp_mcast_group4 224.0.0.19
default_interface $IFACE
}
vrrp_script check-haproxy {
script "/usr/bin/killall -0 haproxy"
interval 5
weight -50
}
vrrp_instance VI-kube-apiserver {
state BACKUP
priority 80
nopreempt
interface $IFACE
dont_track_primary
virtual_router_id 114
advert_int 3
authentication {
auth_type PASS
auth_pass kubeapis
}
virtual_ipaddress {
$APISERVER_VIP dev $IFACE label $IFACE:0
}
track_script {
check-haproxy
}
}
EOF
d3-master-003（BACKUP）：
[root@d3-master-001 kubernetes]# cat > keepalived-$MASTER03_HOSTNAME.conf <<EOF
global_defs {
router_id $MASTER03_HOSTNAME
vrrp_mcast_group4 224.0.0.19
default_interface $IFACE
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -50
}
vrrp_instance VI-kube-apiserver {
state BACKUP
priority 60
nopreempt
interface $IFACE
dont_track_primary
virtual_router_id 114
advert_int 3
authentication {
auth_type PASS
auth_pass kubeapis
}
virtual_ipaddress {
$APISERVER_VIP dev $IFACE label $IFACE:0
}
track_script {
check-haproxy
}
}
EOF
4）分发haproxy.cfg和keepalived.conf配置文件并启动服务
[root@d3-master-001 kubernetes]# cat > pb-apiserver-haproxy.yaml <<EOF
- hosts: k8s-masters
remote_user: root
tasks:
- name: copy haproxy executable file
copy:
src: haproxy-2.0.7/haproxy
dest: /usr/local/bin/haproxy
mode: a+x
- name: copy haproxy.service
copy:
src: haproxy.service
dest: /usr/lib/systemd/system/haproxy.service
- name: copy haproxy.cfg
copy:
src: haproxy.cfg
dest: /etc/haproxy/haproxy.cfg
owner: haproxy
group: haproxy
- name: mkdir chroot directory
file:
path: /var/lib/haproxy
state: directory
- name: restart haproxy
systemd:
name: haproxy.service
state: restarted
enabled: yes
daemon_reload: yes
- name: copy keepalived.conf
copy:
src: 'keepalived-{{ ansible_hostname }}.conf'
dest: '/etc/keepalived/keepalived.conf'
- name: restart keepalived
systemd:
name: keepalived.service
state: restarted
enabled: yes
EOF
[root@d3-master-001 kubernetes]# ansible-playbook pb-apiserver-haproxy.yaml
[root@d3-master-001 kubernetes]# ansible k8s-masters -m shell -a "systemctl status haproxy.service keepalived.service|grep -e Loaded -e Active"
[root@d3-master-001 kubernetes]# ifconfig bond0:0
bond0:0: flags=5187<UP,BROADCAST,RUNNING,MASTER,MULTICAST> mtu 1500
inet 10.24.10.114 netmask 255.255.255.255 broadcast 0.0.0.0
ether 34:80:0d:55:f9:ec txqueuelen 1000 (Ethernet)
- ansible需要增加添加haproxy用户的脚本。
5）检查haproxy状态 浏览器访问$MASTER01_IP:10080检查haproxy状态。
3、手动生成所有master节点的证书
手动生成所有master节点的certs和kubeconfig文件（service account key and public key可以省略），并拷贝到指定目录
1）创建kubernetes-ca证书 证书签名请求文件：
[root@d3-master-001 kubernetes]# cat > cert/ca-csr.json <<EOF
{
"CN": "kubernetes-ca",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
],
"ca": {
"expiry": "262800h"
}
}
EOF
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert -initca cert/ca-csr.json |cfssljson -bare cert/ca
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/ca.pem |head -n 11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
13:00:6e:5f:9f:af:b7:8f:72:95:bb:5b:d3:02:0c:16:57:64:7e:93
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-ca
Validity
Not Before: Oct 10 02:37:00 2019 GMT
Not After : Oct 2 02:37:00 2049 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-ca
2）创建kubernetes-front-proxy-ca证书 证书签名请求文件：
[root@d3-master-001 kubernetes]# cat > cert/front-proxy-ca-csr.json <<EOF
{
"CN": "kubernetes-front-proxy-ca",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
],
"ca": {
"expiry": "262800h"
}
}
EOF
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert -initca cert/front-proxy-ca-csr.json |cfssljson -bare cert/front-proxy-ca
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/front-proxy-ca.pem |head -n 11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
2c:53:00:f3:ba:4f:1d:e5:6b:3b:79:77:32:c6:91:44:18:97:7e:8c
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-front-proxy-ca
Validity
Not Before: Oct 10 02:38:00 2019 GMT
Not After : Oct 2 02:38:00 2049 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-front-proxy-ca
3）创建apiserver使用的kube-apiserver-etcd-client证书 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/apiserver-etcd-client-csr.json <<"EOF"
{
"CN": "kube-apiserver-etcd-client",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "system:masters",
"OU": "aispeech"
}
]
}
EOF
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/etcd-ca.pem \
-ca-key=cert/etcd-ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/apiserver-etcd-client-csr.json| cfssljson -bare cert/apiserver-etcd-client
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/apiserver-etcd-client.pem |head -n 11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
51:31:a2:30:e8:bf:d8:46:be:16:d6:6a:9e:67:aa:b7:cc:fc:2f:83
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=etcd-ca
Validity
Not Before: Oct 10 02:39:00 2019 GMT
Not After : Oct 7 02:39:00 2029 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=system:masters, OU=aispeech, CN=kube-apiserver-etcd-client
4)创建front-proxy-client证书和私钥
[root@d3-master-001 kubernetes]# cat > cert/front-proxy-client-csr.json <<EOF
{
"CN": "front-proxy-client",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
]
}
EOF
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/front-proxy-ca.pem \
-ca-key=cert/front-proxy-ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/front-proxy-client-csr.json| cfssljson -bare cert/front-proxy-client
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/front-proxy-client.pem |head -n11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
60:6e:6c:78:1b:87:f1:7c:f9:05:5f:49:48:5b:20:ad:fd:be:c5:b0
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-front-proxy-ca
Validity
Not Before: Oct 10 02:39:00 2019 GMT
Not After : Oct 7 02:39:00 2029 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=front-proxy-client
5)创建apiserver使用的server证书和私钥 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/apiserver-csr.json <<EOF
{
"CN": "kube-apiserver",
"hosts": [
"127.0.0.1",
"localhost",
"$MASTER01_HOSTNAME",
"$MASTER02_HOSTNAME",
"$MASTER03_HOSTNAME",
"$APISERVER_VIP",
"$MASTER01_IP",
"$MASTER02_IP",
"$MASTER03_IP",
"10.96.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
]
}
EOF
- 其中CN必须为“kube-apiserver”；
- 指定授权使用该证书的IP或域名列表，这里列出了apiserver节点IP、kubernetes服务的Service IP和域名；
- 如果使用了Keepalived等高可用，hosts字段需要加入VIP。
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/ca.pem \
-ca-key=cert/ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/apiserver-csr.json| cfssljson -bare cert/apiserver
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/apiserver.pem|head -n11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
27:b1:91:ce:bf:f3:c6:a5:81:89:65:3a:4c:03:cd:3a:79:b8:e0:13
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-ca
Validity
Not Before: Oct 10 02:41:00 2019 GMT
Not After : Oct 7 02:41:00 2029 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kube-apiserver
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/apiserver.pem|grep -A1 "X509v3 Subject Alternative Name"
X509v3 Subject Alternative Name:
DNS:localhost, DNS:d3-master-001, DNS:d3-master-002, DNS:d3-master-003, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster, DNS:kubernetes.default.svc.cluster.local, IP Address:127.0.0.1, IP Address:10.24.10.114, IP Address:10.24.10.74, IP Address:10.24.10.75, IP Address:10.24.10.76, IP Address:10.96.0.1
- 证书10年有效期。
6)创建apiserver访问kubelet的client证书 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/apiserver-kubelet-client-csr.json <<EOF
{
"CN": "apiserver-kubelet-client",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "system:masters",
"OU": "aispeech"
}
]
}
EOF
- 其中"O": "system:masters"对应ClusterRoleBinding:cluster-admin的Group。
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/ca.pem \
-ca-key=cert/ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/apiserver-kubelet-client-csr.json| cfssljson -bare cert/apiserver-kubelet-client
检查证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/apiserver-kubelet-client.pem|head -n11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
47:8d:80:42:02:f3:1f:c5:10:98:5e:ab:72:51:b7:f4:5a:bb:27:53
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-ca
Validity
Not Before: Oct 10 02:43:00 2019 GMT
Not After : Oct 7 02:43:00 2029 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=system:masters, OU=aispeech, CN=apiserver-kubelet-client
7）创建kubectl使用的admin证书和私钥 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/admin-csr.json <<EOF
{
"CN": "kubernetes-admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "system:masters",
"OU": "aispeech"
}
]
}
EOF
- 其中"O": "system:masters"对应ClusterRoleBinding:cluster-admin的Group，即该证书拥有Group:system:masters的权限——集群的最高权限。
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/ca.pem \
-ca-key=cert/ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/admin-csr.json| cfssljson -bare cert/admin
查看证书信息：
[root@d3-master-001 kubernetes]# openssl x509 -noout -text -in cert/admin.pem|head -n11
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
39:f7:6f:cb:3f:43:e9:ca:b8:63:00:a5:c5:e3:33:42:34:49:63:e0
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=CN, ST=Jiangsu, L=Suzhou, O=k8s, OU=aispeech, CN=kubernetes-ca
Validity
Not Before: Oct 10 02:43:00 2019 GMT
Not After : Oct 7 02:43:00 2029 GMT
Subject: C=CN, ST=Jiangsu, L=Suzhou, O=system:masters, OU=aispeech, CN=kubernetes-admin
创建kubectl使用的kubeconfig文件：
[root@d3-master-001 kubernetes]# kubectl config set-cluster kubernetes \
--certificate-authority=cert/ca.pem \
--server=https://$APISERVER_VIP:8443 \
--embed-certs=true \
--kubeconfig=admin.kubeconfig
Cluster "kubernetes" set.
[root@d3-master-001 kubernetes]# kubectl config set-credentials kubernetes-admin \
--client-certificate=cert/admin.pem \
--client-key=cert/admin-key.pem \
--embed-certs=true \
--kubeconfig=admin.kubeconfig
User "kubernetes-admin" set.
[root@d3-master-001 kubernetes]# kubectl config set-context kubernetes-admin@kubernetes \
--cluster=kubernetes \
--namespace=default \
--user=kubernetes-admin \
--kubeconfig=admin.kubeconfig
Context "kubernetes-admin@kubernetes" set.
[root@d3-master-001 kubernetes]# kubectl config use-context kubernetes-admin@kubernetes \
--kubeconfig=admin.kubeconfig
Switched to context "kubernetes-admin@kubernetes".
- “--embed-certs=true”将kubectl所需的admin证书和私钥嵌入在kubeconfig文件中。
查看kubeconfig文件的配置信息：
[root@d3-master-001 kubernetes]# kubectl config view --minify --kubeconfig=admin.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.24.10.114:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
namespace: default
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
8）创建controller-manager使用的证书和私钥 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/controller-manager-csr.json <<EOF
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"$MASTER01_IP",
"$MASTER02_IP",
"$MASTER03_IP"
],
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
]
}
EOF
- hosts 列表包含所有 kube-controller-manager 节点 IP；
- CN 为 system:kube-controller-manager，kubernetes 内置的 ClusterRoleBindings system:kube-controller-manager 赋予 kube-controller-manager 工作所需的权限。
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/ca.pem \
-ca-key=cert/ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/controller-manager-csr.json | cfssljson -bare cert/controller-manager
创建kubeconfig文件：
[root@d3-master-001 kubernetes]# kubectl config \
set-cluster kubernetes \
--certificate-authority=cert/ca.pem \
--server=https://$APISERVER_VIP:8443 \
--embed-certs=true \
--kubeconfig=controller-manager.kubeconfig
Cluster "kubernetes" set.
[root@d3-master-001 kubernetes]# kubectl config \
set-credentials system:kube-controller-manager \
--client-certificate=cert/controller-manager.pem \
--client-key=cert/controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=controller-manager.kubeconfig
User "system:kube-controller-manager" set.
[root@d3-master-001 kubernetes]# kubectl config \
set-context system:kube-controller-manager@kubernetes \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=controller-manager.kubeconfig
Context "system:kube-controller-manager@kubernetes" set.
[root@d3-master-001 kubernetes]# kubectl config \
use-context system:kube-controller-manager@kubernetes \
--kubeconfig=controller-manager.kubeconfig
Switched to context "system:kube-controller-manager@kubernetes".
查看kubeconfig文件：
[root@d3-master-001 kubernetes]# kubectl config view --kubeconfig=controller-manager.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.24.10.114:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:kube-controller-manager
name: system:kube-controller-manager@kubernetes
current-context: system:kube-controller-manager@kubernetes
kind: Config
preferences: {}
users:
- name: system:kube-controller-manager
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
9）创建scheduler使用的证书和私钥 创建证书签名请求：
[root@d3-master-001 kubernetes]# cat > cert/scheduler-csr.json <<EOF
{
"CN": "system:kube-scheduler",
"hosts": [
"127.0.0.1",
"$MASTER01_IP",
"$MASTER02_IP",
"$MASTER03_IP"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "k8s",
"OU": "aispeech"
}
]
}
EOF
- hosts 列表包含所有 kube-scheduler 节点 IP；
- CN 为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限。
生成证书和私钥：
[root@d3-master-001 kubernetes]# cfssl gencert \
-ca=cert/ca.pem \
-ca-key=cert/ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
cert/scheduler-csr.json | cfssljson -bare cert/scheduler
创建kubeconfig文件：
[root@d3-master-001 kubernetes]# kubectl config set-cluster kubernetes \
--certificate-authority=cert/ca.pem \
--server=https://$APISERVER_VIP:8443 \
--embed-certs=true \
--kubeconfig=scheduler.kubeconfig
Cluster "kubernetes" set.
[root@d3-master-001 kubernetes]# kubectl config set-credentials system:kube-scheduler \
--client-certificate=cert/scheduler.pem \
--client-key=cert/scheduler-key.pem \
--embed-certs=true \
--kubeconfig=scheduler.kubeconfig
User "system:kube-scheduler" set.
[root@d3-master-001 kubernetes]# kubectl config set-context system:kube-scheduler@kubernetes \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=scheduler.kubeconfig
Context "system:kube-scheduler@kubernetes" set.
[root@d3-master-001 kubernetes]# kubectl config use-context system:kube-scheduler@kubernetes \
--kubeconfig=scheduler.kubeconfig
Switched to context "system:kube-scheduler@kubernetes".
查看kubeconfig文件：
[root@d3-master-001 kubernetes]# kubectl config view --kubeconfig scheduler.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.24.10.114:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:kube-scheduler
name: system:kube-scheduler@kubernetes
current-context: system:kube-scheduler@kubernetes
kind: Config
preferences: {}
users:
- name: system:kube-scheduler
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
10）创建master节点kubelet使用的kubeconfig文件
[root@d3-master-001 kubernetes]# mkdir kubelet
[root@d3-master-001 kubernetes]# for node in $MASTER01_HOSTNAME:$MASTER01_IP $MASTER02_HOSTNAME:$MASTER02_IP $MASTER03_HOSTNAME:$MASTER03_IP;do
node_name=$(echo $node|awk -F':' '$0=$1')
node_ip=$(echo $node|awk -F':' '$0=$2')
echo ">>> ${node_name}"
# 创建证书签名请求：
cat > kubelet/kubelet-${node_name}-csr.json <<EOF
{
"CN": "system:node:${node_name}",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"${node_name}"
],
"names": [
{
"C": "CN",
"ST": "Jiangsu",
"L": "Suzhou",
"O": "system:nodes",
"OU": "aispeech"
}
]
}
EOF
# 生成证书和私钥：
cfssl gencert \
-ca=cert/ca.pem \
-ca-key=cert/ca-key.pem \
-config=cert/ca-config.json \
-profile=kubernetes \
kubelet/kubelet-${node_name}-csr.json | cfssljson -bare kubelet/kubelet-${node_name}
# 创建kubeconfig文件：
kubectl config set-cluster kubernetes \
--certificate-authority=cert/ca.pem \
--embed-certs=true \
--server=https://${APISERVER_VIP}:8443 \
--kubeconfig=kubelet/kubelet-${node_name}.kubeconfig
kubectl config set-credentials system:node:${node_name} \
--client-certificate=kubelet/kubelet-${node_name}.pem \
--client-key=kubelet/kubelet-${node_name}-key.pem \
--embed-certs=true \
--kubeconfig=kubelet/kubelet-${node_name}.kubeconfig
kubectl config set-context system:node:${node_name}@kubernetes \
--cluster=kubernetes \
--user=system:node:${node_name} \
--kubeconfig=kubelet/kubelet-${node_name}.kubeconfig
kubectl config use-context system:node:${node_name}@kubernetes \
--kubeconfig=kubelet/kubelet-${node_name}.kubeconfig
done
查看kubeconfig文件：
[root@d3-master-001 kubernetes]# kubectl config view --kubeconfig=kubelet/kubelet-d3-master-001.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.24.10.114:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:node:d3-master-001
name: system:node:d3-master-001@kubernetes
current-context: system:node:d3-master-001@kubernetes
kind: Config
preferences: {}
users:
- name: system:node:d3-master-001
user:
client-certificate-data: REDACTED
client-key-data: REDACTED
11）分发master节点使用的证书和密钥
[root@d3-master-001 kubernetes]# cat > pb-master-certs.yaml <<EOF
- hosts: k8s-masters
remote_user: root
tasks:
- name: mkdir /etc/kubernetes/pki/
file:
path: /etc/kubernetes/pki/
state: directory
- name: copy certs
copy:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
with_items:
- {src: 'cert/ca.pem',dest: '/etc/kubernetes/pki/ca.crt'}
- {src: 'cert/ca-key.pem',dest: '/etc/kubernetes/pki/ca.key'}
- {src: 'cert/apiserver.pem',dest: '/etc/kubernetes/pki/apiserver.crt'}
- {src: 'cert/apiserver-key.pem',dest: '/etc/kubernetes/pki/apiserver.key'}
- {src: 'cert/apiserver-kubelet-client.pem',dest: '/etc/kubernetes/pki/apiserver-kubelet-client.crt'}
- {src: 'cert/apiserver-kubelet-client-key.pem',dest: '/etc/kubernetes/pki/apiserver-kubelet-client.key'}
- {src: 'cert/apiserver-etcd-client.pem',dest: '/etc/kubernetes/pki/apiserver-etcd-client.crt'}
- {src: 'cert/apiserver-etcd-client-key.pem',dest: '/etc/kubernetes/pki/apiserver-etcd-client.key'}
- {src: 'cert/front-proxy-ca.pem',dest: '/etc/kubernetes/pki/front-proxy-ca.crt'}
- {src: 'cert/front-proxy-ca-key.pem',dest: '/etc/kubernetes/pki/front-proxy-ca.key'}
- {src: 'cert/front-proxy-client.pem',dest: '/etc/kubernetes/pki/front-proxy-client.crt'}
- {src: 'cert/front-proxy-client-key.pem',dest: '/etc/kubernetes/pki/front-proxy-client.key'}
- {src: 'admin.kubeconfig',dest: '/etc/kubernetes/admin.conf'}
- {src: 'controller-manager.kubeconfig',dest: '/etc/kubernetes/controller-manager.conf'}
- {src: 'scheduler.kubeconfig',dest: '/etc/kubernetes/scheduler.conf'}
- {src: 'kubelet/kubelet-{{ ansible_hostname }}.kubeconfig',dest: '/etc/kubernetes/kubelet.conf'}
EOF
[root@d3-master-001 kubernetes]# ansible-playbook pb-master-certs.yaml
4、初始化第一个master节点
1）准备ClusterConfiguration配置文件
[root@d3-master-001 kubernetes]# cat > kubeadm-init.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: v1.14.6
controlPlaneEndpoint: $APISERVER_VIP:8443
imageRepository: docker.v2.aispeech.com/gcr.io
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/14
serviceSubnet: 10.96.0.0/12
etcd:
external:
endpoints:
- https://$MASTER01_IP:2379
- https://$MASTER02_IP:2379
- https://$MASTER03_IP:2379
caFile: /etc/kubernetes/pki/etcd/ca.crt
certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF
2）准备所需的docker镜像
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/kube-apiserver:v1.14.6
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/kube-controller-manager:v1.14.6
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/kube-scheduler:v1.14.6
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/kube-proxy:v1.14.6
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/pause:3.1
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/etcd:3.3.10
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/coredns:1.3.1
[root@daoker ~]# docker pull registry.cn-hangzhou.aliyuncs.com/istios/flannel:v0.11.0-amd64
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/kube-apiserver:v1.14.6 docker.v2.aispeech.com/gcr.io/kube-apiserver:v1.14.6
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/kube-controller-manager:v1.14.6 docker.v2.aispeech.com/gcr.io/kube-controller-manager:v1.14.6
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/kube-scheduler:v1.14.6 docker.v2.aispeech.com/gcr.io/kube-scheduler:v1.14.6
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/kube-proxy:v1.14.6 docker.v2.aispeech.com/gcr.io/kube-proxy:v1.14.6
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/pause:3.1 docker.v2.aispeech.com/gcr.io/pause:3.1
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/etcd:3.3.10 docker.v2.aispeech.com/gcr.io/etcd:3.3.10
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/coredns:1.3.1 docker.v2.aispeech.com/gcr.io/coredns:1.3.1
[root@daoker ~]# docker tag registry.cn-hangzhou.aliyuncs.com/istios/flannel:v0.11.0-amd64 docker.v2.aispeech.com/gcr.io/flannel:v0.11.0-amd64
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/kube-apiserver:v1.14.6
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/kube-controller-manager:v1.14.6
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/kube-scheduler:v1.14.6
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/kube-proxy:v1.14.6
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/pause:3.1
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/etcd:3.3.10
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/coredns:1.3.1
[root@daoker ~]# docker push docker.v2.aispeech.com/gcr.io/flannel:v0.11.0-amd64
3）kubeadm init初始化第一个master节点
[root@d3-master-001 kubernetes]# kubeadm init --config=kubeadm-init.yaml --experimental-upload-certs
[init] Using Kubernetes version: v1.14.6
[preflight] Running pre-flight checks
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] External etcd mode: Skipping etcd/peer certificate authority generation
[certs] External etcd mode: Skipping etcd/healthcheck-client certificate authority generation
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/admin.conf"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/kubelet.conf"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/controller-manager.conf"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/scheduler.conf"
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 29.503904 seconds
[upload-config] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.14" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Storing the certificates in ConfigMap "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
d155b1519e44de5edbf2769a33858dea4d2af5799b1a3324054dce8b7d3f4aec
[mark-control-plane] Marking the node d3-master-001 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node d3-master-001 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: bakjyr.2n74ylbxpg81gfqr
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 10.24.10.114:8443 --token bakjyr.2n74ylbxpg81gfqr \
--discovery-token-ca-cert-hash sha256:e5c136324bb642c2d93251f8db5b1ef7dc7fe44aed88ddec224d5bc6b1042b73 \
--experimental-control-plane --certificate-key d155b1519e44de5edbf2769a33858dea4d2af5799b1a3324054dce8b7d3f4aec
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --experimental-upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 10.24.10.114:8443 --token bakjyr.2n74ylbxpg81gfqr \
--discovery-token-ca-cert-hash sha256:e5c136324bb642c2d93251f8db5b1ef7dc7fe44aed88ddec224d5bc6b1042b73
4）拷贝kubectl使用的admin.conf并检查集群状态
[root@d3-master-001 kubernetes]# ansible k8s-masters -m file -a "name=~root/.kube state=directory"
[root@d3-master-001 kubernetes]# ansible k8s-masters -m copy -a "src=/etc/kubernetes/admin.conf dest=~root/.kube/config"
[root@d3-master-001 kubernetes]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
d3-master-001 NotReady master 4m55s v1.14.6
[root@d3-master-001 kubernetes]# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-7d598b68c9-cgjmb 0/1 Pending 0 4m42s <none> <none> <none> <none>
coredns-7d598b68c9-nb9v2 0/1 Pending 0 4m42s <none> <none> <none> <none>
kube-apiserver-d3-master-001 1/1 Running 0 3m38s 10.24.10.74 d3-master-001 <none> <none>
kube-controller-manager-d3-master-001 1/1 Running 0 3m50s 10.24.10.74 d3-master-001 <none> <none>
kube-proxy-895v7 1/1 Running 0 4m42s 10.24.10.74 d3-master-001 <none> <none>
kube-scheduler-d3-master-001 1/1 Running 0 4m1s 10.24.10.74 d3-master-001 <none> <none>
5、部署网络组件flannel
1）flannel的Yaml配置文件
[root@d3-master-001 kubernetes]# cat > kube-flannel-ds-amd64.yaml <<EOF
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: important-kubernetes
value: 1000000000
---
apiVersion: extensions/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp.flannel.unprivileged
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
privileged: false
volumes:
- configMap
- secret
- emptyDir
- hostPath
allowedHostPaths:
- pathPrefix: "/etc/cni/net.d"
- pathPrefix: "/etc/kube-flannel"
- pathPrefix: "/run/flannel"
readOnlyRootFilesystem: false
# Users and groups
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
fsGroup:
rule: RunAsAny
# Privilege Escalation
allowPrivilegeEscalation: false
defaultAllowPrivilegeEscalation: false
# Capabilities
allowedCapabilities: ['NET_ADMIN']
defaultAddCapabilities: []
requiredDropCapabilities: []
# Host namespaces
hostPID: false
hostIPC: false
hostNetwork: true
hostPorts:
- min: 0
max: 65535
# SELinux
seLinux:
# SELinux is unsed in CaaSP
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: flannel
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/14",
"Backend": {
"Type": "vxlan"
}
}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: kube-flannel-ds-amd64
namespace: kube-system
labels:
tier: node
app: flannel
spec:
template:
metadata:
labels:
tier: node
app: flannel
spec:
hostNetwork: true
nodeSelector:
beta.kubernetes.io/arch: amd64
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
priorityClassName: important-kubernetes
initContainers:
- name: install-cni
image: docker.v2.aispeech.com/gcr.io/flannel:v0.11.0-amd64
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: docker.v2.aispeech.com/gcr.io/flannel:v0.11.0-amd64
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "200Mi"
limits:
cpu: "100m"
memory: "200Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
EOF
- 调整了flannel的内存QOS，requests和limits都由50Mi调整为200Mi，减少flannel的Pod内存使用超过limits被重启；
- 调整flannel的Pod优先级，防止极端情况flannel的Pod被驱逐；
- "Network"由"10.244.0.0/16"调整为"10.244.0.0/14"，集群最多支持1024个节点；
- 修改为使用harbor上的容器镜像。
2）应用Yaml配置文件
[root@d3-master-001 kubernetes]# kubectl apply -f kube-flannel-ds-amd64.yaml
priorityclass.scheduling.k8s.io/important-kubernetes created
podsecuritypolicy.extensions/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.extensions/kube-flannel-ds-amd64 created
3）检查集群状态
[root@d3-master-001 kubernetes]# kubectl get pods -n kube-system -l app=flannel -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-flannel-ds-amd64-nqfcc 1/1 Running 0 8s 10.24.10.74 d3-master-001 <none> <none>
[root@d3-master-001 kubernetes]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
d3-master-001 Ready master 5m39s v1.14.6 10.24.10.74 <none> CentOS Linux 7 (Core) 3.10.0-693.el7.x86_64 docker://18.6.3
[root@d3-master-001 kubernetes]# kubectl get pods -n kube-system -l k8s-app=kube-dns
NAME READY STATUS RESTARTS AGE
coredns-7d598b68c9-cgjmb 1/1 Running 0 5m26s
coredns-7d598b68c9-nb9v2 0/1 Running 0 5m26s
6、加入其他master节点
1）加入节点d3-master-002
[root@d3-master-001 kubernetes]# ssh -p $SSH_PORT root@$MASTER02_IP
[root@d3-master-002 ~]# kubeadm join 10.24.10.114:8443 --token bakjyr.2n74ylbxpg81gfqr \
--discovery-token-ca-cert-hash sha256:e5c136324bb642c2d93251f8db5b1ef7dc7fe44aed88ddec224d5bc6b1042b73 \
--experimental-control-plane --certificate-key d155b1519e44de5edbf2769a33858dea4d2af5799b1a3324054dce8b7d3f4aec \
--ignore-preflight-errors=FileAvailable--etc-kubernetes-kubelet.conf
[preflight] Running pre-flight checks
[WARNING FileAvailable--etc-kubernetes-kubelet.conf]: /etc/kubernetes/kubelet.conf already exists
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[preflight] Running pre-flight checks before initializing the new control plane instance
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[download-certs] Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Using the existing "front-proxy-client" certificate and key
[certs] Using the existing "apiserver" certificate and key
[certs] Using the existing "apiserver-kubelet-client" certificate and key
[certs] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[certs] Using the existing "sa" key
[kubeconfig] Generating kubeconfig files
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/admin.conf"
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/scheduler.conf"
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[check-etcd] Skipping etcd check in external mode
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.14" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Activating the kubelet service
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
[control-plane-join] using external etcd - no local stacked instance added
[upload-config] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[mark-control-plane] Marking the node d3-master-002 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node d3-master-002 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
This node has joined the cluster and a new control plane instance was created:
* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
To start administering your cluster from this node, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Run 'kubectl get nodes' to see this node join the cluster.
[root@d3-master-002 ~]# curl localhost:10249/proxyMode
ipvs
[root@d3-master-002 ~]# exit
logout
Connection to 10.24.10.75 closed.
2）加入节点d3-master-003
[root@d3-master-001 kubernetes]# ssh -p $SSH_PORT root@$MASTER03_IP
[root@d3-master-003 ~]# kubeadm join 10.24.10.114:8443 --token bakjyr.2n74ylbxpg81gfqr \
--discovery-token-ca-cert-hash sha256:e5c136324bb642c2d93251f8db5b1ef7dc7fe44aed88ddec224d5bc6b1042b73 \
--experimental-control-plane --certificate-key d155b1519e44de5edbf2769a33858dea4d2af5799b1a3324054dce8b7d3f4aec \
--ignore-preflight-errors=FileAvailable--etc-kubernetes-kubelet.conf
[root@d3-master-003 ~]# exit
logout
Connection to 10.24.10.76 closed.
3）检查集群状态
[root@d3-master-001 kubernetes]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
d3-master-001 Ready master 8m42s v1.14.6 10.24.10.74 <none> CentOS Linux 7 (Core) 3.10.0-693.el7.x86_64 docker://18.6.3
d3-master-002 Ready master 92s v1.14.6 10.24.10.75 <none> CentOS Linux 7 (Core) 3.10.0-693.el7.x86_64 docker://18.6.3
d3-master-003 Ready master 14s v1.14.6 10.24.10.76 <none> CentOS Linux 7 (Core) 3.10.0-693.el7.x86_64 docker://18.6.3
[root@d3-master-001 kubernetes]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-7d598b68c9-cgjmb 1/1 Running 0 8m27s
coredns-7d598b68c9-nb9v2 1/1 Running 0 8m27s
kube-apiserver-d3-master-001 1/1 Running 0 7m23s
kube-apiserver-d3-master-002 1/1 Running 0 96s
kube-apiserver-d3-master-003 1/1 Running 0 19s
kube-controller-manager-d3-master-001 1/1 Running 0 7m35s
kube-controller-manager-d3-master-002 1/1 Running 0 96s
kube-controller-manager-d3-master-003 1/1 Running 0 19s
kube-flannel-ds-amd64-9jt4p 1/1 Running 0 19s
kube-flannel-ds-amd64-jb6dx 1/1 Running 0 97s
kube-flannel-ds-amd64-nqfcc 1/1 Running 0 3m23s
kube-proxy-895v7 1/1 Running 0 8m27s
kube-proxy-b82v5 1/1 Running 0 97s
kube-proxy-h9zls 1/1 Running 0 19s
kube-scheduler-d3-master-001 1/1 Running 0 7m46s
kube-scheduler-d3-master-002 1/1 Running 0 97s
kube-scheduler-d3-master-003 1/1 Running 0 19s
4）调整coredns副本数量为3个
[root@d3-master-001 kubernetes]# kubectl scale deployment -n kube-system coredns --replicas=3
deployment.extensions/coredns scaled