服务器信息
2台Centos7.5.1804 关闭防火墙 关闭selinux
- node03 172.16.50.134
- node04 172.16.50.135
安装docker
node03、node04 均安装
Version: 1.13.1
yum install docker -y
创建配置文件/etc/docker/daemon.json
{
"registry-mirrors": ["https://8ph3bzw4.mirror.aliyuncs.com"],
"graph": "/data/docker"
}
创建Docker工作目录
mkdir /data/docker -p
启动docker
systemctl enable docker && systemctl start docker
参数调整
创建`/etc/sysctl.d/k8s.conf `文件添加如下:
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
生效
sysctl -p /etc/sysctl.d/k8s.conf
安装kubelet kubeadm kubectl
Version: v1.12.3
添加yum源/etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
yum install kubelet kubeadm kubectl -y #如果版本更新; 安装版本可能为最新版本
默认kubeadm初始化集群时生成证书默认一年有效期;如需更改参考 http://blog.51cto.com/11889458/2323328
导入必要docker镜像
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-proxy:v1.12.3 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-apiserver:v1.12.3 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-controller-manager:v1.12.3 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-scheduler:v1.12.3 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/etcd:3.2.24 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/coredns:1.2.2 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/flannel:v0.10.0-amd64 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/pause:3.1 ;\
docker pull registry.cn-hangzhou.aliyuncs.com/kuberimages/kubernetes-dashboard-amd64:v1.10.0
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-proxy:v1.12.3 k8s.gcr.io/kube-proxy:v1.12.3 ;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-apiserver:v1.12.3 k8s.gcr.io/kube-apiserver:v1.12.3;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-controller-manager:v1.12.3 k8s.gcr.io/kube-controller-manager:v1.12.3;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/kube-scheduler:v1.12.3 k8s.gcr.io/kube-scheduler:v1.12.3;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/etcd:3.2.24 k8s.gcr.io/etcd:3.2.24 ;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/coredns:1.2.2 k8s.gcr.io/coredns:1.2.2 ;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/flannel:v0.10.0-amd64 quay.io/coreos/flannel:v0.10.0-amd64 ;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/pause:3.1 k8s.gcr.io/pause:3.1 ;\
docker tag registry.cn-hangzhou.aliyuncs.com/kuberimages/kubernetes-dashboard-amd64:v1.10.0 k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.0
初始化集群
node03节点操作
kubeadm init --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=172.16.50.134 --kubernetes-version=v1.12.3
[root@node03 bin]# kubeadm init --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=172.16.50.134 --kubernetes-version=v1.12.3
[init] using Kubernetes version: v1.12.3
[preflight] running pre-flight checks
[WARNING Service-Docker]: docker service is not enabled, please run 'systemctl enable docker.service'
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[preflight/images] Pulling images required for setting up a Kubernetes cluster
[preflight/images] This might take a minute or two, depending on the speed of your internet connection
[preflight/images] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[preflight] Activating the kubelet service
[certificates] Generated etcd/ca certificate and key.
[certificates] Generated apiserver-etcd-client certificate and key.
[certificates] Generated etcd/peer certificate and key.
[certificates] etcd/peer serving cert is signed for DNS names [node03 localhost] and IPs [172.16.50.134 127.0.0.1 ::1]
[certificates] Generated etcd/healthcheck-client certificate and key.
[certificates] Generated etcd/server certificate and key.
[certificates] etcd/server serving cert is signed for DNS names [node03 localhost] and IPs [127.0.0.1 ::1]
[certificates] Generated ca certificate and key.
[certificates] Generated apiserver certificate and key.
[certificates] apiserver serving cert is signed for DNS names [node03 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 172.16.50.134]
[certificates] Generated apiserver-kubelet-client certificate and key.
[certificates] Generated front-proxy-ca certificate and key.
[certificates] Generated front-proxy-client certificate and key.
[certificates] valid certificates and keys now exist in "/etc/kubernetes/pki"
[certificates] Generated sa key and public key.
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/scheduler.conf"
[controlplane] wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[etcd] Wrote Static Pod manifest for a local etcd instance to "/etc/kubernetes/manifests/etcd.yaml"
[init] waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests"
[init] this might take a minute or longer if the control plane images have to be pulled
[apiclient] All control plane components are healthy after 29.504802 seconds
[uploadconfig] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.12" in namespace kube-system with the configuration for the kubelets in the cluster
[markmaster] Marking the node node03 as master by adding the label "node-role.kubernetes.io/master=''"
[markmaster] Marking the node node03 as master by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "node03" as an annotation
[bootstraptoken] using token: xlev4l.jdlpbrkpwxv6yt63
[bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes master has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of machines by running the following on each node
as root:
kubeadm join 172.16.50.134:6443 --token xlev4l.jdlpbrkpwxv6yt63 --discovery-token-ca-cert-hash sha256:236b7a143abc822cdffcc90d6174aeeefd4c557a9ac2d97624f461cc9b68f305
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
添加节点
node04节点操作
kubeadm join 172.16.50.134:6443 --token xlev4l.jdlpbrkpwxv6yt63 --discovery-token-ca-cert-hash sha256:236b7a143abc822cdffcc90d6174aeeefd4c557a9ac2d97624f461cc9b68f305
[root@node04 docker]# kubeadm join 172.16.50.134:6443 --token xlev4l.jdlpbrkpwxv6yt63 --discovery-token-ca-cert-hash sha256:236b7a143abc822cdffcc90d6174aeeefd4c557a9ac2d97624f461cc9b68f305
[preflight] running pre-flight checks
[WARNING RequiredIPVSKernelModulesAvailable]: the IPVS proxier will not be used, because the following required kernel modules are not loaded: [ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh] or no builtin kernel ipvs support: map[ip_vs_sh:{} nf_conntrack_ipv4:{} ip_vs:{} ip_vs_rr:{} ip_vs_wrr:{}]
you can solve this problem with following methods:
1. Run 'modprobe -- ' to load missing kernel modules;
2. Provide the missing builtin kernel ipvs support
[WARNING Service-Docker]: docker service is not enabled, please run 'systemctl enable docker.service'
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[discovery] Trying to connect to API Server "172.16.50.134:6443"
[discovery] Created cluster-info discovery client, requesting info from "https://172.16.50.134:6443"
[discovery] Requesting info from "https://172.16.50.134:6443" again to validate TLS against the pinned public key
[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "172.16.50.134:6443"
[discovery] Successfully established connection with API Server "172.16.50.134:6443"
[kubelet] Downloading configuration for the kubelet from the "kubelet-config-1.12" ConfigMap in the kube-system namespace
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[preflight] Activating the kubelet service
[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "node04" as an annotation
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the master to see this node join the cluster.
查看集群节点
[root@node03 bin]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
node03 NotReady master 78s v1.12.3
node04 NotReady <none> 55s v1.12.3
安装网络插件
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
[root@node03 bin]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.extensions/kube-flannel-ds-amd64 created
daemonset.extensions/kube-flannel-ds-arm64 created
daemonset.extensions/kube-flannel-ds-arm created
daemonset.extensions/kube-flannel-ds-ppc64le created
daemonset.extensions/kube-flannel-ds-s390x created
查看pods状态
[root@node03 bin]# kubectl get pods --all-namespaces -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kube-system coredns-576cbf47c7-dwjzk 1/1 Running 0 2m13s 10.244.1.3 node04 <none>
kube-system coredns-576cbf47c7-zq8rx 1/1 Running 1 2m13s 10.244.1.2 node04 <none>
kube-system etcd-node03 1/1 Running 0 103s 172.16.50.134 node03 <none>
kube-system kube-apiserver-node03 1/1 Running 0 106s 172.16.50.134 node03 <none>
kube-system kube-controller-manager-node03 1/1 Running 0 104s 172.16.50.134 node03 <none>
kube-system kube-flannel-ds-amd64-s6k7t 1/1 Running 0 31s 172.16.50.135 node04 <none>
kube-system kube-flannel-ds-amd64-w8kr2 1/1 Running 0 31s 172.16.50.134 node03 <none>
kube-system kube-proxy-c68kv 1/1 Running 0 2m9s 172.16.50.135 node04 <none>
kube-system kube-proxy-jxrfg 1/1 Running 0 2m13s 172.16.50.134 node03 <none>
kube-system kube-scheduler-node03 1/1 Running 0 106s 172.16.50.134 node03 <none>
查看集群节点状态
[root@node03 bin]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
node03 Ready master 3m28s v1.12.3
node04 Ready <none> 3m5s v1.12.3
部署仪表盘
kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml
[root@node03 bin]# kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml
secret/kubernetes-dashboard-certs created
serviceaccount/kubernetes-dashboard created
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
deployment.apps/kubernetes-dashboard created
service/kubernetes-dashboard created
查看运行状态
[root@node03 bin]# kubectl get pods --all-namespaces -o wide | grep dashboard
kube-system kubernetes-dashboard-77fd78f978-bkm9r 1/1 Running 0 37m 10.244.1.4 node04 <none>
添加权限
创建kubernetes-dashboard-rbc.yaml文件,内容如下:
# ------------------- ServiceAccount ------------------- #
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-admin
namespace: kube-system
---
# ------------------- ClusterRoleBinding ------------------- #
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
labels:
k8s-app: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard-admin
namespace: kube-system
kubectl create -f kubernetes-dashboard-rbc.yaml
查看token
[root@node03 bin]# kubectl get secrets -n kube-system | grep kubernetes-dashboard-admin
kubernetes-dashboard-admin-token-77hlj kubernetes.io/service-account-token 3 2m56s
[root@node03 bin]# kubectl describe secrets/kubernetes-dashboard-admin-token-77hlj -n kube-system
Name: kubernetes-dashboard-admin-token-77hlj
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: kubernetes-dashboard-admin
kubernetes.io/service-account.uid: f10a8196-f46a-11e8-8466-1e00f5000055
Type: kubernetes.io/service-account-token
Data
====
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbi10b2tlbi03N2hsaiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImYxMGE4MTk2LWY0NmEtMTFlOC04NDY2LTFlMDBmNTAwMDA1NSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTprdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbiJ9.sks04AJ1D0qx9YwhZevawx9iXp66bBZoAm2bqp6ai-qG3piz4NWLYisWZ40QgrMQ6EGf4j5ax7HKKxQKjxa0eqZ5pgLgoyYOC5jrl6y1jPUSj4ThsG5SnKhH1pr3Jzh7iUY-Ak_ckt1rzVn0AWs7PTL9fRqMJLrN4eFbcZgf5640fcwa4Jlu4pnDatIC7tRm9an_87wGVrn41HrxW4E5ZgKP9evEAhJybaZMTRVebh6jjBDf1XsPe6OMK0KwruNYVcuGY7eluJ0xKN-gwNXKP5MdV6Bii6ikAL_ZJ3PnlEjPl-5KHChMxj129RjlgVKu-IwC633hOnCrBlGfRLJVNQ
导出浏览器证书
cat /etc/kubernetes/admin.conf | grep client-certificate-data | awk -F ': ' '{print $2}' | base64 -d > /etc/kubernetes/pki/client.crt
cat /etc/kubernetes/admin.conf | grep client-key-data | awk -F ': ' '{print $2}' | base64 -d > /etc/kubernetes/pki/client.key
openssl pkcs12 -export -inkey /etc/kubernetes/pki/client.key -in /etc/kubernetes/pki/client.crt -out /etc/kubernetes/pki/client.pfx
下载 /etc/kubernetes/pki/client.pfx 文件导入浏览器 (建议使用火狐浏览器)
访问地址https://172.16.50.134:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login
选择使用令牌认证:输入刚才查询到的token