部署zookeeper
節點 128 129 130
- 部署前提安裝jdk
傳送門: zookeeper3.4.14
[[email protected] local]# java -version
java version "1.8.0_221"
Java(TM) SE Runtime Environment (build 1.8.0_221-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.221-b11, mixed mode)
下載下傳并配置
[[email protected] ~]# tar -xf zookeeper-3.4.14.tar.gz -C /usr/local/
[[email protected] ~]# cd /usr/local/
[[email protected] local]# ln -s /usr/local/zookeeper-3.4.14/ /usr/local/zookeeper
[[email protected] conf]# pwd
/usr/local/zookeeper/conf
[[email protected] conf]# cp zoo_sample.cfg zoo.cfg
配置參數
[[email protected] conf]# vi zoo.cfg
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
# the port at which the clients will connect
clientPort=2181
server.1=zk1.od.com:2888:3888
server.2=zk2.od.com:2888:3888
server.3=zk3.od.com:2888:3888
[[email protected] conf]# mkdir -p /data/zookeeper/data
[[email protected] conf]# mkdir -p /data/zookeeper/logs
添加DNS od.com.解析讓以上域名解析成功
配置叢集myid
[[email protected] conf]# vi /data/zookeeper/data/myid
1
[[email protected] conf]# vi /data/zookeeper/data/myid
2
[[email protected] conf]# vi /data/zookeeper/data/myid
3
[[email protected] bin]# /usr/local/zookeeper/bin/zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[[email protected] bin]# netstat -tnlp | grep 2181
tcp 0 0 0.0.0.0:2181 0.0.0.0:* LISTEN 55304/java
130節點為master
[[email protected] bin]# ./zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: leader
部署jenkins至k8s叢集
鏡像傳送門：dockerhub
節點 132
[[email protected] ~]# docker pull jenkins/jenkins:v2.222.4
2.263.4: Pulling from jenkins/jenkins
9a0b0ce99936: Pull complete
db3b6004c61a: Pull complete
4e96cf3bdc20: Pull complete
e47bd954be8f: Pull complete
b2d9d6b1cd91: Pull complete
fa537a81cda1: Pull complete
Digest: sha256:64576b8bd0a7f5c8ca275f4926224c29e7aa3f3167923644ec1243cd23d611f3
Status: Downloaded newer image for jenkins/jenkins:v2.222.4
docker.io/jenkins/jenkins:v2.222.4
[[email protected] ~]# docker tag 22b8b9a84dbe harbor.od.com/public/jenkins:v2.222.4
[[email protected] ~]# docker push harbor.od.com/public/jenkins:v2.222.4
The push refers to repository [harbor.od.com/public/jenkins]
e0485b038afa: Pushed
2950fdd45d03: Pushed
6ce697717948: Pushed
911119b5424d: Pushed
b8f8aeff56a8: Pushed
97041f29baff: Pushed
v2.190.3: digest: sha256:64576b8bd0a7f5c8ca275f4926224c29e7aa3f3167923644ec1243cd23d611f3 size: 4087
配置dockerfile
[[email protected] ~]# ssh-keygen -t rsa -b 2048 -C "[email protected]" -N "" -f /root/.ssh/id_rsa
Generating public/private rsa key pair.
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:jo0UhlYUk+xszsNIpHt64iUvasvyWSzTaZAE7Xhcfd4 [email protected]
The key's randomart image is:
+---[RSA 2048]----+
|.. +=o |
| .. oo+.. |
|o.+X. |
|+*=+. |
+----[SHA256]-----+
[[email protected] ~]# mkdir -p /data/dockerfile/jenkins/
[[email protected] jenkins]# vi Dockerfile
擷取jenkins鏡像
FROM harbor.od.com/public/jenkins:v2.222.4
使用root使用者執行
USER root
拷貝時區到容器
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
echo 'Asia/Shanghai' >/etc/timezone
添加密鑰檔案到容器
ADD id_rsa /root/.ssh/id_rsa
加入登陸私有倉庫檔案
ADD config.json /root/.docker/config.json
安裝docker用戶端
ADD get-docker.sh /get-docker.sh
修改ssh用戶端認證
RUN echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&\
/get-docker.sh
拷貝密鑰到目前目錄
[[email protected] jenkins]# cp ~/.ssh/id_rsa .
拷貝docker配置到目前目錄
[[email protected] jenkins]# cp ~/.docker/config.json .
[[email protected] jenkins]# wget https://get.docker.com/
[[email protected] jenkins]# chmod +x get-docker.sh
[[email protected] jenkins]# vi get-docker.sh
#!/bin/sh
set -e
# This script is meant for quick & easy install via:
# $ curl -fsSL get.docker.com -o get-docker.sh
# $ sh get-docker.sh
#
# For test builds (ie. release candidates):
# $ curl -fsSL test.docker.com -o test-docker.sh
# $ sh test-docker.sh
#
# NOTE: Make sure to verify the contents of the script
# you downloaded matches the contents of install.sh
# located at https://github.com/docker/docker-install
# before executing.
#
# Git commit from https://github.com/docker/docker-install when
# the script was uploaded (Should only be modified by upload job):
SCRIPT_COMMIT_SHA=e749601
# This value will automatically get changed for:
# * edge
# * test
# * experimental
DEFAULT_CHANNEL_VALUE="edge"
if [ -z "$CHANNEL" ]; then
CHANNEL=$DEFAULT_CHANNEL_VALUE
fi
DEFAULT_DOWNLOAD_URL="https://download.docker.com"
if [ -z "$DOWNLOAD_URL" ]; then
DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL
fi
SUPPORT_MAP="
x86_64-centos-7
x86_64-fedora-26
x86_64-fedora-27
x86_64-debian-wheezy
x86_64-debian-jessie
x86_64-debian-stretch
x86_64-debian-buster
x86_64-ubuntu-trusty
x86_64-ubuntu-xenial
x86_64-ubuntu-artful
s390x-ubuntu-xenial
s390x-ubuntu-artful
ppc64le-ubuntu-xenial
ppc64le-ubuntu-artful
aarch64-ubuntu-xenial
aarch64-debian-jessie
aarch64-debian-stretch
aarch64-fedora-26
aarch64-fedora-27
aarch64-centos-7
armv6l-raspbian-jessie
armv7l-raspbian-jessie
armv6l-raspbian-stretch
armv7l-raspbian-stretch
armv7l-debian-jessie
armv7l-debian-stretch
armv7l-debian-buster
armv7l-ubuntu-trusty
armv7l-ubuntu-xenial
armv7l-ubuntu-artful
"
mirror=''
DRY_RUN=${DRY_RUN:-}
while [ $# -gt 0 ]; do
case "$1" in
--mirror)
mirror="$2"
shift
;;
--dry-run)
DRY_RUN=1
;;
--*)
echo "Illegal option $1"
;;
esac
shift $(( $# > 0 ? 1 : 0 ))
done
case "$mirror" in
Aliyun)
DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce"
;;
AzureChinaCloud)
DOWNLOAD_URL="https://mirror.azure.cn/docker-ce"
;;
esac
command_exists() {
command -v "$@" > /dev/null 2>&1
}
is_dry_run() {
if [ -z "$DRY_RUN" ]; then
return 1
else
return 0
fi
}
deprecation_notice() {
distro=$1
date=$2
echo
echo "DEPRECATION WARNING:"
echo " The distribution, $distro, will no longer be supported in this script as of $date."
echo " If you feel this is a mistake please submit an issue at https://github.com/docker/docker-install/issues/new"
echo
sleep 10
}
get_distribution() {
lsb_dist=""
# Every system that we officially support has /etc/os-release
if [ -r /etc/os-release ]; then
lsb_dist="$(. /etc/os-release && echo "$ID")"
fi
# Returning an empty string here should be alright since the
# case statements don't act unless you provide an actual value
echo "$lsb_dist"
}
add_debian_backport_repo() {
debian_version="$1"
backports="deb http://ftp.debian.org/debian $debian_version-backports main"
if ! grep -Fxq "$backports" /etc/apt/sources.list; then
(set -x; $sh_c "echo \"$backports\" >> /etc/apt/sources.list")
fi
}
echo_docker_as_nonroot() {
if is_dry_run; then
return
fi
if command_exists docker && [ -e /var/run/docker.sock ]; then
(
set -x
$sh_c 'docker version'
) || true
fi
your_user=your-user
[ "$user" != 'root' ] && your_user="$user"
# intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output
echo "If you would like to use Docker as a non-root user, you should now consider"
echo "adding your user to the \"docker\" group with something like:"
echo
echo " sudo usermod -aG docker $your_user"
echo
echo "Remember that you will have to log out and back in for this to take effect!"
echo
echo "WARNING: Adding a user to the \"docker\" group will grant the ability to run"
echo " containers which can be used to obtain root privileges on the"
echo " docker host."
echo " Refer to https://docs.docker.com/engine/security/security/#docker-daemon-attack-surface"
echo " for more information."
}
# Check if this is a forked Linux distro
check_forked() {
# Check for lsb_release command existence, it usually exists in forked distros
if command_exists lsb_release; then
# Check if the `-u` option is supported
set +e
lsb_release -a -u > /dev/null 2>&1
lsb_release_exit_code=$?
set -e
# Check if the command has exited successfully, it means we're in a forked distro
if [ "$lsb_release_exit_code" = "0" ]; then
# Print info about current distro
cat <<-EOF
You're using '$lsb_dist' version '$dist_version'.
EOF
# Get the upstream release info
lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]')
# Print info about upstream distro
cat <<-EOF
Upstream release is '$lsb_dist' version '$dist_version'.
EOF
else
if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then
if [ "$lsb_dist" = "osmc" ]; then
# OSMC runs Raspbian
lsb_dist=raspbian
else
# We're Debian and don't even know it!
lsb_dist=debian
fi
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
case "$dist_version" in
9)
dist_version="stretch"
;;
8|'Kali Linux 2')
dist_version="jessie"
;;
7)
dist_version="wheezy"
;;
esac
fi
fi
fi
}
semverParse() {
major="${1%%.*}"
minor="${1#$major.}"
minor="${minor%%.*}"
patch="${1#$major.$minor.}"
patch="${patch%%[-.]*}"
}
ee_notice() {
echo
echo
echo " WARNING: $1 is now only supported by Docker EE"
echo " Check https://store.docker.com for information on Docker EE"
echo
echo
}
do_install() {
echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA"
if command_exists docker; then
docker_version="$(docker -v | cut -d ' ' -f3 | cut -d ',' -f1)"
MAJOR_W=1
MINOR_W=10
semverParse "$docker_version"
shouldWarn=0
if [ "$major" -lt "$MAJOR_W" ]; then
shouldWarn=1
fi
if [ "$major" -le "$MAJOR_W" ] && [ "$minor" -lt "$MINOR_W" ]; then
shouldWarn=1
fi
cat >&2 <<-'EOF'
Warning: the "docker" command appears to already exist on this system.
If you already have Docker installed, this script can cause trouble, which is
why we're displaying this warning and provide the opportunity to cancel the
installation.
If you installed the current Docker package using this script and are using it
EOF
if [ $shouldWarn -eq 1 ]; then
cat >&2 <<-'EOF'
again to update Docker, we urge you to migrate your image store before upgrading
to v1.10+.
You can find instructions for this here:
https://github.com/docker/docker/wiki/Engine-v1.10.0-content-addressability-migration
EOF
else
cat >&2 <<-'EOF'
again to update Docker, you can safely ignore this message.
EOF
fi
cat >&2 <<-'EOF'
You may press Ctrl+C now to abort this script.
EOF
( set -x; sleep 20 )
fi
user="$(id -un 2>/dev/null || true)"
sh_c='sh -c'
if [ "$user" != 'root' ]; then
if command_exists sudo; then
sh_c='sudo -E sh -c'
elif command_exists su; then
sh_c='su -c'
else
cat >&2 <<-'EOF'
Error: this installer needs the ability to run commands as root.
We are unable to find either "sudo" or "su" available to make this happen.
EOF
exit 1
fi
fi
if is_dry_run; then
sh_c="echo"
fi
# perform some very rudimentary platform detection
lsb_dist=$( get_distribution )
lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
case "$lsb_dist" in
ubuntu)
if command_exists lsb_release; then
dist_version="$(lsb_release --codename | cut -f2)"
fi
if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then
dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")"
fi
;;
debian|raspbian)
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
case "$dist_version" in
9)
dist_version="stretch"
;;
8)
dist_version="jessie"
;;
7)
dist_version="wheezy"
;;
esac
;;
centos)
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
;;
rhel|ol|sles)
ee_notice "$lsb_dist"
exit 1
;;
*)
if command_exists lsb_release; then
dist_version="$(lsb_release --release | cut -f2)"
fi
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
;;
esac
# Check if this is a forked Linux distro
check_forked
# Check if we actually support this configuration
if ! echo "$SUPPORT_MAP" | grep "$(uname -m)-$lsb_dist-$dist_version" >/dev/null; then
cat >&2 <<-'EOF'
Either your platform is not easily detectable or is not supported by this
installer script.
Please visit the following URL for more detailed installation instructions:
https://docs.docker.com/engine/installation/
EOF
exit 1
fi
# Run setup for each distro accordingly
case "$lsb_dist" in
ubuntu|debian|raspbian)
pre_reqs="apt-transport-https ca-certificates curl"
if [ "$lsb_dist" = "debian" ]; then
if [ "$dist_version" = "wheezy" ]; then
add_debian_backport_repo "$dist_version"
fi
# libseccomp2 does not exist for debian jessie main repos for aarch64
if [ "$(uname -m)" = "aarch64" ] && [ "$dist_version" = "jessie" ]; then
add_debian_backport_repo "$dist_version"
fi
fi
# TODO: August 31, 2018 delete from here,
if [ "$lsb_dist" = "ubuntu" ] && [ "$dist_version" = "artful" ]; then
deprecation_notice "$lsb_dist $dist_version" "August 31, 2018"
fi
# TODO: August 31, 2018 delete to here,
if ! command -v gpg > /dev/null; then
pre_reqs="$pre_reqs gnupg"
fi
apt_repo="deb [arch=$(dpkg --print-architecture)] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL"
(
if ! is_dry_run; then
set -x
fi
$sh_c 'apt-get update -qq >/dev/null'
$sh_c "apt-get install -y -qq $pre_reqs >/dev/null"
$sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | apt-key add -qq - >/dev/null"
$sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list"
if [ "$lsb_dist" = "debian" ] && [ "$dist_version" = "wheezy" ]; then
$sh_c 'sed -i "/deb-src.*download\.docker/d" /etc/apt/sources.list.d/docker.list'
fi
$sh_c 'apt-get update -qq >/dev/null'
)
pkg_version=""
if [ ! -z "$VERSION" ]; then
if is_dry_run; then
echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
else
# Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel
pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist"
search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | cut -d' ' -f 4"
pkg_version="$($sh_c "$search_command")"
echo "INFO: Searching repository for VERSION '$VERSION'"
echo "INFO: $search_command"
if [ -z "$pkg_version" ]; then
echo
echo "ERROR: '$VERSION' not found amongst apt-cache madison results"
echo
exit 1
fi
pkg_version="=$pkg_version"
fi
fi
(
if ! is_dry_run; then
set -x
fi
$sh_c "apt-get install -y -qq --no-install-recommends docker-ce$pkg_version >/dev/null"
)
echo_docker_as_nonroot
exit 0
;;
centos|fedora)
yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/docker-ce.repo"
if [ "$lsb_dist" = "fedora" ]; then
if [ "$dist_version" -lt "26" ]; then
echo "Error: Only Fedora >=26 are supported"
exit 1
fi
pkg_manager="dnf"
config_manager="dnf config-manager"
enable_channel_flag="--set-enabled"
pre_reqs="dnf-plugins-core"
pkg_suffix="fc$dist_version"
else
pkg_manager="yum"
config_manager="yum-config-manager"
enable_channel_flag="--enable"
pre_reqs="yum-utils"
pkg_suffix="el"
fi
(
if ! is_dry_run; then
set -x
fi
$sh_c "$pkg_manager install -y -q $pre_reqs"
$sh_c "$config_manager --add-repo $yum_repo"
if [ "$CHANNEL" != "stable" ]; then
$sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL"
fi
$sh_c "$pkg_manager makecache"
)
pkg_version=""
if [ ! -z "$VERSION" ]; then
if is_dry_run; then
echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
else
pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix"
search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
pkg_version="$($sh_c "$search_command")"
echo "INFO: Searching repository for VERSION '$VERSION'"
echo "INFO: $search_command"
if [ -z "$pkg_version" ]; then
echo
echo "ERROR: '$VERSION' not found amongst $pkg_manager list results"
echo
exit 1
fi
# Cut out the epoch and prefix with a '-'
pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)"
fi
fi
(
if ! is_dry_run; then
set -x
fi
$sh_c "$pkg_manager install -y -q docker-ce$pkg_version"
)
echo_docker_as_nonroot
exit 0
;;
esac
exit 1
}
# wrapped up in a function so that we have some protection against only getting
# half the file during "curl | sh"
do_install
建構dockerfile
- harbor建立infra私有項目
[[email protected] jenkins]# docker build . -t harbor.od.com/infra/jenkins:v2.222.4
[[email protected] jenkins]# docker push harbor.od.com/infra/jenkins:v2.222.4
建立名稱空間
節點 130
[[email protected] bin]# kubectl create ns infra
namespace/infra created
授權叢集拉取harbor私有項目
[[email protected] bin]# kubectl create secret docker-registry harbor --docker-server=harbor.od.com --docker-username=admin --docker-password=12345 -n infra
secret/harbor created
類型：docker-registry
名稱： harbor
私倉庫名稱： infra
下載下傳nfs
節點 130 131 132
配置nfs服務端
節點 132
[[email protected] ~]# vi /etc/exports
/data/nfsvolume 192.168.108.0/24(rw,no_root_squash)
[[email protected] ~]# mkdir /data/nfsvolume/jenkins_home
[[email protected] ~]# systemctl start nfs
[[email protected] ~]# systemctl enable nfs
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
建立jenkins資源清單
[[email protected] k8s-yaml]# mkdir jenkins
[[email protected] k8s-yaml]# cd jenkins/
[[email protected] jenkins]# cat dp.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
name: jenkins
namespace: infra
labels:
name: jenkins
spec:
replicas: 1
selector:
matchLabels:
name: jenkins
template:
metadata:
labels:
app: jenkins
name: jenkins
spec:
volumes:
- name: data
nfs:
server: ceshi-132.host.com
path: /data/nfsvolume/jenkins_home
- name: docker
hostPath:
path: /run/docker.sock
type: ''
containers:
- name: jenkins
image: harbor.od.com/infra/jenkins:v2.222.4
imagePullPolicy: IfNotPresent 如果本地沒有，就去遠端倉庫拉鏡像
ports:
- containerPort: 8080 容器端口
protocol: TCP 協定
env: 環境變量
- name: JAVA_OPTS
value: -Xmx512m -Xms512m 最小最大堆記憶體512m
volumeMounts: 挂載路徑
- name: data
mountPath: /var/jenkins_home
- name: docker
mountPath: /run/docker.sock 将主控端docker sock檔案挂載到jenkins，也就是說在jenkins容器裡就像在主控端使用docker，檢視docker鏡像 運作容器都是和主控端一樣的效果
imagePullSecrets: 拉取私有倉庫鏡像必須加次參數否則拉不到
- name: harbor 聲明名稱
securityContext:
runAsUser: 0 root啟動
strategy:
type: RollingUpdate 預設滾動更新
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
revisionHistoryLimit: 7
progressDeadlineSeconds: 600 啟動失敗逾時時長
[[email protected] jenkins]# cat svc.yaml
kind: Service
apiVersion: v1
metadata:
name: jenkins
namespace: infra
spec:
ports:
- protocol: TCP
port: 80 叢集網絡端口
targetPort: 8080 容器端口
selector:
app: jenkins
[[email protected] jenkins]# cat ingress.yaml
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: jenkins
namespace: infra
spec:
rules:
- host: jenkins.od.com
http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 80
建構pods
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/jenkins/dp.yaml
deployment.extensions/jenkins created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/jenkins/svc.yaml
service/jenkins created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/jenkins/ingress.yaml
ingress.extensions/jenkins created
驗證pod
節點 130 131
- 測試git前提必須在鏡像打包之前将公鑰上傳至gitee安全設定的SSH公鑰（使用SSH公鑰可以讓你在你的電腦和 Gitee 通訊的時候使用安全連接配接）
[[email protected] ~]# kubectl get pod -n infra
NAME READY STATUS RESTARTS AGE
jenkins-698b4994c8-hm5wf 1/1 Running 0 5h21m
[[email protected] ~]# kubectl exec -it jenkins-698b4994c8-hm5wf bash -n infra
[email protected]-698b4994c8-hm5wf:/# whoami
root
[email protected]-698b4994c8-hm5wf:/# date
Wed Aug 18 16:33:53 CST 2021
測試以ssh連通性
[email protected]-698b4994c8-hm5wf:/# ssh -i /root/.ssh/id_rsa -T [email protected]
Hi 劉江旭! You've successfully authenticated, but GITEE.COM does not provide shell access.
測試harbor倉庫連通性
[email protected]-698b4994c8-hm5wf:/# docker login harbor.od.com
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Login Succeeded
部署maven
節點 132
傳送門：maven:3.6.2
配置maven
[[email protected] ~]# mkdir /data/nfsvolume/jenkins_home/maven-3.6.2-8u242
[[email protected] ~]# tar xf apache-maven-3.6.2-bin.tar.gz
[[email protected] ~]# mv apache-maven-3.6.2/* /data/nfsvolume/jenkins_home/maven-3.6.2-8u242/
[[email protected] ~]# vi /data/nfsvolume/jenkins_home/maven-3.6.2-8u242/conf/settings.xml
<mirror>
<id>nexus-aliyun</id>
<mirrorOf>*</mirrorOf>
<name>Nexus aliyun</name>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
</mirror>
下載下傳java運作所需鏡像
[[email protected] ~]# docker pull stanleyws/jre8:8u112
[[email protected] ~]# docker tag fa3a085d6ef1 harbor.od.com/public/jre8:8u112
[[email protected] ~]# docker push harbor.od.com/public/jre8:8u112
The push refers to repository [harbor.od.com/public/jre8]
[[email protected] ~]# mkdir /data/dockerfile/jre8
[[email protected] jre8]# vi Dockerfile
來自私有倉庫
FROM harbor.od.com/public/jre8:8u112
設定時區
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
echo 'Asia/Shanghai' >/etc/timezone
添加監控配置檔案
ADD config.yml /opt/prom/config.yml
收集jvm資訊
ADD jmx_javaagent-0.3.1.jar /opt/prom/
工作目錄
WORKDIR /opt/project_dir
docker運作預設啟動腳本
ADD entrypoint.sh /entrypoint.sh
CMD ["/entrypoint.sh"]
[[email protected] jre8]# wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar -O jmx_javaagent-0.3.1.jar
[[email protected] jre8]# vi config.yml
---
rules:
- pattern: '.*'
[[email protected] jre8]# vi entrypoint.sh
#!/bin/sh
M_OPTS="-Duser.timezone=Asia/Shanghai -javaagent:/opt/prom/jmx_javaagent-0.3.1.jar=$(hostname -i):${M_PORT:-"12346"}:/opt/prom/config.yml"
C_OPTS=${C_OPTS}
JAR_BALL=${JAR_BALL}
exec java -jar ${M_OPTS} ${C_OPTS} ${JAR_BALL}
建構dockerfile
[[email protected] jre8]# docker build . -t harbor.od.com/base/jre8:8u112
Successfully built 7f36e75aac28
Successfully tagged harbor.od.com/base/jre8:8u112
[[email protected] jre8]# docker push harbor.od.com/base/jre8:8u112
8d4d1ab5ff74: Mounted from public/jre8
8u112: digest: sha256:72d4bd870605ae17f9f23e5cb9c453c34906d7ff86ce97c0c2ef89b68c1dcb6f size: 2405
下載下傳Jenkins插件
Blue Ocean
Jenkins建立流水項目
New Item》pipeline》Configure》Discard old builds
New Item》pipeline》Configure》This project is parameterized
-
Add Parameter -> String Parameter
Name : app_name
Default Value :
Description : 項目名稱
-
Add Parameter -> String Parameter
Name : image_name
Default Value :
Description : 鏡像名稱
-
Add Parameter -> String Parameter
Name : git_repo
Default Value :
Description : 項目所在git中央倉庫位址
-
Add Parameter -> String Parameter
Name : git_ver
Default Value :
Description : 項目在git中央倉庫所對應的分支或者版本号
-
Add Parameter -> String Parameter
Name : add_tag
Default Value :
Description : docker鏡像标簽時間部分
-
Add Parameter -> String Parameter
Name : mvn_dir
Default Value : ./
Description : 編譯項目目錄路徑
-
Add Parameter -> String Parameter
Name : target_dir
Default Value : ./target
Description : 項目編譯完成項目後産生的war/jar的目錄
-
Add Parameter -> String Parameter
Name : mvn_cmd
Default Value : mvn clean package -Dmaven.test.skip=true
Description : 執行編譯所用的指令
-
Add Parameter -> Choice Paramete
Name : base_image
Default Value :
base/jre7:7u80
base/jre8:8u112
Description : 項目使用底包做鏡像
-
Add Parameter -> Choice Parameter
Name : maven
Default Value :
3.6.0-8u181
3.2.5-6u025
Description : 執行編譯使用maven軟體版本
Pipeline》
pipeline {
agent any
stages {
stage('pull') { //get project code from repo
steps {
sh "git clone ${params.git_repo} ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.app_name}/${env.BUILD_NUMBER} && git checkout ${params.git_ver}"
}
}
stage('build') { //exec mvn cmd
steps {
sh "cd ${params.app_name}/${env.BUILD_NUMBER} && /var/jenkins_home/maven-${params.maven}/bin/${params.mvn_cmd}"
}
}
stage('package') { //move jar file into project_dir
steps {
sh "cd ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.target_dir} && mkdir project_dir && mv *.jar ./project_dir"
}
}
stage('image') { //build image and push to registry
steps {
writeFile file: "${params.app_name}/${env.BUILD_NUMBER}/Dockerfile", text: """FROM harbor.od.com/${params.base_image}
ADD ${params.target_dir}/project_dir /opt/project_dir"""
sh "cd ${params.app_name}/${env.BUILD_NUMBER} && docker build -t harbor.od.com/${params.image_name}:${params.git_ver}_${params.add_tag} . && docker push harbor.od.com/${params.image_name}:${params.git_ver}_${params.add_tag}"
}
}
}
}
建構項目
編輯資源配置清單
節點 132
[[email protected] ~]# cd /data/k8s-yaml/
[[email protected] k8s-yaml]# mkdir dubbo-demo-service
[[email protected] k8s-yaml]# cd dubbo-demo-service/
[[email protected] dubbo-demo-service]# vi dp.yaml
kind: Deployment 定義資源類别，要建立的是POD就寫為pod、Deployment、StatefulSet等
apiVersion: extensions/v1beta1 定義版本
metadata: 中繼資料資訊，包含資源名稱、namespace等。namespace用于給資源進行分類，預設會有一個default名稱空間
name: dubbo-demo-service 名稱
namespace: app 名稱空間
labels: 标簽
name: dubbo-demo-service 名稱
spec: 聲明資源的屬性狀态，也就是說希望deployment是什麼樣的
replicas: 1 副本數量
selector: 控制器選擇器，通過他指定該控制器管理那些pod
matchLabels: labels比對規則
name: dubbo-demo-service 名稱
template: 模闆，當副本數量不足根據下面模闆建立pod副本
metadata: 中繼資料
labels: 标簽
app: dubbo-demo-service 給自己打标簽
name: dubbo-demo-service 名稱
spec: 聲明資源的屬性狀态，也就是說希望deployment是什麼樣的
containers: 建立容器
- name: dubbo-demo-service 名稱
image: harbor.od.com/app/dubbo-demo-service:master_20210825_1139 鏡像位址
ports: 端口設定
- containerPort: 20880 暴露20880端口
protocol: TCP 協定
env: 設定變量
- name: JAR_BALL JAR_BALL=dubbo-server.jar
value: dubbo-server.jar
imagePullPolicy: IfNotPresent 優先使用本地image，本地沒有再去下載下傳
imagePullSecrets: 引用建立secrets，私有倉庫必須加此參數
- name: harbor 當時建立secret時的名稱
restartPolicy: Always 容器停止運作時的重新開機政策
terminationGracePeriodSeconds: 30 等待pod緩沖時長，預設30秒，比如當你啟動pod超過30秒還沒running将被強制結束，此值根據實際業務
securityContext: 容器的程序都以UserID 0 的身份運作
runAsUser: 0 root方式啟動
schedulerName: default-scheduler 排程運算節點：預設排程方式
strategy: 将現有pod替換為新pod的部署政策
type: RollingUpdate 滾動更新配置參數，僅當類型為RollingUpdate
rollingUpdate:
maxUnavailable: 1 和期望ready的副本數比例
maxSurge: 1 滾動更新過程産生的最大pod數量
revisionHistoryLimit: 7 定義保留的更新記錄數
progressDeadlineSeconds: 600 滾動更新的最大時間600秒
kubectl添加名稱空間
[[email protected] ~]# kubectl create namespace app
namespace/app created
kubectl授權私有倉庫鏡像拉取權限
[[email protected] ~]# kubectl create secret docker-registry harbor --docker-server=harbor.od.com --docker-username=admin --docker-password=12345 -n app
secret/harbor created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-service/dp.yaml
deployment.extensions/dubbo-demo-service created
dubbo-monitor工具
傳送門：dubbo-monitor-master
節點 132
[[email protected] ~]# unzip dubbo-monitor-master.zip
[[email protected] conf]# vi /root/dubbo-monitor/dubbo-monitor-simple/conf/dubbo_origin.properties
dubbo.container=log4j,spring,registry,jetty
dubbo.application.name=simple-monitor
dubbo.application.owner=liujiangxu
dubbo.registry.address=zookeeper://zk1.od.com:2181?backup=zk2.od.com:2181,zk3.od.com:2181
dubbo.protocol.port=20880
dubbo.jetty.port=8080
dubbo.jetty.directory=/dubbo-monitor-simple/monitor
dubbo.charts.directory=/dubbo-monitor-simple/charts
dubbo.statistics.directory=/dubbo-monitor-simple/statistics
dubbo.log4j.file=logs/dubbo-monitor-simple.log
dubbo.log4j.level=WARN
建構dockerfile
[[email protected] ~]# cp -a dubbo-monitor /data/dockerfile/
[[email protected] ~]# cd /data/dockerfile/dubbo-monitor
[[email protected] dubbo-monitor]# docker build . -t harbor.od.com/infra/dubbo-monitor:latest
[[email protected] dubbo-monitor]# docker push harbor.od.com/infra/dubbo-monitor:latest
傳遞k8s配置資源清單
[[email protected] dubbo-monitor]# cat dp.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
name: dubbo-monitor
namespace: infra
labels:
name: dubbo-monitor
spec:
replicas: 1
selector:
matchLabels:
name: dubbo-monitor
template:
metadata:
labels:
app: dubbo-monitor
name: dubbo-monitor
spec:
containers:
- name: dubbo-monitor
image: harbor.od.com/infra/dubbo-monitor:latest
ports:
- containerPort: 8080
protocol: TCP
- containerPort: 20880
protocol: TCP
imagePullPolicy: IfNotPresent
imagePullSecrets:
- name: harbor
restartPolicy: Always
terminationGracePeriodSeconds: 30
securityContext:
runAsUser: 0
schedulerName: default-scheduler
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
revisionHistoryLimit: 7
progressDeadlineSeconds: 600
[[email protected] dubbo-monitor]# cat svc.yaml
kind: Service
apiVersion: v1
metadata:
name: dubbo-monitor
namespace: infra
spec:
ports:
- protocol: TCP
port: 8080 叢集網絡端口（叢集ip隻有一個服務，按道理端口随便配置）
targetPort: 8080 容器内端口
selector:
app: dubbo-monitor
[[email protected] dubbo-monitor]# cat ingress.yaml
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: dubbo-monitor
namespace: infra
spec:
rules:
- host: dubbo-monitor.od.com
http:
paths:
- path: /
backend:
serviceName: dubbo-monitor
servicePort: 8080 對應service.yaml中port端口配置
DNS解析ingress配置中 host域名
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-monitor/dp.yaml
deployment.extensions/dubbo-monitor created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-monitor/svc.yaml
service/dubbo-monitor created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-monitor/ingress.yaml
ingress.extensions/dubbo-monitor created
[[email protected] ~]# kubectl get pods -n infra -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
dubbo-monitor-5bb45c8b97-fnwt7 1/1 Running 0 6m59s 172.7.21.9 ceshi-130.host.com <none> <none>
jenkins-698b4994c8-hm5wf 1/1 Running 0 7d6h 172.7.22.9 ceshi-131.host.com <none> <none>
傳遞dubbo服務消費者
編輯資源配置清單
節點 132
[[email protected] ~]# cd /data/k8s-yaml/
[[email protected] k8s-yaml]# mkdir dubbo-demo-web
[[email protected] k8s-yaml]# cd dubbo-demo-web/
[[email protected] dubbo-demo-web]# vi dp.yaml
kind: Deployment 定義資源類别，要建立的是POD就寫為pod、Deployment、StatefulSet等
apiVersion: extensions/v1beta1 定義版本
metadata: 中繼資料資訊，包含資源名稱、namespace等。namespace用于給資源進行分類，預設會有一個default名稱空間
name: dubbo-demo-consumer 名稱
namespace: app 名稱空間
labels: 标簽
name: dubbo-demo-consumer 名稱
spec: 聲明資源的屬性狀态，也就是說希望deployment是什麼樣的
replicas: 1 副本數量
selector: 控制器選擇器，通過他指定該控制器管理那些pod
matchLabels: labels比對規則
name: dubbo-demo-consumer 名稱
template: 模闆，當副本數量不足根據下面模闆建立pod副本
metadata: 中繼資料
labels: 标簽
app: dubbo-demo-consumer給自己打标簽
name: dubbo-demo-consumer名稱
spec: 聲明資源的屬性狀态，也就是說希望deployment是什麼樣的
containers: 建立容器
- name: dubbo-demo-consumer 名稱
image: harbor.od.com/app/dubbo-demo-consumer:master_20210826_1040 鏡像位址
ports: 端口設定
- containerPort: 8080
protocol: TCP
- containerPort: 20880
protocol: TCP
env: 設定變量
- name: JAR_BALL JAR_BALL=dubbo-client.jar
value: dubbo-client.jar
imagePullPolicy: IfNotPresent 優先使用本地image，本地沒有再去下載下傳
imagePullSecrets: 引用建立secrets，私有倉庫必須加此參數
- name: harbor 當時建立secret時的名稱
restartPolicy: Always 容器停止運作時的重新開機政策
terminationGracePeriodSeconds: 30 等待pod緩沖時長，預設30秒，比如當你啟動pod超過30秒還沒running将被強制結束，此值根據實際業務
securityContext: 容器的程序都以UserID 0 的身份運作
runAsUser: 0 root方式啟動
schedulerName: default-scheduler 排程運算節點：預設排程方式
strategy: 将現有pod替換為新pod的部署政策
type: RollingUpdate 滾動更新配置參數，僅當類型為RollingUpdate
rollingUpdate:
maxUnavailable: 1 和期望ready的副本數比例
maxSurge: 1 滾動更新過程産生的最大pod數量
revisionHistoryLimit: 7 定義保留的更新記錄數
progressDeadlineSeconds: 600 滾動更新的最大時間600秒
[[email protected] dubbo-demo-web]# vi svc.yaml
kind: Service
apiVersion: v1
metadata:
name: dubbo-demo-consumer
namespace: app
spec:
ports:
- protocol: TCP
port: 8080
targetPort: 8080
selector:
app: dubbo-demo-consumer
[[email protected] dubbo-demo-web]# vi ingress.yaml
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: dubbo-demo-consumer
namespace: app
spec:
rules:
- host: demo.od.com
http:
paths:
- path: /
backend:
serviceName: dubbo-demo-consumer
servicePort: 8080
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-web/dp.yaml
deployment.extensions/dubbo-demo-consumer created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-web/svc.yaml
service/dubbo-demo-consumer created
[[email protected] ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-web/ingress.yaml
ingress.extensions/dubbo-demo-consumer created