一、部署方式对比¶
1.1 原有部署方式（Jenkins + Yaml/Helm + Kubernetes）¶
基于 Yaml|Helm 做全部应用的管理，但是仅支持底层 CLI 模式，缺乏界面，而且各应用的配置不相同， Charts 维护成本高。
1.2 现有部署方式（Jenkins + ArgoCD + Kubernetes）¶
1、研发通过git客户端将代码推送到GitLab
2、通过Gitlab的webhook通知并触发jenkins相关流水线任务
3、jenkins使用Dockerfile构建镜像
4、jenkins将构建完成的镜像发送到私有镜像仓库Harbor
5、将相关的yaml文件上传到GitLab
6、GitLab通过webHook触发argo进行部署
7、k8s集群通过yaml文件指定的镜像通过Harbor进行拉取
利用 Gitops 思想，去触发 Argocd 部署。CI 部分基于 Jenkins 更加高效畅快，CD 部分基于 ArgoCD 提升部署+管控效率！
二、基于 Jenkins + ArgoCD + Kubernetes 智能化流程¶
2.1 Jenkins Pipeline¶
改造来自 CICD 章节的：KUBERNETES_APPS_DEPLOYMENT
//定义git相关数据
def git_address = "http://gitlab.example.com/demoteam/${GIT_PROJECT_NAME}.git"
def git_auth = "2e1860e9-8891-4a18-95aa-72bbefc0fda6"
//构建版本的名称
def tag = "latest"
//Harbor私服地址
def harbor_url = "harbor.example.com"
//Harbor的项目名称
def harbor_project_name = "demo"
//Harbor的凭证
def harbor_auth = "8b75c989-f6f9-46bb-a706-3630a21149d6"
//启动时间
def start = new Date().format('yyyy-MM-dd HH:mm:ss')
//创建一个Pod的模板，label为jenkins-slave
podTemplate(label: 'jenkins-slave-app-argocd', cloud: 'kubernetes', nodeSelector: 'jenkins-jnlp=yes', containers: [
containerTemplate(
name: 'jnlp',
image: "harbor.example.com/library/jenkins-slave-maven:jdk-11-2",
ttyEnabled: true
),
containerTemplate(
name: 'docker',
image: "harbor.example.com/library/docker:stable",
ttyEnabled: true,
command: 'cat'
),
],
volumes: [
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
],
)
{
node("jenkins-slave-app-argocd"){
// 第一步
stage('Pull'){
checkout([$class: 'GitSCM', branches: [[name: "${branch}"]], extensions: [], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
}
stage('BuildDescription') {
// 自定义设置构建历史显示的名称和描述信息
// 不同的部署方式设置构建历史显示的名称和描述信息方式不一样，根据自己的部署方式自行百度找到设置方法
script {
//设置buildName
wrap([$class: 'BuildUser']) {
//修改Description
buildDescription "${BUILD_USER} > ${project_name} > ${branch}"
}
}
}
// 第二步
stage('Build&Tag&Push&Deploy'){
//把选择的项目信息转为数组
def selectedProjects = "${project_name}".split(',')
for(int i=0;i<selectedProjects.size();i++){
//取出每个项目的名称
def currentProjectName = selectedProjects[i];
//定义镜像名称
def imageName = "${currentProjectName}:${tag}"
//定义newTag
def newTag = sh(returnStdout: true,script: 'echo `date +"%Y%m%d%H%M"_``git describe --tags --always`').trim()
//编译，构建本地镜像
//sh "sed -i 's#ACTIVEPROFILE#${springProfilesActive}#g' Dockerfile"
if ( "${GIT_PROJECT_NAME}" == "java_kubernetes" ) {
sh "mvn clean package -Dmaven.test.skip=true"
}
container('docker') {
//镜像编译
sh "docker build -t ${imageName} ."
//给镜像打标签
sh "docker tag ${imageName} ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
//登录Harbor，并上传镜像
withCredentials([usernamePassword(credentialsId: "${harbor_auth}", passwordVariable: 'password', usernameVariable: 'username')])
{
//登录
sh "docker login -u ${username} -p ${password} ${harbor_url}"
//上传镜像
sh "docker push ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
}
//删除本地镜像
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
}
//修改最新的镜像
sh """
sed -i 's@image: .*@image: harbor.example.com/demo/spring-java-demo:${newTag}@' yaml/deployment.yaml
cat yaml/deployment.yaml
"""
//打tag并推送
withCredentials([usernamePassword(credentialsId: "${git_auth}", passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')])
{
sh '''
git config user.email "zhdya@zhdya.cn"
git config user.name "root"
git add yaml/deployment.yaml
git commit -m "Update deployment version ${BUILD_NUMBER}"
git push http://$GIT_USERNAME:$GIT_PASSWORD@gitlab.example.com/demoteam/${GIT_PROJECT_NAME}.git HEAD:main
'''
}
}
}
}
}
2.2 项目 Gitlab 配置 webhook¶
2.2.1 新建项目¶
1、依次点击【项目】-【创建空白项目】
2、定义项目名称为JAVA Kubernetes Argocd，取消勾选【使用自述文件初始化仓库】
2.2.2 配置webhook¶
1、进入管理中心中，依次点击【设置】-【网络】-【出站请求】，勾选【允许来自 webhooks 和集成对本地网络的请求】
2、进入JAVA Kubernetes Argocd这个项目里面，依次点击【设置】-【webhook】,定义如下内容：
-
Url：https://argocd.example.com/api/webhook
-
Secret token：<gitlab-webhook-secret>
-
按需勾选：push event
3、添加完成webhook后，点击【测试】-【推送事件】，正常回显如下：
Hook executed successfully: HTTP 200
2.3 创建 Application¶
2.3.1 开始创建¶
调整gitlab的ingress配置文件，允许上传大文件
# 添加下面内容
[root@master01 4]# vim gitlab-ing.yaml
...
...
nginx.ingress.kubernetes.io/enable-cors: 'true'
nginx.ingress.kubernetes.io/proxy-body-size: 20G
# 完整配置文件
[root@master01 4]# cat gitlab-ing.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/enable-cors: 'true'
nginx.ingress.kubernetes.io/proxy-body-size: 20G
name: gitlab-ing
namespace: devops
spec:
ingressClassName: nginx
rules:
- host: gitlab.example.com
http:
paths:
- backend:
service:
name: gitlab
port:
number: 80
path: /
pathType: Prefix
# 应用
[root@master01 4]# kaf gitlab-ing.yaml
上传代码文件到gitlab
# 准备代码文件放到java_kubernetes_argocd目录下
[root@master01 ~]# cd /root/17/argo-rollouts
[root@master01 argo-rollouts]# mkdir java_kubernetes_argocd
# 上传代码文件到gitlab
## 初始化
[root@master01 ~]# cd /root/17/argo-rollouts/java_kubernetes_argocd
[root@master01 java_kubernetes_argocd]# git init
## 添加远端仓库
[root@master01 java_kubernetes_argocd]# git remote add origin http://gitlab.example.com/demoteam/java-kubernetes-argocd.git
## 验证查看
[root@master01 java_kubernetes_argocd]# git remote -v
origin http://gitlab.example.com/demoteam/java-kubernetes-argocd.git (fetch)
origin http://gitlab.example.com/demoteam/java-kubernetes-argocd.git (push)
## 添加到暂存区
[root@master01 java_kubernetes_argocd]# git add .
## 提交到本地仓库
[root@master01 java_kubernetes_argocd]# git commit -m "first for java_kubernetes_argocd"
## 切换到main分支
[root@master01 java_kubernetes_argocd]# git branch -M main
## 上传到main分支
[root@master01 java_kubernetes_argocd]# git push -uf origin main
Username for 'http://gitlab.example.com': root
Password for 'http://root@gitlab.example.com':
<gitlab-password>
...
...
# 编写zhdya-java-demo.yaml文件
[root@master01 ~]# cd /root/17/argo-rollouts
[root@master01 argo-rollouts]#
cat <<EOF >> zhdya-java-demo.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: zhdya-java-demo
namespace: argocd
spec:
destination:
name: ''
namespace: default
server: 'https://kubernetes.default.svc'
source:
path: yaml
repoURL: 'http://gitlab.example.com/demoteam/java_kubernetes_argocd.git'
targetRevision: main
sources: []
project: default
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
selfHeal: true
prune: true
EOF
# 验证
[root@master01 java_kubernetes_argocd]# ll
total 4
-rw-r--r-- 1 root root 482 Apr 25 12:48 zhdya-java-demo.yaml
2.3.2 遇到问题-413报错¶
报错信息
# 提示error: RPC failed; result=22, HTTP code = 413
[root@master01 java_kubernetes_argocd]# git push -uf origin main
Username for 'http://gitlab.example.com': root
Password for 'http://root@gitlab.example.com':
Counting objects: 45, done.
Delta compression using up to 8 threads.
Compressing objects: 100% (28/28), done.
error: RPC failed; result=22, HTTP code = 413
fatal: The remote end hung up unexpectedly
Writing objects: 100% (45/45), 15.15 MiB | 0 bytes/s, done.
Total 45 (delta 1), reused 0 (delta 0)
fatal: The remote end hung up unexpectedly
Everything up-to-date
报错解决方法
# 添加下面内容
[root@master01 4]# vim gitlab-ing.yaml
...
...
nginx.ingress.kubernetes.io/enable-cors: 'true'
nginx.ingress.kubernetes.io/proxy-body-size: 20G
# 完整配置文件
[root@master01 4]# vim gitlab-ing.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/enable-cors: 'true'
nginx.ingress.kubernetes.io/proxy-body-size: 20G
name: gitlab-ing
namespace: devops
spec:
ingressClassName: nginx
rules:
- host: gitlab.example.com
http:
paths:
- backend:
service:
name: gitlab
port:
number: 80
path: /
pathType: Prefix
# 重新应用
[root@master01 4]# kaf gitlab-ing.yaml
2.4 Argocd 添加应用 repo 仓库¶
# 先使用密码登录
[root@master01 17]# argocd login argocd.example.com
Username: admin
Password: <new-admin-password>
# 添加repo源（添加前要保证对面项目不为空）
[root@master01 ~]# argocd repo add http://gitlab.example.com/demoteam/java-kubernetes-argocd.git --username root --password <gitlab-password> --insecure-skip-server-verification
2.5 应用¶
目前只是生成一个空的任务，任务名叫zhdya-java-demo
[root@master01 ~]# cd /root/17/argo-rollouts
[root@master01 argo-rollouts]# kubectl apply -f zhdya-java-demo.yaml
三、Jenkins + ArgoCD 验证¶
涉及到的底层 deployment 控制器文件：
apiVersion: v1
kind: Service
metadata:
name: spring-java-demo
spec:
type: ClusterIP
selector:
app: spring-java-demo
ports:
- name: web
port: 8080
protocol: TCP
targetPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: spring-java-demo
spec:
replicas: 1
selector:
matchLabels:
app: spring-java-demo
template:
metadata:
labels:
app: spring-java-demo
spec:
containers:
- name: spring-java-demo
image: harbor.example.com/demo/spring-java-demo:202409091002_8998f8a
resources:
limits:
memory: 0.5
cpu: 500
requests:
memory: 0.5
cpu: 500
ports:
- containerPort: 8080
name: web
livenessProbe:
httpGet:
port: web
path: /apptwo
timeoutSeconds: 2 # 表示容器必须在2s内做出相应反馈给probe，否则视为探测失败
periodSeconds: 30 # 探测周期，每30s探测一次
readinessProbe:
tcpSocket:
port: web
initialDelaySeconds: 10 # 容器启动后10s开始探测
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: java-api-ingress
spec:
ingressClassName: nginx
rules:
- host: java-argo.example.com
http:
paths:
- backend:
service:
name: spring-java-demo
port:
number: 8080
path: /
pathType: Prefix
Jenkins 执行编译后的镜像：
Gitlab 成功 webhook 推送：
开始自动更新：
验证镜像数据：
四、改造基于金丝雀的自动化发布¶
4.1 创建 Rollouts¶
修改如上应用的 deployment 为 rollouts 资源：
cat <<EOF >> deployment.yaml
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
name: spring-java-canary-rollouts-analysis
namespace: demo
spec:
replicas: 3
strategy:
canary:
analysis:
templates:
- templateName: success-rate-java # 使用的 AnalysisTemplates
startingStep: 2 # step 的索引，从第 2 个 step 开始分析（40%），第 1 个是 20% 初始的时候
args: # 传入 AnalysisTemplates 的参数
- name: ingress
value: spring-java-stable-ing
canaryService: spring-java-canary
stableService: spring-java-stable
trafficRouting:
nginx:
stableIngress: spring-java-stable-ing
# 发布的节奏
steps:
- setWeight: 20
- pause: {} # 需要手动确认通过
- setWeight: 40
- pause: {duration: 20s}
- setWeight: 60
- pause: {duration: 20s}
- setWeight: 80
- pause: {duration: 20s}
revisionHistoryLimit: 2
selector:
matchLabels:
app: spring-java-demo
template:
metadata:
labels:
app: spring-java-demo
spec:
containers:
- name: spring-java-demo
image: harbor.example.com/demo/spring-java-demo:202409111530_3fa1a9a
ports:
- name: http
containerPort: 8080
protocol: TCP
livenessProbe:
httpGet:
port: http
path: /apptwo
timeoutSeconds: 2 # 表示容器必须在2s内做出相应反馈给probe，否则视为探测失败
periodSeconds: 30 # 探测周期，每30s探测一次
readinessProbe:
tcpSocket:
port: http
initialDelaySeconds: 10 # 容器启动后10s开始探测
EOF
4.2 创建应用的 stable/canary SVC¶
cat <<EOF >> svc.yaml
apiVersion: v1
kind: Service
metadata:
name: spring-java-canary
namespace: demo
spec:
ports:
- port: 8080
targetPort: http
protocol: TCP
name: http
selector:
app: spring-java-demo
---
apiVersion: v1
kind: Service
metadata:
name: spring-java-stable
namespace: demo
spec:
ports:
- port: 8080
targetPort: http
protocol: TCP
name: http
selector:
app: spring-java-demo
EOF
4.3 创建应用的 ingress 路由¶
cat <<EOF >> ing.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: spring-java-stable-ing
namespace: demo
spec:
ingressClassName: nginx
rules:
- host: java-argo.example.com
http:
paths:
- backend:
service:
name: spring-java-stable
port:
number: 8080
path: /
pathType: Prefix
EOF
4.4 创建自动分析 AnalysisTemplate 模板¶
cat <<EOF >> analysis-success.yaml
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
name: success-rate-java
namespace: demo
spec:
args:
- name: ingress
metrics:
- name: success-rate
initialDelay: 3s # 延迟 3s 后启动
interval: 3s # 查询指标的频率
failureLimit: 2 # 2 次不满足 successCondition 则视为失败
successCondition: result[0] > 0.90 # 成功条件：测量值为空（指标还没采集到）或者大于 90%
provider:
prometheus:
address: http://prometheus.monitor.svc:9090 # Prometheus 地址
query: >+ # 查询语句
sum(
rate(nginx_ingress_controller_requests{ingress="{{args.ingress}}",status!~"[4-5].*"}[60s]))
/
sum(rate(nginx_ingress_controller_requests{ingress="{{args.ingress}}"}[60s])
)
EOF
五、整体联调验证¶
5.1 应用迭代成功上线¶
测试访问：
# for i in {1..5000}; do curl java-argo.example.com/appone; done
Jenkins 发布：
Gitlab 验证：
webhook 验证：
Argocd UI 验证：
Argo Rollouts 验证：
成功上线：
5.2 应用迭代失败回滚¶
参照视频中描述的步骤来实现即可！