相对于基于控制器文件部署的方式有哪些优点：

  • 利于形成DEVOPS标准化；
  • 控制器方式需要维护大量的yaml文件；
  • 相对于helm方式，控制器方式低效且不够灵活；

当前环境基于控制器文件部署(涉及部署需参考前面内容)：

// 定义git相关数据
def git_address = "http://gitlab.zhang-qing.com/demoteam/java-kubernetes.git"
def git_auth = "Gitlab-username"

// 构建版本的名称
def tag = "latest"
// Harbor私服地址
def harbor_url = "harbor.zhang-qing.com"
// Harbor的项目名称
def harbor_project_name = "demo"
// Harbor的凭证
def harbor_auth = "Harbor-username"
// 启动时间
def start = new Date().format('yyyy-MM-dd HH:mm:ss')

// 创建一个Pod的模板，label为jenkins-slave
podTemplate(
    label: 'jenkins-slave-java',
    cloud: 'study-kubernetes',
    containers: [
        containerTemplate(
            name: 'jnlp',
            image: "harbor.zhang-qing.com/library/jenkins-slave-maven:v1",
            ttyEnabled: true
        ),

        containerTemplate(
            name: 'docker',
            image: "harbor.zhang-qing.com/library/docker:stable",
            ttyEnabled: true,
            command: 'cat'
        )
    ],
    volumes: [
        hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock')
    ]
) {
    node("jenkins-slave-java") {
        // 第一步
        stage('Pull') {
            checkout([
                $class: 'GitSCM',
                branches: [[name: "${branch}"]],
                extensions: [],
                userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]
            ])
        }

        stage('BuildDescription') {
            // 自定义设置构建历史显示的名称和描述信息
            // 不同的部署方式设置构建历史显示的名称和描述信息方式不一样，根据自己的部署方式自行百度找到设置方法
            script {
                // 设置buildName
                wrap([$class: 'BuildUser']) {
                    // 修改Description
                    buildDescription "${BUILD_USER} > ${project_name} > ${branch}"
                }
            }
        }

        // 第二步
        stage('Build&Tag&Push&Deploy') {
            // 把选择的项目信息转为数组
            def selectedProjects = "${project_name}".split(',')

            for(int i = 0; i < selectedProjects.size(); i++) {
                // 取出每个项目的名称
                def currentProjectName = selectedProjects[i];

                // 定义镜像名称
                def imageName = "${currentProjectName}:${tag}"

                // 定义newTag
                def newTag = sh(returnStdout: true, script: 'echo `date +"%Y%m%d%H%M"_``git describe --tags --always`').trim()

                // 编译，构建本地镜像
                // sh "sed -i 's#ACTIVEPROFILE#${springProfilesActive}#g' Dockerfile"
                sh "mvn clean package -Dmaven.test.skip=true"
                container('docker') {

                    // 镜像编译
                    sh "docker build -t ${imageName} ."

                    // 给镜像打标签
                    sh "docker tag ${imageName} ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"

                    // 登录Harbor，并上传镜像
                    withCredentials([usernamePassword(credentialsId: "${harbor_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
                        // 登录
                        sh "docker login -u ${username} -p ${password} ${harbor_url}"
                        // 上传镜像
                        sh "docker push ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
                    }

                    // 删除本地镜像
                    sh "docker rmi -f ${imageName}"
                    sh "docker rmi -f ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
                }
                def deploy_image_name = "${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"

                // 基于控制器的方式部署到K8S
                sh """
                    sed -i 's#\$IMAGE_NAME#${deploy_image_name}#' deployment.yaml
                    sed -i 's#\$APP_NAME#${currentProjectName}#' deployment.yaml
                    sed -i 's#\$APP_REPLICAS#${replicas}#' deployment.yaml
                    sed -i 's#\$NAMESPACE#${namespaces}#' deployment.yaml
                    sed -i 's#\$SPRINGENV#${springProfilesActive}#' deployment.yaml
                    sed -i 's#\$PODMEMORY#${podsMem}#' deployment.yaml
                    sed -i 's#\$PODCPU#${podsCpu}#' deployment.yaml
                    cat deployment.yaml
                """
                // 部署到K8S
                kubernetesDeploy(kubeconfigId: "kubeconfig", configs: "deployment.yaml")
            }
        }
    }
}

一、推送与安装Chart

# 生成指定包
[root@master01 ~]# cd /root/5/demo-helm/
[root@master01 demo-helm]# helm package .
Successfully packaged chart and saved it to: /root/5/demo-helm/demohelm-0.2.2.tgz

#添加repo
[root@master01 helm-push]# helm repo add devopsrepo http://harbor.zhang-qing.com/chartrepo/demo \
  --username admin \
  --password 'Harbor12345'

# 上传包到harbor仓库
[root@master01 demo-helm]#
helm cm-push demohelm-0.2.2.tgz https://harbor.zhang-qing.com/chartrepo/demo \
  --username=admin \
  --password='Harbor12345'

Pushing demohelm-0.2.2.tgz to https://harbor.zhang-qing.com/chartrepo/demo...
Done.

浏览器输入https://harbor.zhang-qing.com/登录harbor,账号密码为admin/Harbor12345

image-20250508195127862

问题现象：

在推送和安装Chart时出现，如下报错信息

[root@master01 demo-helm]#
helm cm-push demohelm-0.2.2.tgz https://harbor.zhang-qing.com/chartrepo/demo \
  --username=admin \
  --password='Harbor12345'

Error: 308: could not properly parse response JSON: <html>
<head><title>308 Permanent Redirect</title></head>
<body>
<center><h1>308 Permanent Redirect</h1></center>
<hr><center>nginx/1.22.0</center>
</body>
</html>
...
...

问题处理：

将http://harbor.zhang-qing.com/chartrepo/demo替换为https://harbor.zhang-qing.com/chartrepo/demo

[root@master01 demo-helm]#
helm cm-push demohelm-0.2.2.tgz https://harbor.zhang-qing.com/chartrepo/demo \
  --username=admin \
  --password='Harbor12345'

二、基于helm的部署方式

1、准备k8s-helm容器Dockerfile文件

(1)国内环境(本实验使用)

[root@master01 ~]# cd /root/5/demo-helm/
[root@master01 demo-helm]# vim Dockerfile
# 使用新版Alpine镜像
FROM registry.cn-hangzhou.aliyuncs.com/abroad_images/alpine:3.18

# Metadata
LABEL \
  org.opencontainers.image.title="lachlanevenson/k8s-helm" \
  org.opencontainers.image.url="https://helm.sh/docs/"

ENV HELM_VERSION="v3.7.2"
ARG TARGETARCH="amd64"

# 设置阿里云镜像源
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories

# 安装依赖
RUN apk add --no-cache --update \
    ca-certificates \
    wget \
    git \
    openssl \
    bash && \
    wget -q "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" && \
    tar -xzf "helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" && \
    mv "linux-${TARGETARCH}/helm" /usr/local/bin/helm && \
    rm -rf "helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" && \
    chmod +x /usr/local/bin/helm

ENTRYPOINT ["helm"]
CMD ["help"]

(2)国外环境

[root@master01 ~]# cd /root/5/demo-helm/
[root@master01 demo-helm]# vim Dockerfile
FROM registry.cn-hangzhou.aliyuncs.com/abroad_images/alpine:3.6

# Metadata
LABEL \
  org.opencontainers.image.title="lachlanevenson/k8s-helm" \
  org.opencontainers.image.url="https://helm.sh/docs/"

ENV HELM_LATEST_VERSION="v3.7.2"

ARG TARGETARCH
ENV TARGETARCH=${TARGETARCH:-amd64}

RUN \
  apk add --no-cache --update ca-certificates && \
  apk add --no-cache --update -t deps wget git openssl bash && \
  wget -q "https://get.helm.sh/helm-${HELM_LATEST_VERSION}-linux-${TARGETARCH}.tar.gz" && \
  tar -xf "helm-${HELM_LATEST_VERSION}-linux-${TARGETARCH}.tar.gz" && \
  mv "linux-${TARGETARCH}/helm" /usr/local/bin && \
  apk del --purge deps && \
  rm -rf /var/cache/apk/* && \
  rm -f "helm-${HELM_LATEST_VERSION}-linux-${TARGETARCH}.tar.gz"

ENTRYPOINT ["helm"]
CMD ["help"]

2、构建镜像

[root@master01 demo-helm]#docker build -t harbor.zhang-qing.com/library/k8s-helm:v3.7.2 .

3、上传镜像到Harbor

[root@master01 demo-helm]# docker push harbor.zhang-qing.com/library/k8s-helm:v3.7.2

4、定义一个名为JAVA-Demo-Test-Helm的Pipline

// 定义git相关数据
def git_address = "http://gitlab.zhang-qing.com/demoteam/java-kubernetes.git"
def git_auth = "Gitlab-username"

// 构建版本的名称
def tag = "latest"
// Harbor私服地址
def harbor_url = "harbor.zhang-qing.com"
// Harbor的项目名称
def harbor_project_name = "demo"
// Harbor的凭证
def harbor_auth = "Harbor-username"
// 启动时间
def start = new Date().format('yyyy-MM-dd HH:mm:ss')

// 创建一个Pod的模板，label为jenkins-slave
podTemplate(
    label: 'jenkins-slave-java',
    cloud: 'study-kubernetes',
    containers: [
        containerTemplate(
            name: 'jnlp',
            image: "harbor.zhang-qing.com/library/jenkins-slave-maven:v1",
            ttyEnabled: true
        ),
        containerTemplate(
            name: 'docker',
            image: "harbor.zhang-qing.com/library/docker:stable",
            ttyEnabled: true,
            command: 'cat'
        ),
        containerTemplate(
            name: 'helm3',
            image: "harbor.zhang-qing.com/library/k8s-helm:v3.7.2",
            ttyEnabled: true,
            command: 'cat'
        )
    ],
    volumes: [
        hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock')
    ]
) {
    node("jenkins-slave-java") {
        // 第一步：拉取代码
        stage('Pull') {
            checkout([
                $class: 'GitSCM',
                branches: [[name: branch]],  // 直接使用变量名
                extensions: [],
                userRemoteConfigs: [[credentialsId: git_auth, url: git_address]]
            ])
        }

        // 构建描述信息
        stage('BuildDescription') {
            script {
                // 设置buildName
                wrap([$class: 'BuildUser']) {
                    // 修改Description
                    buildDescription "${BUILD_USER} > ${project_name} > ${branch}"
                }
            }
        }

        // 第二步
        stage('Build&Tag&Push&Deploy') {
            // 把选择的项目信息转为数组
            def selectedProjects = "${project_name}".split(',')

            for (int i = 0; i < selectedProjects.size(); i++) {
                // 取出每个项目的名称
                def currentProjectName = selectedProjects[i];

                // 定义镜像名称
                def imageName = "${currentProjectName}:${tag}"

                // 定义newTag
                def newTag = sh(returnStdout: true, script: 'echo `date +"%Y%m%d%H%M"_``git describe --tags --always`').trim()

                // 编译，构建本地镜像
                // sh "sed -i 's#ACTIVEPROFILE#${springProfilesActive}#g' Dockerfile"
                sh "mvn clean package -Dmaven.test.skip=true"

                container('docker') {
                    // 镜像编译
                    sh "docker build -t ${imageName} ."

                    // 给镜像打标签
                    sh "docker tag ${imageName} ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"

                    // 登录Harbor，并上传镜像
                    withCredentials([usernamePassword(
                        credentialsId: "${harbor_auth}",
                        passwordVariable: 'password',
                        usernameVariable: 'username'
                    )]) {
                        // 登录
                        sh "docker login -u ${username} -p ${password} ${harbor_url}"
                        // 上传镜像
                        sh "docker push ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
                    }

                    // 删除本地镜像
                    sh "docker rmi -f ${imageName}"
                    sh "docker rmi -f ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
                }

                def deploy_image_name = "${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"

                // Helm部署
                container('helm3') {
                    withCredentials([usernamePassword(
                        credentialsId: "${harbor_auth}",
                        passwordVariable: 'password',
                        usernameVariable: 'username'
                    )]) {
                        sh """
                        # 添加Helm仓库
                        helm repo add --username=${username} --password=${password} devopsrepo https://harbor.zhang-qing.com/chartrepo/demo \
                        """
                    }

                    withCredentials([file(credentialsId: 'kubernetes', variable: 'KUBECONFIG')]) {
                        sh """
                        mkdir -p /root/.kube/ && echo $KUBECONFIG >/root/.kube/config
                        echo "Helm应用配置信息确认..."
                        helm upgrade --install --dry-run --debug ${currentProjectName} --namespace ${namespaces} devopsrepo/demohelm \
                        --set replicaCount=${replicas} \
                        --set image.repository=${deploy_image_name} \
                        --set service.type=ClusterIP \
                        --set springActive=${springProfilesActive} \
                        --set resources.limits.memory=${limiteMem} \
                        --set ingress.enabled=${isIngress}

                        echo "应用部署..."
                        helm upgrade --install ${currentProjectName} --namespace ${namespaces} devopsrepo/demohelm \
                        --set replicaCount=${replicas} \
                        --set image.repository=${deploy_image_name} \
                        --set service.type=ClusterIP \
                        --set springActive=${springProfilesActive} \
                        --set resources.limits.memory=${limiteMem} \
                        --set ingress.enabled=${isIngress}
                        """
                    }
                }
            }
        }
    }
}

5、定义字符串参数

依次点击【参数化构建过程】-【字符参数】

  • 第一个字符参数：

  • 名称：branch

  • 默认值：main
  • 描述：Git 分支

  • 第二个字符参数：

  • 名称：project_name

  • 默认值：springboot
  • 描述：项目名称

  • 第三个字符参数：

  • 名称：replicas

  • 默认值：1
  • 描述：副本数

  • 第四个字符参数：

  • 名称：namespaces

  • 默认值：demo
  • 描述：K8s 命名空间

  • 第五个字符参数：

  • 名称：springProfilesActive

  • 默认值：dev

  • 描述：Spring 环境

  • 第六个字符参数：

  • 名称：limiteMem
  • 默认值：2Gi
  • 描述：内存 限制
  • 第七个字符参数：
  • 名称：isIngress
  • 默认值：false
  • 描述：Ingress开关

6、填写完成后，点击【应用】-【Save】

7、点击【Build with Parameters】,默认值即可，点击【Build】进行构建即可

8、构建完成后，验证查看

#查看pod、svc和ingress
[root@master01 ~]# kg deploy,svc,ingress -n demo
NAME                         READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/springboot   1/1     1            1           6m9s

NAME                 TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
service/springboot   ClusterIP   192.168.36.172   <none>        8080/TCP   6m9s

NAME                                       CLASS   HOSTS                     ADDRESS     PORTS   AGE
ingress.networking.k8s.io/springboot-ing   nginx   api-test.zhang-qing.com   10.0.0.11   80      6m9s

三、环境还原

[root@master01 ~]# k delete ingress springboot-ing -n demo
[root@master01 ~]# k delete svc,deploy springboot   -n demo

四、遇到的问题

问题报错信息

ERROR: Credentials 'kubeconfig' is of type 'Kubernetes configuration (kubeconfig)' where 'org.jenkinsci.plugins.plaincredentials.FileCredentials' was expected
Finished: FAILURE

问题报错原因

helm部署不支持Kubernetes configuration (kubeconfig)格式的凭据，这里需要增加Secret file格式的凭据

问题处理

添加下面内容

                    withCredentials([file(credentialsId: 'kubernetes', variable: 'KUBECONFIG')]) {

完整配置文件

// 定义git相关数据
def git_address = "http://gitlab.zhang-qing.com/demoteam/java-kubernetes.git"
def git_auth = "Gitlab-username"

// 构建版本的名称
def tag = "latest"
// Harbor私服地址
def harbor_url = "harbor.zhang-qing.com"
// Harbor的项目名称
def harbor_project_name = "demo"
// Harbor的凭证
def harbor_auth = "Harbor-username"
// 启动时间
def start = new Date().format('yyyy-MM-dd HH:mm:ss')

// 创建一个Pod的模板，label为jenkins-slave
podTemplate(
    label: 'jenkins-slave-java',
    cloud: 'study-kubernetes',
    containers: [
        containerTemplate(
            name: 'jnlp',
            image: "harbor.zhang-qing.com/library/jenkins-slave-maven:v1",
            ttyEnabled: true
        ),
        containerTemplate(
            name: 'docker',
            image: "harbor.zhang-qing.com/library/docker:stable",
            ttyEnabled: true,
            command: 'cat'
        ),
        containerTemplate(
            name: 'helm3',
            image: "harbor.zhang-qing.com/library/k8s-helm:v3.7.2",
            ttyEnabled: true,
            command: 'cat'
        )
    ],
    volumes: [
        hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock')
    ]
) {
    node("jenkins-slave-java") {
        // 第一步：拉取代码
        stage('Pull') {
            checkout([
                $class: 'GitSCM',
                branches: [[name: branch]],  // 直接使用变量名
                extensions: [],
                userRemoteConfigs: [[credentialsId: git_auth, url: git_address]]
            ])
        }

        // 构建描述信息
        stage('BuildDescription') {
            script {
                // 设置buildName
                wrap([$class: 'BuildUser']) {
                    // 修改Description
                    buildDescription "${BUILD_USER} > ${project_name} > ${branch}"
                }
            }
        }

        // 第二步
        stage('Build&Tag&Push&Deploy') {
            // 把选择的项目信息转为数组
            def selectedProjects = "${project_name}".split(',')

            for (int i = 0; i < selectedProjects.size(); i++) {
                // 取出每个项目的名称
                def currentProjectName = selectedProjects[i];

                // 定义镜像名称
                def imageName = "${currentProjectName}:${tag}"

                // 定义newTag
                def newTag = sh(returnStdout: true, script: 'echo `date +"%Y%m%d%H%M"_``git describe --tags --always`').trim()

                // 编译，构建本地镜像
                // sh "sed -i 's#ACTIVEPROFILE#${springProfilesActive}#g' Dockerfile"
                sh "mvn clean package -Dmaven.test.skip=true"

                container('docker') {
                    // 镜像编译
                    sh "docker build -t ${imageName} ."

                    // 给镜像打标签
                    sh "docker tag ${imageName} ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"

                    // 登录Harbor，并上传镜像
                    withCredentials([usernamePassword(
                        credentialsId: "${harbor_auth}",
                        passwordVariable: 'password',
                        usernameVariable: 'username'
                    )]) {
                        // 登录
                        sh "docker login -u ${username} -p ${password} ${harbor_url}"
                        // 上传镜像
                        sh "docker push ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
                    }

                    // 删除本地镜像
                    sh "docker rmi -f ${imageName}"
                    sh "docker rmi -f ${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"
                }

                def deploy_image_name = "${harbor_url}/${harbor_project_name}/${currentProjectName}:${newTag}"

                // Helm部署
                container('helm3') {
                    withCredentials([usernamePassword(
                        credentialsId: "${harbor_auth}",
                        passwordVariable: 'password',
                        usernameVariable: 'username'
                    )]) {
                        sh """
                        # 添加Helm仓库
                        helm repo add --username=${username} --password=${password} devopsrepo https://harbor.zhang-qing.com/chartrepo/demo \
                        """
                    }

                    withCredentials([file(credentialsId: 'kubernetes', variable: 'KUBECONFIG')]) {
                        sh """
                        mkdir -p /root/.kube/ && echo $KUBECONFIG >/root/.kube/config
                        echo "Helm应用配置信息确认..."
                        helm upgrade --install --dry-run --debug ${currentProjectName} --namespace ${namespaces} devopsrepo/demohelm \
                        --set replicaCount=${replicas} \
                        --set image.repository=${deploy_image_name} \
                        --set service.type=ClusterIP \
                        --set springActive=${springProfilesActive} \
                        --set resources.limits.memory=${limiteMem} \
                        --set ingress.enabled=${isIngress}

                        echo "应用部署..."
                        helm upgrade --install ${currentProjectName} --namespace ${namespaces} devopsrepo/demohelm \
                        --set replicaCount=${replicas} \
                        --set image.repository=${deploy_image_name} \
                        --set service.type=ClusterIP \
                        --set springActive=${springProfilesActive} \
                        --set resources.limits.memory=${limiteMem} \
                        --set ingress.enabled=${isIngress}
                        """
                    }
                }
            }
        }
    }
}