使用GitLabCI模板库的流水线优化实践

作业模板
jobs/build.yml 构建作业模板
包含两个作业模板,分别是普通的构建模板(maven/npm/gradle)和docker 镜像构建模板。
## build相关作业.build:stage: buildscript:- |${BUILD_SHELL}variables:GIT_CHECKOUT: "true"rules:- if: " $RUN_PIPELINE_BUILD == 'no' "when: never- when: always## 构建镜像.build-docker:stage: buildimagescript:- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWD $CI_REGISTRY- docker build -t ${IMAGE_NAME} -f ${DOCKER_FILE_PATH} .- docker push ${IMAGE_NAME}- docker rmi ${IMAGE_NAME}rules:- if: " $RUN_BUILD_IMAGE == 'no' "when: never- when: always
jobs/test.yml 测试作业模板
主要用于运行项目单元测试,例如maven、gradle、npm单元测试。
#单元测试.test:stage: testscript:- $TEST_SHELLartifacts:reports:junit: ${JUNIT_REPORT_PATH}rules:- if: " $RUN_PIPELINE_TEST == 'no' "when: never- when: always
jobs/codeanalysis.yml 代码扫描模板
包含两个作业模板,分别为扫描作业和获取扫描结果。这里面将扫描参数进行了分类,通用的项目参数、特殊的合并请求参数、自定义的项目参数。
##代码扫描######.code_analysis:variables:GLOBAL_PROJECT_ARGS: "-Dsonar.projectKey=${CI_PROJECT_NAME}-Dsonar.projectName=${CI_PROJECT_NAME}-Dsonar.projectVersion=${CI_COMMIT_REF_NAME}-Dsonar.projectDescription=${CI_PROJECT_TITLE}"GLOBAL_SERVER_ARGS: "-Dsonar.ws.timeout=30-Dsonar.links.homepage=${CI_PROJECT_URL}-Dsonar.host.url=${SONAR_SERVER_URL}-Dsonar.login=${SONAR_SERVER_LOGIN}-Dsonar.sourceEncoding=UTF-8 "GLOBAL_MR_ARGS: " -Dsonar.pullrequest.key=${CI_MERGE_REQUEST_IID}-Dsonar.pullrequest.branch=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}-Dsonar.pullrequest.base=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}-Dsonar.gitlab.ref_name=${CI_COMMIT_REF_NAME}-Dsonar.gitlab.commit_sha=${CI_COMMIT_SHA}-Dsonar.gitlab.project_id=${CI_PROJECT_PATH}-Dsonar.pullrequest.gitlab.repositorySlug=${CI_PROJECT_ID} "MULTI_BRANCH_ARGS: "-Dsonar.branch.name=${CI_COMMIT_REF_NAME}"stage: code_analysisscript:- echo ${GLOBAL_PROJECT_ARGS} ${GLOBAL_SERVER_ARGS} ${SONAR_SCAN_ARGS} ${GLOBAL_MR_ARGS}#sonar-scanner $GLOBAL_PROJECT_ARGS $GLOBAL_SERVER_ARGS $SCAN_JAVA_ARGS- |if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]thenecho "sonar-scanner ${GLOBAL_PROJECT_ARGS} ${GLOBAL_SERVER_ARGS} ${SONAR_SCAN_ARGS} "sonar-scanner ${GLOBAL_PROJECT_ARGS} ${GLOBAL_SERVER_ARGS} ${SONAR_SCAN_ARGS}elseecho "sonar-scanner ${GLOBAL_PROJECT_ARGS} ${GLOBAL_SERVER_ARGS} ${SONAR_SCAN_ARGS} ${MULTI_BRANCH_ARGS}"sonar-scanner ${GLOBAL_PROJECT_ARGS} ${GLOBAL_SERVER_ARGS} ${SONAR_SCAN_ARGS} ${MULTI_BRANCH_ARGS}firules:- if: " $RUN_CODE_ANALYSIS == 'no' "when: never- when: always#### 获取代码扫描结果.get_analysis_result:stage: get_analysis_resultscript:- |SONAR_REPORT_URL=$(grep "ceTaskUrl" .scannerwork/report-task.txt | awk -F = '{OFS="=";print $2,$3}')echo ${SONAR_REPORT_URL}for i in {1..10}docurl -k -u "${SONAR_SERVER_LOGIN}":"" ${SONAR_REPORT_URL} -o sonar_result.txt -sgrep '"status":"SUCCESS"' sonar_result.txt && SONAR_SCAN_RESULT='SUCCESS'if [ ${SONAR_SCAN_RESULT} == 'SUCCESS' ]thenecho "${SONAR_SCAN_RESULT}"SONAR_SCAN_RESULT=SUCCESScurl -k -u "${SONAR_SERVER_LOGIN}":"" "${SONAR_SERVER_URL}/api/qualitygates/project_status?projectKey=$CI_PROJECT_NAME&branch=$CI_COMMIT_REF_NAME" -o result.txt -secho "result info ---->>>>>"cat result.txtresult=`cat result.txt | awk -F ':' '{print $3}' | awk -F '"' '{print$2}'`echo $resultif [ $result == 'ERROR' ]thenecho "${result}"exit 122break;elseecho "success!"break;fielseSONAR_SCAN_RESULT='ERROR'echo "第$i次获取结果信息,不是成功状态,睡眠10秒!"cat sonar_result.txtsleep 10fidonerules:- if: " $RUN_CODE_ANALYSIS == 'no' "when: never- when: always
jobs/artifactory.yml 制品管理作业
包含两个作业,制品上传与下载。使用artifactory制品库接口。
## 制品库相关##.deploy-artifact:stage: deploy-artifactscript:- echo "curl -u${ARTIFACT_USER}:${ARTIFACT_PASSWD} -T ${ARTIFACT_PATH} $ARTIFACTORY_URL/$ARTIFACTORY_NAME/$TARGET_FILE_PATH/$TARGET_ARTIFACT_NAME"- curl -u${ARTIFACT_USER}:${ARTIFACT_PASSWD} -T ${ARTIFACT_PATH} "$ARTIFACTORY_URL/$ARTIFACTORY_NAME/$TARGET_FILE_PATH/$TARGET_ARTIFACT_NAME"rules:- if: " $RUN_DEPLOY_ARTIFACTS == 'no' "when: never- when: always.down-artifact:stage: down-artifactscript:- curl -u${ARTIFACT_USER}:${ARTIFACT_PASSWD} -O "$ARTIFACTORY_URL/$ARTIFACTORY_NAME/$TARGET_FILE_PATH/$TARGET_ARTIFACT_NAME"- ls
jobs/deploy.yml 发布作业模板
发布基于k8s的发布和回滚配置。
## 应用发布## 使用kubectl镜像发布.deploy_k8s:stage: deployscript:- echo $KUBE_TOKEN- kubectl config set-cluster my-cluster --server=${KUBE_URL} --certificate-authority="${KUBE_CA_PEM_FILE}"- kubectl config set-credentials admin --token=${KUBE_TOKEN}- ls -a- sed -i "s#__namespace__#${NAMESPACE}#g" ${DEPLOY_FILE}- sed -i "s#__appname__#${APP_NAME}#g" ${DEPLOY_FILE}- sed -i "s#__containerport__#${CONTAINER_PORT}#g" ${DEPLOY_FILE}- sed -i "s#__nodeport__#${NODE_PORT}#g" ${DEPLOY_FILE}- sed -i "s#__imagename__#${IMAGE_NAME}#g" ${DEPLOY_FILE}- sed -i "s#__CI_ENVIRONMENT_SLUG__#${CI_ENVIRONMENT_SLUG}#g" ${DEPLOY_FILE}- sed -i "s#__CI_PROJECT_PATH_SLUG__#${CI_PROJECT_PATH_SLUG}#g" ${DEPLOY_FILE}- sed -i "s#__ingressdomain__#${ENV_URL}#g" ${DEPLOY_FILE}- cat ${DEPLOY_FILE}- "kubectl create secret docker-registry ${APP_NAME} \--docker-server=${CI_REGISTRY} \--docker-username=$CI_REGISTRY_USER \--docker-password=${CI_REGISTRY_PASSWD} \--docker-email=test@test.com -n ${NAMESPACE} || echo 'secrets already exists'"- kubectl apply -f ${DEPLOY_FILE}rules:- if: " $RUN_DEPLOY_K8S == 'no'"when: never- when: manualenvironment:name: "${ENV_NAME}"url: "http://${ENV_NAME}.${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME}.devops.com"## 回滚.rollout_k8s:stage: deployscript:- rm -rf $HOME/.kube- kubectl config set-cluster my-cluster --server=${KUBE_URL} --certificate-authority="${KUBE_CA_PEM_FILE}"- kubectl config set-credentials admin --token=${KUBE_TOKEN}- kubectl rollout history deployment ${APP_NAME} -n ${NAMESPACE}- kubectl rollout undo deployment ${APP_NAME} -n ${NAMESPACE}rules:- if: " $RUN_DEPLOY_K8S == 'no' "when: never- when: manualenvironment:name: "${ENV_NAME}"action: stop
default流水线模板
templates/default-pipeline.yml 模板分成个部分
include导入作业模板
variables 定义全局变量
workflow 定义流水线控制
jobs 构建与发布作业
include:- project: 'cidevops/cidevops-newci-service'ref: masterfile: 'jobs/build.yml'- project: 'cidevops/cidevops-newci-service'ref: masterfile: 'jobs/test.yml'- project: 'cidevops/cidevops-newci-service'ref: masterfile: 'jobs/codeanalysis.yml'- project: 'cidevops/cidevops-newci-service'ref: masterfile: 'jobs/deploy.yml'- project: 'cidevops/cidevops-newci-service'ref: masterfile: 'jobs/artifactory.yml'
variables 定义全局变量
variables:## 全局配置GIT_CLONE_PATH: ${CI_BUILDS_DIR}/builds/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/${CI_PIPELINE_ID}GIT_CHECKOUT: "false"CACHE_DIR: ""## 作业控制RUN_PIPELINE_BUILD: "" #是否运行构建 yes/noRUN_PIPELINE_TEST: "" #是否运行测试 yes/noRUN_CODE_ANALYSIS: "" #是否代码扫描 yes/noRUN_BUILD_IMAGE: "" #是否生成镜像 yes/noRUN_DEPLOY_ARTIFACTS: "" #是否上传制品 yes/noRUN_DEPLOY_K8S: "" #是否发布K8S yes/no## 依赖容器镜像BUILD_IMAGE: ""CURL_IMAGE: "curlimages/curl:7.70.0"SONAR_IMAGE: "sonarsource/sonar-scanner-cli:latest"KUBECTL_IMAGE: "lucj/kubectl:1.17.2"## 构建测试参数MAVEN_OPTS: "" #maven构建参数GRADLE_OPTS: "" #gradle构建参数BUILD_SHELL: '' #构建命令## 单元测试参数TEST_SHELL : 'mvn test --settings=./settings.xml ' #测试命令JUNIT_REPORT_PATH: 'target/surefire-reports/TEST-*.xml' #单元测试报告## 代码扫描SONAR_SOURCE_DIR : "src" #项目源码目录SONAR_SERVER_URL: "http://192.168.1.200:30090" #SonarQube服务器信息SONAR_SERVER_LOGIN: "ee2bcb37deeb6dfe3a07fe08fb529559b00c1b7b" #Sonar Token最好在项目中定义。SONAR_SCAN_ARGS: "-Dsonar.sources=${SONAR_SOURCE_DIR}" #项目扫描参数## 构建镜像CI_REGISTRY: 'registry.cn-beijing.aliyuncs.com' #镜像仓库地址CI_REGISTRY_USER: 'xxxxxx' #仓库用户信息#CI_REGISTRY_PASSWD: 'xxxxxxxx.' #仓库用户密码IMAGE_NAME: "${CI_REGISTRY}/${CI_PROJECT_PATH}:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" #镜像名称DOCKER_FILE_PATH: "./Dockerfile" #Dockerfile位置## 上传制品库(artifactory)ARTIFACTORY_URL: "http://192.168.1.200:30082/artifactory" #制品库地址ARTIFACTORY_NAME: "${CI_PROJECT_NAMESPACE}" #制品库名称ARTIFACT_PACKAGE: "jar" #制品类型ARTIFACT_PATH: "target/*.${ARTIFACT_PACKAGE}" #制品位置TARGET_FILE_PATH: "${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" #目标制品位置(目录结构)TARGET_ARTIFACT_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.${ARTIFACT_PACKAGE}" #目标制品名称## 部署应用k8sAPP_NAME: "$CI_PROJECT_NAME" #应用名称 <--> deploymentNameCONTAINER_PORT: "8081" #服务端口 <--> servicesPortNAMESPACE: "$CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG" #名称空间ENV_URL: "${ENV_NAME}.${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME}.devops.com" #IngressHosts
流水线设置
## 流水线控制workflow:rules:- if: "$CI_MERGE_REQUEST_ID" #过滤合并请求when: never- if: "$CI_PIPELINE_SOURCE == 'web'" #允许在web页面发布- if: "$CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ " #过滤版本分支和主干分支提交代码触发when: never- if: "$CI_COMMIT_BEFORE_SHA == '0000000000000000000000000000000000000000'" #过滤分支创建请求when: never### 默认策略- when: always## 运行阶段stages:- build- test- parallel01- get_analysis_result- deploy-artifact- deploy-feature- rollout-feature- deploy-uat- rollout-uat- deploy-stag- rollout-stag- deploy-prod- rollout-prodcache:paths:- ${CACHE_DIR}before_script:- export
构建作业配置
################# Jobs Configure ####################### 构建作业build:image: ${BUILD_IMAGE}extends: .build## 测试作业test:image: ${BUILD_IMAGE}extends: .testbefore_script:lsls ${CACHE_DIR}## 代码扫描code_analysis:stage: parallel01image: ${SONAR_IMAGE}extends: .code_analysis## 获取构建结果get_analysis_result:image: ${CURL_IMAGE}extends: .get_analysis_resultneeds:code_analysis## 构建镜像build_image:image: docker:latestservices:name: docker:dindstage: parallel01extends: .build-docker## 上传制品deploy_artifact:image: ${CURL_IMAGE}stage: deploy-artifactextends: .deploy-artifact## 下载制品#down_artifact:# image: ${CURL_IMAGE}# stage: down_artifact# extends: .down-artifact
发布部署作业配置
#################Deploy Feature Jobs Configure ####################### feature发布应用deploy_feature:variables:DEPLOY_FILE: 'deployment.yaml'ENV_NAME: 'feature'stage: deploy-featureimage: ${KUBECTL_IMAGE}extends: .deploy_k8senvironment:on_stop: "rollout_feature"rules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: never- when: manual## 应用回滚rollout_feature:variables:DEPLOY_FILE: 'deployment.yaml'ENV_NAME: 'feature'stage: rollout-featureimage: ${KUBECTL_IMAGE}extends: .rollout_k8sneeds:- deploy_featurerules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: never- when: on_failure#################Deploy UAT Jobs Configure ####################### UATdeploy_uat:variables:DEPLOY_FILE: 'config/deployment-uat.yaml'ENV_NAME: 'uat'stage: deploy-uatimage: ${KUBECTL_IMAGE}extends: .deploy_k8senvironment:on_stop: "rollout_uat"rules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: manual- when: never## UAT应用回滚rollout_uat:variables:DEPLOY_FILE: 'config/deployment-uat.yaml'ENV_NAME: 'uat'stage: rollout-uatimage: ${KUBECTL_IMAGE}extends: .rollout_k8sneeds:- deploy_uatrules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: on_failure- when: never#################Deploy STAG Jobs Configure ####################### STAGdeploy_stag:variables:DEPLOY_FILE: 'config/deployment-stag.yaml'ENV_NAME: 'stag'stage: deploy-stagimage: ${KUBECTL_IMAGE}extends: .deploy_k8senvironment:on_stop: "rollout_stag"needs:- deploy_uatrules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: manual- when: never## STAG应用回滚rollout_stag:variables:DEPLOY_FILE: 'config/deployment-stag.yaml'ENV_NAME: 'stag'stage: rollout-stagimage: ${KUBECTL_IMAGE}extends: .rollout_k8sneeds:- deploy_stagrules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: on_failure- when: never#################Deploy PROD Jobs Configure ####################### PRODdeploy_prod:variables:DEPLOY_FILE: 'config/deployment-prod.yaml'ENV_NAME: 'prod'stage: deploy-prodimage: ${KUBECTL_IMAGE}extends: .deploy_k8senvironment:on_stop: "rollout_prod"needs:- deploy_stagrules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: manual- when: never## PROD应用回滚rollout_prod:variables:DEPLOY_FILE: 'config/deployment-prod.yaml'ENV_NAME: 'prod'stage: rollout-prodimage: ${KUBECTL_IMAGE}extends: .rollout_k8sneeds:- deploy_prodrules:- if: $RUN_DEPLOY_K8S == 'no'when: never- if: $CI_COMMIT_REF_NAME =~ /^RELEASE-*/ || $CI_COMMIT_REF_NAME =~ /master/ || $CI_COMMIT_TAGwhen: on_failure- when: never
JAVA项目交付流水线
导入作业模板
include:- project: 'cidevops/cidevops-newci-service'ref: masterfile: 'templates/default-pipeline.yml'
配置项目参数
variables:## 全局配置GIT_CLONE_PATH: ${CI_BUILDS_DIR}/builds/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/${CI_PIPELINE_ID}GIT_CHECKOUT: "false"CACHE_DIR: "target/"## 作业控制RUN_PIPELINE_BUILD: "yes" #是否运行构建 yes/noRUN_PIPELINE_TEST: "yes" #是否运行测试 yes/noRUN_CODE_ANALYSIS: "yes" #是否代码扫描 yes/noRUN_BUILD_IMAGE: "yes" #是否生成镜像 yes/noRUN_DEPLOY_ARTIFACTS: "no" #是否上传制品 yes/noRUN_DEPLOY_K8S: "yes" #是否发布K8S yes/no## 依赖容器镜像BUILD_IMAGE: "maven:3.6.3-jdk-8"CURL_IMAGE: "curlimages/curl:7.70.0"SONAR_IMAGE: "sonarsource/sonar-scanner-cli:latest"KUBECTL_IMAGE: "lucj/kubectl:1.17.2"## 构建测试参数MAVEN_OPTS: "-Dmaven.repo.local=/home/gitlab-runner/ci-build-cache/maven " #maven构建参数BUILD_SHELL: 'mvn clean package -DskipTests --settings=./settings.xml ' #构建命令#GRADLE_OPTS: "" #gradle构建参数## 单元测试参数TEST_SHELL : 'mvn test --settings=./settings.xml ' #测试命令JUNIT_REPORT_PATH: 'target/surefire-reports/TEST-*.xml' #单元测试报告## 代码扫描SONAR_SOURCE_DIR : "src" #项目源码目录SONAR_SERVER_URL: "http://192.168.1.200:30090" #SonarQube服务器信息SONAR_SERVER_LOGIN: "ee2bcb37deeb6dfe3a07fe08fb529559b00c1b7b" #Sonar Token最好在项目中定义。SONAR_SCAN_ARGS: "-Dsonar.sources=${SONAR_SOURCE_DIR}-Dsonar.java.binaries=target/classes-Dsonar.java.test.binaries=target/test-classes-Dsonar.java.surefire.report=target/surefire-reports " #项目扫描参数## 构建镜像CI_REGISTRY: 'registry.cn-beijing.aliyuncs.com' #镜像仓库地址CI_REGISTRY_USER: 'xxxxxx' #仓库用户信息#CI_REGISTRY_PASSWD: 'xxxxxxxx.' #仓库用户密码IMAGE_NAME: "${CI_REGISTRY}/${CI_PROJECT_PATH}:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" #镜像名称DOCKER_FILE_PATH: "./Dockerfile" #Dockerfile位置## 上传制品库(artifactory)#ARTIFACTORY_URL: "http://192.168.1.200:30082/artifactory" #制品库地址#ARTIFACTORY_NAME: "${CI_PROJECT_NAMESPACE}" #制品库名称#ARTIFACT_PACKAGE: "jar" #制品类型#ARTIFACT_PATH: "target/*.${ARTIFACT_PACKAGE}" #制品位置#TARGET_FILE_PATH: "${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" #目标制品位置(目录结构)#TARGET_ARTIFACT_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.${ARTIFACT_PACKAGE}" #目标制品名称## 部署应用k8sAPP_NAME: "$CI_PROJECT_NAME" #应用名称 <-->deploymentNameCONTAINER_PORT: "8081" #服务端口 <--> servicesPortNAMESPACE: "$CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG" #名称空间ENV_URL: "${ENV_NAME}.${CI_PROJECT_NAMESPACE}.${CI_PROJECT_NAME}.devops.com" #IngressHosts
指定CI文件

运行流水线测试

docker镜像仓库

部署环境演示


Kubernetes集群中应用状态

评论
