CI/CD初步
现在利用已有的k8s环境部署一套CI/CD环境
DevOps
理解CI/CD，持续集成和持续交付的基本过程包括如下几个步骤：
- 软件更新或者迭代——Gitlab
- 新版软件打包成镜像——Jenkins
- 新的镜像在k8s中集成——Registry
当代码提交到gitlab之后，会立马触发jenkins将新代码编译成镜像，然后再在kmaster上部署新的镜像。
镜像仓库
Registry部署
# 部署docker并修改docker启动参数
[root@Gitlab ~]# vim /usr/lib/systemd/system/docker.service
# 新增--insecure-registry=192.168.10.9:5000 -H tcp://0.0.0.0:2376
ExecStart=/usr/bin/dockerd --insecure-registry=192.168.10.9:5000 -H tcp://0.0.0.0:2376 -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
# 重新加载docker服务
[root@Gitlab ~]# systemctl daemon-reload
[root@Gitlab ~]# systemctl restart docker
[root@Gitlab ~]# docker pull registry
Using default tag: latest
latest: Pulling from library/registry
Status: Downloaded newer image for registry:latest
docker.io/library/registry:latest
[root@Gitlab ~]# docker pull nginx
Using default tag: latest
latest: Pulling from library/nginx
Status: Image is up to date for nginx:latest
docker.io/library/nginx:latest
# 创建镜像存储空间
[root@Gitlab ~]# mkdir /data/registry
[root@Gitlab ~]# pvcreate /dev/sdb
Physical volume "/dev/sdb" successfully created.
[root@Gitlab ~]# vgcreate vg_data /dev/sdb
Volume group "vg_data" successfully created
[root@Gitlab ~]# lvcreate -n lv_data vg_data -l 100%free
Logical volume "lv_data" created.
[root@Gitlab ~]# mkfs.xfs /dev/mapper/vg_data-lv_data
meta-data=/dev/mapper/vg_data-lv_data isize=512 agcount=4, agsize=655104 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=0, sparse=0
data = bsize=4096 blocks=2620416, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0 ftype=1
log =internal log bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
# 新增mount挂点
[root@Gitlab ~]# vim /etc/fstab
/dev/mapper/vg_data-lv_data /data xfs defaults 0 0
[root@Gitlab ~]# mount -a
# 建立registry容器,映射端口5000
[root@Gitlab ~]# docker run -d --name registry -p 5000:5000 --restart=always -v /data/registry:/var/lib/registry registry
d6af24382a2e05583c50faf566f9411474666f81d923102e9ac38d8b38cb30e
[root@Gitlab ~]# netstat -tlnp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:5000 0.0.0.0:* LISTEN 2197/docker-proxy
tcp6 0 0 :::5000 :::* LISTEN 2203/docker-proxy
# 同时在kubernetes群集上完成docker配置文件的修改，即将10.9主机设置为镜像下载来源和dockerca认证源。Harbor部署
# 部署2core4GB主机
# 创建/data目录，并映射独立20GB磁盘空间
[root@harbor harbor]# df -Th
Filesystem Type Size Used Avail Use% Mounted on
/dev/mapper/centos-root xfs 17G 8.4G 8.7G 50% /
/dev/mapper/vg_data-lv_data xfs 20G 33M 20G 1% /data
# 部署Harbor
[root@harbor ~]# tar zxvf harbor-offline-installer-v2.7.3.tgz
harbor/harbor.v2.7.3.tar.gz
harbor/prepare
harbor/LICENSE
harbor/install.sh
harbor/common.sh
harbor/harbor.yml.tmpl
[root@Harbor ~]# mv harbor /opt/
[root@Harbor ~]# cd /opt/harbor
[root@harbor harbor]# docker load -i harbor.v2.7.3.tar.gz
[root@harbor harbor]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
goharbor/harbor-exporter v2.7.3 44f17702b0d6 3 weeks ago 96.9MB
goharbor/chartmuseum-photon v2.7.3 e21f928bea75 3 weeks ago 229MB
goharbor/redis-photon v2.7.3 68ef52d98298 3 weeks ago 120MB
goharbor/trivy-adapter-photon v2.7.3 aabf279df9cc 3 weeks ago 463MB
goharbor/notary-server-photon v2.7.3 992cbac9892b 3 weeks ago 113MB
goharbor/notary-signer-photon v2.7.3 e384f965170c 3 weeks ago 110MB
goharbor/harbor-registryctl v2.7.3 0adcdbbc67c8 3 weeks ago 140MB
goharbor/registry-photon v2.7.3 91fa7c3c922c 3 weeks ago 78.7MB
goharbor/nginx-photon v2.7.3 a780e583d37f 3 weeks ago 116MB
goharbor/harbor-log v2.7.3 48a9ddf4a380 3 weeks ago 128MB
goharbor/harbor-jobservice v2.7.3 265eda6d72aa 3 weeks ago 260MB
goharbor/harbor-core v2.7.3 1a415c050c9c 3 weeks ago 222MB
goharbor/harbor-portal v2.7.3 9a0f808a9eed 3 weeks ago 125MB
goharbor/harbor-db v2.7.3 731c8c0fe6ca 3 weeks ago 174MB
goharbor/prepare v2.7.3 36fd5b190502 3 weeks ago 168MB
[root@harbor harbor]# cp harbor.yml.tmpl harbor.yml
# 注释掉443端口和修改主机名
[root@harbor harbor]# vim harbor.yml
# 环境准备
[root@harbor harbor]# ./prepare
prepare base dir is set to /opt/harbor
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
Generated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
# 部署Harbor
[root@harbor harbor]# ./install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 24.0.6
[Step 1]: checking docker-compose is installed ...
Note: Docker Compose version v2.21.0
[Step 2]: loading Harbor images ...
[Step 5]: starting Harbor ...
[+] Running 10/10
✔ Network harbor_harbor Created
✔ Container harbor-log Started
✔ Container registry Started
✔ Container registryctl Started
✔ Container harbor-portal Started
✔ Container redis Started
✔ Container harbor-db Started
✔ Container harbor-core Started
✔ Container nginx Started
✔ Container harbor-jobservice Started
✔ ----Harbor has been installed and started successfully.----
# 修改docker的daemon配置文件
[root@harbor harbor]# vim /etc/docker/daemon.json
{
"registry-mirrors": ["https://37y8py0j.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"insecure-registries":["192.168.10.8"]
}
[root@harbor harbor]# systemctl daemon-reload
[root@harbor harbor]# systemctl restart docker
# 拉起Harbor服务
[root@harbor harbor]# docker-compose up -d
Recreating harbor-log ... done
Recreating harbor-portal ...
Recreating registryctl ...
Recreating redis ...
Recreating registry ...
Recreating registry ... done
Recreating harbor-core ... done
Recreating harbor-jobservice ...
Recreating nginx ... done
# 设置Harbor服务的systemd启动脚本
[root@harbor harbor]# cat /etc/systemd/system/harbor.service
[Unit]
Description=Harbor Image Service
After=docker.service systemd-networkd.service systemd-resolved.service
Requires=docker.service
Documentation=http://github.com/vmware/harbor
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/bash -c "docker-compose -f /opt/harbor/docker-compose.yml up "
ExecStart=/bin/bash -c "docker-compose -f /opt/harbor/docker-compose.yml stop"
[Install]
WantedBy=multi-user.target
[root@harbor harbor]# systemctl daemon-reload
# 设置Harbor的自启动
[root@harbor harbor]# systemctl enable --now harbor.service
Created symlink from /etc/systemd/system/multi-user.target.wants/harbor.service to /etc/systemd/system/harbor.service.
# 配置/etc/hosts的解析
[root@harbor harbor]# cat /etc/hosts
192.168.10.8 reg.sujx.net
# 登陆Harbor
[root@harbor harbor]# docker login 192.168.10.8
Username: sujx
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
# 推送镜像
[root@harbor harbor]# docker tag nginx:latest 192.168.10.8/library/nignx:v1.24
[root@harbor harbor]# docker push 192.168.10.8/library/nignx:v1.24
The push refers to repository [192.168.10.8/library/nignx]
d874fd2bc83b: Pushed
32ce5f6a5106: Pushed
f1db227348d0: Pushed
b8d6e692a25e: Pushed
e379e8aedd4d: Pushed
2edcec3590a4: Pushed
v1.24: digest: sha256:ee89b00528ff4f02f2405e4ee221743ebc3f8e8dd0bfd5c4c20a2fa2aaa7ede3 size: 1570- 登陆Harbor
- 创建用户
- 查看上传镜像
代码管理
部署Gitlab
# 单独部署gitlab需要4G内存，整合部署需要8G内存
# 获取gitlab-ce
[root@Gitlab ~]# docker pull gitlab/gitlab-ce
# 创建数据目录
[root@Gitlab ~]# mkdir -pv /data/gitlab/{etc,log,data}
[root@Gitlab ~]# tree /data/gitlab/
/data/gitlab/
├── data
├── etc
└── log
3 directories, 0 files
# 建立容器
[root@Gitlab ~]# docker run -dit --name=gitlab --restart=always -p 443:443 -p 80:80 -p 2022:22 -v /data/gitlab/etc:/etc/gitlab -v /data/gitlab/log:/var/log/gitlab -v /data/gitlab/data:/var/opt/gitlab --privileged=true gitlab/gitlab-ce
3507cf28a460c8448efa77f784f02ab7585fbf71f01e628fe11d4988625bbaf1
# 修改容器时区
[root@Gitlab ~]# docker exec -it gitlab sh -c "ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime"
# 修改gitlab配置文件
# 设定gitlab使用https访问
# 设定gitlab的时区为上海
[root@Gitlab ~]# docker stop gitlab
[root@Gitlab ~]# vim /data/gitlab/etc/gitlab.rb
# 配置访问地址
external_url 'https://git.sujx.net'
# 内部ssh地址
gitlab_rails['gitlab_ssh_host'] = 'git.sujx.net'
# 配置时区
gitlab_rails['time_zone'] = 'Asia/Shanghai'
# 配置ssh端口号，因为宿主机还要使用22端口，所以使用2022端口
gitlab_rails['gitlab_shell_ssh_port'] = 2022
gitlab_rails['gitlab_shell_git_timeout'] = 800
# 配置Nginx开启https服务
nginx['enable'] = true
nginx['client_max_body_size'] = '250m'
nginx['redirect_http_to_https'] = true
nginx['redirect_http_to_https_port'] = 80
# 放置SSl证书，这个路径是Docker内部看到的路径
nginx['ssl_certificate'] = "/etc/gitlab/ssl/git.sujx.net.pem"
nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/git.sujx.net.key"
#将对应的证书放入外部被映射到docker的路径下
[root@gitlab ~]# mkdir /data/gitlab/etc/ssl
[root@gitlab ~]# ls /data/gitlab/etc/ssl
git.sujx.net.key git.sujx.net.pem
# 重启gitlab容器
[root@Gitlab ~]# docker start gitlab
# 查看gitlab的root初始密码
[root@Gitlab ~]# docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password
Password: KPcQ2ei7K4cTfQFsJAE5kU05+j5dBi7TcTV1elGWMLE=登陆Gitlab
- gitlab首页
- 项目新建
- 新建P1项目
- 克隆项目
部署测试
[root@Gitlab ~]# yum install git
Loaded plugins: fastestmirror, versionlock
Loading mirror speeds from cached hostfile
Package git-1.8.3.1-25.el7_9.x86_64 already installed and latest version
Nothing to do
[root@Gitlab ~]# git clone http://192.168.10.9/root/p1.git
Cloning into 'p1'...
remote: Enumerating objects: 3, done.
remote: Counting objects: 100% (3/3), done.
remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 0
Unpacking objects: 100% (3/3), done.
[root@Gitlab ~]# cd p1/
[root@Gitlab p1]# git config --global user.name "sujx"
[root@Gitlab p1]# git config --global user.email sujx@live.cn
[root@Gitlab p1]# git config --global push.default simple
[root@Gitlab p1]# echo 1111 > index.html
[root@Gitlab p1]# git add .
[root@Gitlab p1]# git commit -m 111
[main 45a4e3b] 111
1 file changed, 1 insertion(+)
create mode 100644 index.html
[root@Gitlab p1]# git push
Username for 'http://192.168.10.9': root
Password for 'http://root@192.168.10.9':
Counting objects: 4, done.
Delta compression using up to 2 threads.
Compressing objects: 100% (2/2), done.
Writing objects: 100% (3/3), 266 bytes | 0 bytes/s, done.
Total 3 (delta 0), reused 0 (delta 0)
To http://192.168.10.9/root/p1.git
1c92ec5..45a4e3b main -> main- 验证结果
部署管理
部署Jenkins
# 下载jenkins镜像
[root@Jenkins ~]# docker pull jenkins/jenkins
Using default tag: latest
latest: Pulling from jenkins/jenkins
Digest: sha256:c3fa8e7f70d1e873ea6aa87040c557aa53e6707eb1d5ecace7f6884a87588ac8
Status: Image is up to date for jenkins/jenkins:latest
docker.io/jenkins/jenkins:latest
# 创建数据目录，并赋权 uid和gid为1000
[root@Jenkins ~]# mkdir /data/jenkins ; chown 1000:1000 /data/jenkins
# 创建容器
[root@Jenkins ~]# docker run -dit -p 8080:8080 -p 50000:50000 --name jenkins --privileged=true --restart=always -v /data/jenkins:/var/jenkins_home jenkins/jenkins
061b8fc6a51351451cb4b764a52fcf496d75ce877a816b822e7fa40c7bd8438a
# 查看Jenkins密码
[root@Jenkins ~]# docker exec -it jenkins cat /var/jenkins_home/secrets/initialAdminPassword
3517870dedb14ab99614ad7150dba69e
# 配置容器时区为上海
[root@Jenkins ~]# docker exec -it -u root jenkins sh -c "ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime"
# 停止Jenkins
[root@Jenkins ~]# docker stop jenkins
jenkins
# 修改jenkins的仓库地址,避免安装时的离线实例提示
[root@Jenkins ~]# sed -i "s@https://updates.jenkins.io/@https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/@" /data/jenkins/hudson.model.UpdateCenter.xml
[root@Jenkins ~]# sed -i "s@https://www.google.com@https://www.baidu.com/@" /data/jenkins/updates/default.json
# 升级Jenkins
[root@Jenkins ~]# wget https://updates.jenkins.io/download/war/2.427/jenkins.war
[root@Jenkins ~]# docker cp jenkins.war jenkins:/usr/share/jenkins/初始化环境
初始登陆界面(可忽略插件安装失败)
升级和安装插件(安装Docker、docker-build-step、Generic Webhook Trigger)
重启Jenkins(使用URL/restart重启，使用URL/stop停止)
配置docker，新增cloud
[root@devops ~]# vim /usr/lib/systemd/system/docker.service # 将docker的监听方式由unix socket修改为TCP # 修改配置文件 ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock 修改为： ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H fd:// --containerd=/run/containerd/containerd.sock
与Gitlab联动
- Jenkins上安装generic-webhook-trigger、Docker、docker-build-step 插件
- 在Jenkins上创建账号的Token
- 使用root账号登陆gitlab进行配置
实际案例
游戏代码
俄罗斯方块这是一个经典的小游戏，现在我们使用这个小游戏来演示CI/CD的实现过程。
# 获取俄罗斯方块Javascript版源码
[sujx@Dev ~]$ git clone https://github.com/LeeYiyuan/tetrisai.git
Cloning into 'tetrisai'...
remote: Enumerating objects: 280, done.
remote: Counting objects: 100% (8/8), done.
remote: Compressing objects: 100% (8/8), done.
remote: Total 280 (delta 0), reused 2 (delta 0), pack-reused 272
Receiving objects: 100% (280/280), 54.71 KiB | 40.00 KiB/s, done.
Resolving deltas: 100% (139/139), done.
# 删除git信息
[sujx@Dev ~]$ rm -rf tetrisai/.git/
[sujx@Dev ~]$ cd tetrisai/
[sujx@Dev tetrisai]$ tar zcvf ~/build/tetris.tar.gz ./
# 创建Docker镜像构建目录
[sujx@Dev tetrisai]$ mkdir ~/build
[sujx@Dev tetrisai]$ cd ~/build
# 编辑Dockerfile创建spaceinvaders:v1镜像
[sujx@Dev build]$ cat > Dockerfile <<EOF
FROM nginx:latest
MAINTAINER sujx@live.cn
ADD tetris.tar.gz /usr/share/nginx/html
EOF
# 构建镜像
[sujx@Dev build]$ docker build -t tetris:v1 .
[+] Building 0.1s (7/7) FINISHED
……
=> => naming to docker.io/library/tetris:v1
# 运行测试容器镜像OK
[sujx@Dev build]$ docker run -itd --name tetris -p 80:80 tetris:v1
f6cc0a5f33caa89402e95a9d51c0b417208c7f73965b703113f8db020718ea39
# 重新打标签，准备推送到内网镜像库
[sujx@Dev ~]$ docker tag tetris:v1 harbor.sujx.net/sujx/tetris:v1
[sujx@Dev ~]$ docker login harbor.sujx.net
Username: sujx
Password:
Login Succeeded
# 推送成功
[sujx@Dev ~]$ docker push harbor.sujx.net/sujx/tetris:v1
The push refers to repository [harbor.sujx.net/sujx/tetris]
ebe1bde802c1: Pushed
d874fd2bc83b: Pushed
32ce5f6a5106: Pushed
f1db227348d0: Pushed
b8d6e692a25e: Pushed
e379e8aedd4d: Pushed
2edcec3590a4: Pushed
v1: digest: sha256:5607cb3d8d89803f0c1cbab72d74168c801bc202d5a545b0420f0afbf9739512 size: 1778上传代码
在前面搭建的gitlab上面，注册sujx的账号，并新建tetris项目。
# 创建用户使用的公钥,并将内容粘贴到gitlab-ce中
[sujx@Dev ~]$ ssh-keygen -t rsa -b 2048 -C "sujx@live.cn"
# 测试免密登陆，使用-p参数指定gitlab-ce的2022端口
[sujx@Dev ~]$ ssh -T git@git.sujx.net -p 2022
Welcome to GitLab, @sujx!
# 创建gitlab用户信息
[sujx@Dev ~]$ git config --global user.name "sujx"
[sujx@Dev ~]$ git config --global user.email sujx@live.cn
[sujx@Dev ~]$ git config --global push.default simple
# 免密克隆项目，这里使用ssh路径，而非前述的https
[sujx@Dev ~]$ git clone ssh://git@git.sujx.net:2022/sujx/tetris.git
Cloning into 'tetris'...
# 将从github上下载的代码复制到tetris目录中
[sujx@Dev ~]$ cd ~/tetris
[sujx@Dev tetris]$ cp ~/tetrisai ./
[sujx@Dev tetris]$ git add .
[sujx@Dev tetris]$ git commit -m 'init game code'
[main 41d1eef] init game code
13 files changed, 1329 insertions(+), 2 deletions(-)
create mode 100644 License.md
create mode 100644 index.html
create mode 100644 js/ai.js
create mode 100644 js/game_manager.js
create mode 100644 js/grid.js
create mode 100644 js/piece.js
create mode 100644 js/polyfill.js
create mode 100644 js/random_piece_generator.js
create mode 100644 js/stopwatch.js
create mode 100644 js/timer.js
create mode 100644 js/tuner.js
create mode 100644 style/main.css
# 实现免密推送
[sujx@Dev tetris]$ git push
Enumerating objects: 7, done.
Counting objects: 100% (7/7), done.
Compressing objects: 100% (4/4), done.
Writing objects: 100% (4/4), 441 bytes | 441.00 KiB/s, done.
Total 4 (delta 2), reused 0 (delta 0), pack-reused 0
To ssh://git.sujx.net:2022/sujx/tetris.git
41d1eef..92aa7b5 main -> main自动化构建
使用Token连接Jenkins和Gitlab
连接测试
设置测试job,开启webhook trigger
设置任务
测试自动执行
# 在开发机上新增文件a [sujx@Dev tetris]$ touch a [sujx@Dev tetris]$ git add . [sujx@Dev tetris]$ git commit -m "add a" [main cbfba5b] add a 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 a [sujx@Dev tetris]$ git push Enumerating objects: 4, done. Counting objects: 100% (4/4), done. Compressing objects: 100% (2/2), done. Writing objects: 100% (3/3), 244 bytes | 244.00 KiB/s, done. Total 3 (delta 1), reused 0 (delta 0), pack-reused 0 To ssh://git.sujx.net:2022/sujx/tetris.git 1775d7f..cbfba5b main -> main # 在Jenkins主机上查看结果 [root@Jenkins ~]# docker exec -it jenkins cat /tmp/a HelloWorld!
配置Jenkins任务，添加执行shell和docker构建项
cd /var/jenkins_home/tetris git clone https://git.sujx.net/sujx/tetris.git cd tetris tar zcf ../tetris.tar.gz ./
执行任务
查看镜像库，除了前述上传的v1版本外，新增了7版本
发布到Kubernets
再次到Jenkins中新增构建步骤，添加执行shell
export KUBECONFIG=/kc1 /kubectl set image deployment/web1 tetris="harbor.sujx.net/sujx/tetris:${BUILD_NUMBER}" -n nscicd
All articles in this blog are licensed under CC BY-NC-SA 4.0 unless stating additionally.