From 4ad07be353b5329b919e41c2ecb7c27bf87d72a9 Mon Sep 17 00:00:00 2001
From: Quentin Duchemin <quentinduchemin@tuta.io>
Date: Wed, 29 Jan 2020 23:11:15 +0100
Subject: [PATCH] [CI] Remove CI-build images - less storage, more performance
Until now, a unique image was built and pushed for each commit, tagged with the commit number.
If this has the avantage to identify which commit corresponds to which image, it causes other problems.
For example, if the pipeline fails because of an approved CVE, you may just want to whitelist the CVE.
Problem : you will need to rebuild the entire image even if nothing has changed, because the "clair" stage tries to pull the image with the current commit id in tag.
So the following was done :
- As soon as an image in successfully built, push it to the registry. OVERWRITE THE OLD IMAGE. This is ok.
- Now, you rebuild the image ONLY if the Dockerfile has changed.
- You run Clair if the clair-whitelist.yml (or Dockerfile, docker-compose) has changed. "clair" stage will be able to find the last previously built image. This is good, no need to build again ! Faster.
- Remove the push-test stage : already done earlier.
---
.gitlab-ci.yml | 34 ++++++++--------------------------
1 file changed, 8 insertions(+), 26 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3a24350d..a00f570e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -27,7 +27,7 @@ stages:
before_script:
- echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
- source variables
- - docker pull $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+ - docker pull $MODIFIED_IMAGE_FULL_TEST
# Build the base image used for all further steps : this is done only when pica-ci's Dockerfile is modified
pica-ci-base:
@@ -74,14 +74,13 @@ build:
- echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
- source variables
script:
- - docker build -f $MODIFIED_IMAGE/Dockerfile $MODIFIED_IMAGE -t $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
- - docker push $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+ - docker build -f $MODIFIED_IMAGE/Dockerfile $MODIFIED_IMAGE -t $MODIFIED_IMAGE_FULL_TEST
+ - docker push $MODIFIED_IMAGE_FULL_TEST
after_script:
- docker logout $REGISTRY
only:
changes:
- "**/Dockerfile"
- - "**/docker-compose.yml"
refs:
- master
- dev-ci
@@ -97,7 +96,7 @@ clair:
- mv clair-scanner_linux_amd64 clair-scanner
- chmod +x clair-scanner
- while( ! wget -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; done
- - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+ - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $MODIFIED_IMAGE_FULL_TEST
artifacts:
paths:
- clair-report.json
@@ -106,6 +105,7 @@ clair:
changes:
- "**/Dockerfile"
- "**/docker-compose.yml"
+ - "**/clair-whitelist.yml"
refs:
- master
- dev-ci
@@ -116,7 +116,7 @@ docker-bench-security:
<<: *pull-modified-image
script:
# Change the Docker Compose to use the "testing" image, not yet pushed on production registry
- - sed -i -e "s|$MODIFIED_IMAGE_FULL|$REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA|g" $MODIFIED_IMAGE/docker-compose.yml
+ - sed -i -e "s|$MODIFIED_IMAGE_FULL|$MODIFIED_IMAGE_FULL_TEST|g" $MODIFIED_IMAGE/docker-compose.yml
# If *.example secrets files exist, remove the .example extension to be able to start the container
# Indeed these file are used in Docker Compose with env_file directive
- if [[ -d $MODIFIED_IMAGE/secrets ]]; then for i in $MODIFIED_IMAGE/secrets/*.example ; do cp $i $(echo $i| cut -d '.' -f1,2); done; fi;
@@ -127,7 +127,7 @@ docker-bench-security:
- docker-compose up -d
- git clone https://github.com/docker/docker-bench-security.git
- cd docker-bench-security
- - sh docker-bench-security.sh -c container_images,container_runtime,docker_security_operations -e check_5_12,check_4_1 -l ../../report.txt
+ - sh docker-bench-security.sh -c container_images,container_runtime,docker_security_operations,community_checks -l ../../report.txt
artifacts:
paths:
- report.txt
@@ -141,24 +141,6 @@ docker-bench-security:
- master
- dev-ci
-push-test:
- stage: push
- <<: *pull-modified-image
- script:
- - docker tag $REGISTRY_TEST/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA $MODIFIED_IMAGE_FULL_TEST
- - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
- # MODIFIED_IMAGE_FULL_TEST already should include the registry URL
- - docker push $MODIFIED_IMAGE_FULL_TEST
- after_script:
- - docker logout $REGISTRY
- only:
- changes:
- - "**/Dockerfile"
- - "**/docker-compose.yml"
- refs:
- - master
- - dev-ci
-
# Push the generated image on the production registry,
# once it passed all security tests and has been successfully built
# and run on the test virtual machine
@@ -166,7 +148,7 @@ push-prod:
stage: push
<<: *pull-modified-image
script:
- - docker tag $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA $MODIFIED_IMAGE_FULL
+ - docker tag $MODIFIED_IMAGE_FULL_TEST $MODIFIED_IMAGE_FULL
- echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
# MODIFIED_IMAGE_FULL already should include the registry URL
- docker push $MODIFIED_IMAGE_FULL
--
GitLab