Remove intermediate pica-ci image using CI cache, various improvements

.gitlab-ci.yml

@@ -2,182 +2,180 @@ image: docker:19.03.0
 
 # Disable TLS just for the docker daemon running locally, TLS is still used to deploy built images!
 variables:
-    DOCKER_TLS_CERTDIR: ""
-    DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_DRIVER: overlay2
 
 services:
     - docker:19.03.0-dind
 
+# The file variables will be kept accross jobs : it contains useful informations about modified
+# Dockerfile / docker-compose in the last commit
+cache:
+ paths:
+  - variables
+
 stages:
-    - pica-ci-base
-    - metabuild
-    - build
-    - static_tests
-    - dynamic_tests
-    - push-prod
+  - ci-base
+  - build
+  - security-tests
+  - push
+  - clean
+
+# Hidden key meant to be included in other jobs, for factorization
+.pull-modified-image: &pull-modified-image
+  image: $REGISTRY_PROD/pica-ci-base
+  tags: [build]
+  before_script:
+    - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
+    - docker pull $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+    - source variables
 
 # Build the base image used for all further steps : this is done only when pica-ci's Dockerfile is modified
 pica-ci-base:
-    stage: pica-ci-base
-    before_script:
-        - echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
-    script:
-        - docker build -f pica-ci-base/Dockerfile . -t $REGISTRY_PROD/pica-ci-base:latest
-        - docker push $REGISTRY_PROD/pica-ci-base:latest
-    after_script:
-        - docker logout $REGISTRY_PROD
-    tags: [build]
-    only:
-        changes:
-            - "pica-ci-base/*"
-        refs:
-            - master
-            - dev-ci
+  stage: ci-base
+  tags: [build]
+  script:
+    - docker build -f pica-ci-base/Dockerfile . -t $REGISTRY_PROD/pica-ci-base:latest
+    - docker push $REGISTRY_PROD/pica-ci-base:latest
+  only:
+    changes:
+      - "pica-ci-base/*"
+    refs:
+      - master
+      - dev-ci
 
-# Create a image based on pica-ci with only a few environment variables added (which Dockerfile has been modified...)
-metabuid:
-    stage: metabuild
-    before_script:
-        - echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
-    script:
-        - docker build -f pica-ci/Dockerfile . -t $REGISTRY/pica-ci:$CI_COMMIT_SHA
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker push $REGISTRY/pica-ci:$CI_COMMIT_SHA
-    after_script:
-        - docker logout $REGISTRY
-    tags: [build]
-    only:
-        changes:
-            - "pica-dokuwiki/*"
-            - "pica-etherpad/*"
-            - "pica-oh-my-form/*"
-            - "pica-wekan/*"
-            - "pica-sympa/*"
-        refs:
-            - master
-            - dev-ci
-            - dev-sympa
+# Create a file a few environment variables added (which Dockerfile has been modified, the complete name of the image...)
+# The file will be an artifact shared with further steps
+set-variables:
+  stage: ci-base
+  tags: [build]
+  script:
+    - chmod +x ./get-modified-image.sh
+    - ./get-modified-image.sh
+  only:
+    changes:
+      - "pica-dokuwiki/*"
+      - "pica-etherpad/*"
+      - "pica-oh-my-form/*"
+      - "pica-wekan/*"
+      - "pica-sympa/*"
+    refs:
+      - master
+      - dev-ci
+      - dev-sympa
 
 # Build the image that was modified
 build:
-    stage: build
-    image: $REGISTRY/pica-ci:$CI_COMMIT_SHA
-    before_script:
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - source /etc/profile.d/ci-variables
-    script:
-        - docker build -f $MODIFIED_IMAGE/Dockerfile $MODIFIED_IMAGE -t $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
-        - docker push $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
-    after_script:
-        - docker logout $REGISTRY
-    tags: [build]
-    only:
-        changes:
-            - "pica-dokuwiki/*"
-            - "pica-etherpad/*"
-            - "pica-oh-my-form/*"
-            - "pica-wekan/*"
-            - "pica-sympa/*"
-        refs:
-            - master
-            - dev-ci
-            - dev-sympa
+  stage: build
+  tags: [build]
+  image: $REGISTRY_PROD/pica-ci-base
+  before_script:
+    - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
+    - source variables
+  script:
+    - docker build -f $MODIFIED_IMAGE/Dockerfile $MODIFIED_IMAGE -t $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+    - docker push $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+  after_script:
+    - docker logout $REGISTRY
+  only:
+    changes:
+      - "pica-dokuwiki/*"
+      - "pica-etherpad/*"
+      - "pica-oh-my-form/*"
+      - "pica-wekan/*"
+      - "pica-sympa/*"
+    refs:
+      - master
+      - dev-ci
+      - dev-sympa
 
-# run CoreOS' Clair and make the CI failed if a critical vulnerability isn't in the whitelist
+# Run CoreOS' Clair and make the CI failed if a critical vulnerability isn't in the whitelist
 clair:
-    stage: static_tests
-    image: $REGISTRY/pica-ci:$CI_COMMIT_SHA
-    before_script:
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - source /etc/profile.d/ci-variables
-        - docker pull $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
-        - docker logout $REGISTRY
-    script:
-        - docker run -d --name db arminc/clair-db:latest
-        - docker run -p 6060:6060 -d --link db:postgres --name clair --restart on-failure arminc/clair-local-scan:latest
-        - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
-        - mv clair-scanner_linux_amd64 clair-scanner
-        - chmod +x clair-scanner
-        - while( ! wget -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; done
-        - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
-    artifacts:
-        paths:
-            - clair-report.json
-            - clair.log
-    tags: [build]
-    only:
-        changes:
-            - "pica-dokuwiki/*"
-            - "pica-etherpad/*"
-            - "pica-oh-my-form/*"
-            - "pica-wekan/*"
-            - "pica-sympa/*"
-        refs:
-            - master
-            - dev-ci
-            - dev-sympa
+  stage: security-tests
+  <<: *pull-modified-image
+  script:
+    - docker run -d --name db arminc/clair-db:latest
+    - docker run -p 6060:6060 -d --link db:postgres --name clair --restart on-failure arminc/clair-local-scan:latest
+    - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
+    - mv clair-scanner_linux_amd64 clair-scanner
+    - chmod +x clair-scanner
+    - while( ! wget -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; done
+    - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
+  artifacts:
+    paths:
+      - clair-report.json
+      - clair.log
+  only:
+    changes:
+      - "pica-dokuwiki/*"
+      - "pica-etherpad/*"
+      - "pica-oh-my-form/*"
+      - "pica-wekan/*"
+      - "pica-sympa/*"
+    refs:
+      - master
+      - dev-ci
+      - dev-sympa
 
 # Run docker-bench-security and upload the results
 docker-bench-security:
-    stage: dynamic_tests
-    image: $REGISTRY/pica-ci:$CI_COMMIT_SHA
-    before_script:
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - source /etc/profile.d/ci-variables
-        - docker pull $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
-        - docker logout $REGISTRY
-        - sed -i -e "s/$MODIFIED_IMAGE_FULL/$REGISTRY\/ci-builds\/$MODIFIED_IMAGE:$CI_COMMIT_SHA/g" $MODIFIED_IMAGE/docker-compose.yml
-        # if secrets.example files exist, remove the .example extension to be able to start the container
-        - if [[ -d $MODIFIED_IMAGE/secrets ]]; then for i in $MODIFIED_IMAGE/secrets/*.example ; do cp $i $(echo $i| cut -d '.' -f1,2); done; fi;
-        # let docker-compose create the required volumes and networks
-        - "sed -i -e 's/external: true/external: false/g' $MODIFIED_IMAGE/docker-compose.yml"
-        - cat $MODIFIED_IMAGE/docker-compose.yml
-    script:
-        - cd $MODIFIED_IMAGE
-        - docker-compose up -d
-        - git clone https://github.com/docker/docker-bench-security.git
-        - cd docker-bench-security
-        - sh docker-bench-security.sh -c container_images,container_runtime,docker_security_operations -e check_5_12,check_4_1 -l ../../report.txt
-    artifacts:
-        paths:
-            - report.txt
-    after_script:
-        - docker logout $REGISTRY
-    tags: [build]
-    only:
-        changes:
-            - "pica-dokuwiki/*"
-            - "pica-etherpad/*"
-            - "pica-oh-my-form/*"
-            - "pica-wekan/*"
-            - "pica-sympa/*"
-        refs:
-            - master
-            - dev-ci
-            - dev-sympa
+  stage: security-tests
+  <<: *pull-modified-image
+  script:
+    # Change the Docker Compose to use the "testing" image, not yet pushed on production registry
+    - sed -i -e "s/$MODIFIED_IMAGE_FULL/$REGISTRY\/ci-builds\/$MODIFIED_IMAGE:$CI_COMMIT_SHA/g" $MODIFIED_IMAGE/docker-compose.yml
+    # If *.example secrets files exist, remove the .example extension to be able to start the container
+    # Indeed these file are used in Docker Compose with env_file directive
+    - if [[ -d $MODIFIED_IMAGE/secrets ]]; then for i in $MODIFIED_IMAGE/secrets/*.example ; do cp $i $(echo $i| cut -d '.' -f1,2); done; fi;
+    # Let docker-compose create the required volumes and networks
+    - "sed -i -e 's/external: true/external: false/g' $MODIFIED_IMAGE/docker-compose.yml"
+    - cat $MODIFIED_IMAGE/docker-compose.yml
+    - cd $MODIFIED_IMAGE
+    - docker-compose up -d
+    - git clone https://github.com/docker/docker-bench-security.git
+    - cd docker-bench-security
+    - sh docker-bench-security.sh -c container_images,container_runtime,docker_security_operations -e check_5_12,check_4_1 -l ../../report.txt
+  artifacts:
+    paths:
+      - report.txt
+  after_script:
+    - docker logout $REGISTRY
+  only:
+    changes:
+      - "pica-dokuwiki/*"
+      - "pica-etherpad/*"
+      - "pica-oh-my-form/*"
+      - "pica-wekan/*"
+      - "pica-sympa/*"
+    refs:
+      - master
+      - dev-ci
+      - dev-sympa
 
 # Push the generated image on the production registry,
 # once it passed all security tests and has been successfully built
 push-prod:
-    stage: push-prod
-    image: $REGISTRY/pica-ci:$CI_COMMIT_SHA
-    before_script:
-        - source /etc/profile.d/ci-variables
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker pull $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA
-        - docker logout $REGISTRY
-    script:
-        - docker tag $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA $MODIFIED_IMAGE_FULL
-        - echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
-        - docker push $REGISTRY_PROD/$MODIFIED_IMAGE_FULL
-    after_script:
-        - docker logout $REGISTRY_PROD
-    tags: [build]
-    only:
-        changes:
-            - "pica-dokuwiki/*"
-            - "pica-etherpad/*"
-            - "pica-wekan/*"
-            - "pica-oh-my-form/*"
-        refs:
-            - master
+  stage: push
+  <<: *pull-modified-image
+  script:
+    - docker tag $REGISTRY/ci-builds/$MODIFIED_IMAGE:$CI_COMMIT_SHA $MODIFIED_IMAGE_FULL
+    - echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
+    # MODIFIED_IMAGE_FULL already should include the registry URL
+    - docker push $MODIFIED_IMAGE_FULL
+  after_script:
+    - docker logout $REGISTRY_PROD
+  only:
+    changes:
+      - "pica-dokuwiki/*"
+      - "pica-etherpad/*"
+      - "pica-wekan/*"
+      - "pica-oh-my-form/*"
+    refs:
+      - master
+
+# Remove cached variables once all pipeline passed
+clean-up:
+  stage: clean
+  tags: [build]
+  script:
+    - rm variables

get-modified-image.sh

 #!/bin/sh
 
-# retrieve the name and the version of the image that was modified in the latest commit
-# this script should become obsolete as soon as a proper way of getting the modified files is added to Gitlab CI
+# Retrieve the name and the version of the image that was modified in the latest commit
+# This script should become obsolete as soon as a proper way of getting the modified files is added to Gitlab CI
 
+# Image name, without registry nor tag
 RES=""
 for i in $(git log -m -1 --name-only --pretty="format:" --first-parent)
 do
 	case "$i" in
-		*pica*) RES=$(echo $i | cut -d '/' -f1);;
+		*pica*)
+			RES=$(echo $i | cut -d '/' -f1)
+			break
+			;;
 	esac
 done
+echo "export MODIFIED_IMAGE=${RES}" >> variables
 
-if [[ "$RES" == "" ]]
-then
-	echo "No image modified"
-	exit 1
-fi
-
+# Image name with wanted registry and tag, fetched from Docker Compose
 RES=$(cat $RES/docker-compose.yml | grep $RES | cut -d ':' -f2-)
-if [[ "$RES" == "" ]]
-then
-	echo "No image modified"
-	exit 1
-else
-	echo $RES
-fi
-
+echo "export MODIFIED_IMAGE_FULL=${RES}" >> variables

pica-ci-base/Dockerfile

@@ -2,7 +2,6 @@ FROM docker:19.03.0
 RUN apk update && \
     apk add build-base \
     git \
-    gnupg \
     iproute2 \
     libffi-dev \
     openssl-dev \

pica-ci/Dockerfile

-FROM registry.picasoft.net/pica-ci-base:latest
-
-WORKDIR /workdir
-COPY . /workdir/
-
-RUN chmod +x get-modified-image.sh get-host-by-image.sh decrypt-secrets.sh && \
-    export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh) && \
-    export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1) && \
-    echo "MODIFIED_IMAGE=$MODIFIED_IMAGE" >> /etc/profile.d/ci-variables && \
-    echo "MODIFIED_IMAGE_FULL=$MODIFIED_IMAGE_FULL" >> /etc/profile.d/ci-variables