Merge branch 'dev-ci' into 'master'

Mise à jour de pica-dokuwiki et de la CI

See merge request picasoft/dockerfiles!27

.gitlab-ci.yml

@@ -3,22 +3,33 @@ services:
     - docker:dind
  
 stages:
+    - metabuild
     - build
-    - static_tests
-    - dynamic_tests
+    - static_tests   
+    - dynamic_tests     
     - deployment
 
+# build the container that further steps will run in in order to avoid duplicating instructions between steps
+metabuild:
+    stage: metabuild
+    before_script:
+        - echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
+    script:
+        - docker build -f pica-ci/Dockerfile . -t $REGISTRY_PROD/pica-ci
+        - docker push $REGISTRY_PROD/pica-ci:latest
+    tags: [build]
+    only:
+        changes:
+            - pica-dokuwiki/*
+        
+# build the container that was modified
 build:
     stage: build
+    image: $REGISTRY_PROD/pica-ci:latest
     before_script:
         - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - apk update
-        - apk add git
-        - chmod +x ./get-modified-image.sh
-        - ./get-modified-image.sh
-        - export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh)
-        - export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1)
-        - echo -e "Building container $MODIFIED_IMAGE"
+        - echo $MODIFIED_IMAGE
+        - source /etc/profile.d/ci-variables
     script:
         - docker build -f $MODIFIED_IMAGE/Dockerfile $MODIFIED_IMAGE -t $REGISTRY/ci-builds/$CI_COMMIT_SHA
         - docker push $REGISTRY/ci-builds/$CI_COMMIT_SHA
@@ -27,55 +38,49 @@ build:
     tags: [build]
     only:
         changes:
-            - pica-etherpad/*
             - pica-dokuwiki/*
 
+# run CoreOS' Clair and make the CI failed if a critical vulnerability isn't in the whitelist
 clair:
     stage: static_tests
+    image: $REGISTRY_PROD/pica-ci:latest
     before_script:
         - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
         - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
-        - apk update
-        - apk add git    
-        - chmod +x get-modified-image.sh 
-        - export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh)
-        - export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1)
-        - echo -e "Performing static analysis for container $MODIFIED_IMAGE"    
+        - docker logout $REGISTRY
+        - source /etc/profile.d/ci-variables
     script:
         - docker run -d --name db arminc/clair-db:latest
         - docker run -p 6060:6060 -d --link db:postgres --name clair --restart on-failure arminc/clair-local-scan:v2.0.6
         - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
         - mv clair-scanner_linux_amd64 clair-scanner
         - chmod +x clair-scanner
-        - echo "Waiting for Clair daemon to start"wget
         - while( ! wget -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; done 
         - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $REGISTRY/ci-builds/$CI_COMMIT_SHA 
     artifacts:
         paths:
             - clair-report.json
             - clair.log
-    after_script:
-        - docker logout $REGISTRY 
     tags: [build]
     only:
         changes:
-            - pica-etherpad/*
             - pica-dokuwiki/*
     allow_failure: false
 
+# run docker-bench-security and upload the results
 docker-bench-security:
     stage: dynamic_tests
+    image: $REGISTRY_PROD/pica-ci:latest
     before_script:
-        - apk update
-        - apk add wget py-pip git iproute2
-        - pip install docker-compose
-        - chmod +x get-modified-image.sh 
-        - export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh)
-        - export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1)
-        - sed -i -e "s/$MODIFIED_IMAGE_FULL/$REGISTRY\/ci-builds\/$CI_COMMIT_SHA:latest/g" $MODIFIED_IMAGE/docker-compose.yml 
         - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
         - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
+        - docker logout $REGISTRY
+        - source /etc/profile.d/ci-variables
+        - sed -i -e "s/$MODIFIED_IMAGE_FULL/$REGISTRY\/ci-builds\/$CI_COMMIT_SHA:latest/g" $MODIFIED_IMAGE/docker-compose.yml
+        # remove links to external networks to be able to start the container locally 
+        - sed -i -e '/networks/,+3d' $MODIFIED_IMAGE/docker-compose.yml
     script:
+        # if secrets.example files exist, remove the .example extension to be able to start the container
         - if [[ -d $MODIFIED_IMAGE/secrets ]]; then for i in $MODIFIED_IMAGE/secrets/* ; do cp $i $(echo $i| cut -d '.' -f1,2); done; fi;
         - cd $MODIFIED_IMAGE
         - docker-compose up -d
@@ -90,23 +95,14 @@ docker-bench-security:
     tags: [build]
     only:
         changes:
-            - pica-etherpad/*
             - pica-dokuwiki/*
 
+# automatically deploy the container on pica01-test
 deployment-test: 
     stage: deployment
+    image: $REGISTRY_PROD/pica-ci:latest
     before_script:
-        - apk update
-        - apk add wget py-pip git iproute2
-        - pip install docker-compose        
-        - chmod +x get-modified-image.sh 
-        - export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh)
-        - export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1)
-        - export CURRENT_CONTAINER_ID=$(docker container ls -a | grep pica-dokuwiki| cut -d ' ' -f1)
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
-        - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest  $REGISTRY/$MODIFIED_IMAGE_FULL
-        - docker push $REGISTRY/$MODIFIED_IMAGE_FULL
+        - source /etc/profile.d/ci-variables
         - export DOCKER_HOST=tcp://pica01-test.picasoft.net:2376
         - export DOCKER_TLS_VERIFY=1
         - export DOCKER_CERT_PATH=/tmp/certs
@@ -114,58 +110,53 @@ deployment-test:
         - echo "$DEV_DOCKER_CA_CERT" > $DOCKER_CERT_PATH/ca.pem
         - echo "$DEV_DOCKER_CLIENT_CERT" > $DOCKER_CERT_PATH/cert.pem
         - echo "$DEV_DOCKER_CLIENT_KEY" > $DOCKER_CERT_PATH/key.pem
-    script:   
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker pull $REGISTRY/$MODIFIED_IMAGE_FULL
-        - docker tag $REGISTRY/$MODIFIED_IMAGE_FULL $MODIFIED_IMAGE_FULL
-        - cd /docker
-        - echo $(cat docker-compose.yml | grep $MODIFIED_IMAGE -B1 | head -n1 | cut -d ':' -f1)
-        - docker-compose up -d --force-recreate --remove-orphans $(cat docker-compose.yml | grep $MODIFIED_IMAGE -B1 | head -n1 | cut -d ':' -f1)
+    script:        
+        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin     
+        - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
+        - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest  $MODIFIED_IMAGE_FULL
+        - cd $MODIFIED_IMAGE
+        - sed -i -e  s/picasoft.net/test.picasoft.net/g docker-compose.yml
+        - if [[ $(docker container ls --format "{{.Names}}" | grep $CONTAINER_NAME) ]]; then docker stop $CONTAINER_NAME | xargs docker rm; fi
+        - docker-compose up -d --force-recreate --remove-orphans $CONTAINER_NAME
     after_script:
         - rm -rf $DOCKER_CERT_PATH
         - docker logout $REGISTRY
     tags: [build]
     only:
         changes:
-            - pica-etherpad/*
             - pica-dokuwiki/*
 
-.deployment-prod: 
+# automatically deploy the container on the production host associated with the modified image
+# this will only happen after manually triggering the deployment
+deployment-prod: 
     stage: deployment
+    image: $REGISTRY_PROD/pica-ci:latest
     before_script:
-        - apk update
-        - apk add wget py-pip git iproute2
-        - pip install docker-compose        
-        - chmod +x get-modified-image.sh 
-        - export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh)
-        - export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1)
-        - export CURRENT_CONTAINER_ID=$(docker container ls -a | grep pica-dokuwiki| cut -d ' ' -f1)
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
-        - docker logout $REGISTRY
-        - echo $REGISTRY_PROD_PASSWORD | docker login $REGISTRY_PROD -u $REGISTRY_PROD_USERNAME --password-stdin
-        - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest  $REGISTRY_PROD/$MODIFIED_IMAGE_FULL
-        - docker push $REGISTRY_PROD/$MODIFIED_IMAGE_FULL
-        - docker logout $REGISTRY_PROD
-        - export REMOTE_HOSTNAME=pica01
-        - export DOCKER_HOST=tcp://$REMOTE_HOSTNAME.picasoft.net:2376
+        - source /etc/profile.d/ci-variables
+        - TMP_DOCKER_CA_CERT="${HOST}_DOCKER_CA_CERT" && eval DOCKER_CA_CERT_VARIABLE=\$$TMP_DOCKER_CA_CERT
+        - TMP_DOCKER_CLIENT_CERT="${HOST}_DOCKER_CLIENT_CERT" && eval DOCKER_CLIENT_CERT_VARIABLE=\$$TMP_DOCKER_CLIENT_CERT
+        - TMP_DOCKER_CLIENT_KEY="${HOST}_DOCKER_CLIENT_KEY" && eval DOCKER_CLIENT_KEY_VARIABLE=\$$TMP_DOCKER_CLIENT_KEY
+        - export DOCKER_HOST=tcp://$HOST.picasoft.net:2376
         - export DOCKER_TLS_VERIFY=1
         - export DOCKER_CERT_PATH=/tmp/certs
         - mkdir -p $DOCKER_CERT_PATH
-        - echo "$PROD_DOCKER_CA_CERT" > $DOCKER_CERT_PATH/ca.pem
-        - echo "$PROD_DOCKER_CLIENT_CERT" > $DOCKER_CERT_PATH/cert.pem
-        - echo "$PROD_DOCKER_CLIENT_KEY" > $DOCKER_CERT_PATH/key.pem
-    script:   
+        - echo "$DOCKER_CA_CERT_VARIABLE" > $DOCKER_CERT_PATH/ca.pem
+        - echo "$DOCKER_CLIENT_CERT_VARIABLE" > $DOCKER_CERT_PATH/cert.pem
+        - echo "$DOCKER_CLIENT_KEY_VARIABLE" > $DOCKER_CERT_PATH/key.pem
+    script:  
         - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker pull $REGISTRY/$MODIFIED_IMAGE_FULL
-        - docker logout $REGISTRY
-        - cd pica-etherpad
-        - docker-compose up -d --force-recreate --remove-orphans $(cat docker-compose.yml | grep $MODIFIED_IMAGE -B1 | head -n1 | cut -d ':' -f1)
+        - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest 
+        - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest  $MODIFIED_IMAGE_FULL
+        - cd $MODIFIED_IMAGE
+        - if [[ $(docker container ls --format "{{.Names}}" | grep $CONTAINER_NAME) ]]; then docker stop $CONTAINER_NAME | xargs docker rm; fi
+        - docker-compose up -d --force-recreate --remove-orphans $CONTAINER_NAME
     after_script:
+        - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest $REGISTRY_PROD/$MODIFIED_IMAGE_FULL
+        - docker push $REGISTRY_PROD/$MODIFIED_IMAGE_FULL
         - rm -rf $DOCKER_CERT_PATH
+        - docker logout $REGISTRY
     tags: [build]
     only:
         changes:
-            - pica-etherpad/*
-    when: manual
-
+            - pica-dokuwiki/*
+    when: manual
\ No newline at end of file

get-host-by-image.sh

+#!/bin/sh
+
+# return the host the service is hosted on
+case $MODIFIED_IMAGE in
+        pica-etherpad) RES="PICA01"
+        ;;
+        pica-dokuwiki) RES="PICA02"
+esac
+echo $RES
\ No newline at end of file

get-modified-image.sh

@@ -10,9 +10,10 @@ do
 		*pica*) RES=$i ;;
 	esac
 done
-if [[ "$i" == "" ]]
+RES=$(cat $RES/docker-compose.yml | grep image: | head -n1 | cut -d ':' -f2-)
+if [[ "$RES" == "" ]]
 then
 	exit 1
+else
+	echo $RES
 fi
-RES=$(cat $RES/docker-compose.yml | grep image: | head -n1 | cut -d ':' -f2-)
-echo $RES
\ No newline at end of file

pica-ci/Dockerfile

+FROM docker:stable
+WORKDIR /workdir
+COPY . /workdir/
+
+RUN apk update && \
+    apk add build-base \
+            git \
+            iproute2 \
+            libffi-dev \
+            openssl-dev \
+            py-pip \
+            python2-dev \
+            sed \
+            wget && \
+    pip install docker-compose && \
+    export CI_COMMIT_SHA=$(git rev-parse HEAD) && \
+    chmod +x get-modified-image.sh get-host-by-image.sh && \
+    export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh) && \
+    export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1) && \
+    export CONTAINER_NAME=$(cat $MODIFIED_IMAGE/docker-compose.yml | grep $MODIFIED_IMAGE -B1 | head -n1 | cut -d ':' -f1 | xargs) && \
+    mkdir -p /root/.docker && \
+    export HOST=$(./get-host-by-image.sh) && \
+    echo "MODIFIED_IMAGE=$MODIFIED_IMAGE" >> /etc/profile.d/ci-variables && \
+    echo "MODIFIED_IMAGE_FULL=$MODIFIED_IMAGE_FULL" >> /etc/profile.d/ci-variables && \
+    echo "CONTAINER_NAME=$CONTAINER_NAME" >> /etc/profile.d/ci-variables && \
+    echo "HOST=$HOST" >> /etc/profile.d/ci-variables && \
+    cat /etc/profile.d/ci-variables

pica-dokuwiki/clair-whitelist.yml

 generalwhitelist:
-    CVE-2018-6954: systemd -> Pas de contre mesure
     CVE-2018-15686: systemd -> Pas de contre mesure
-    CVE-2017-16997: glibc -> Pas de contre mesure
     CVE-2018-6551: glibc -> La contre mesure est dans des paquets plus anciens et il est dangereux d'installer une version fixe, ou dans sid -> Pas de conter mesure pour stretch
     CVE-2018-1000001: glibc -> Pas de contre-mesure
-    CVE-2017-18269: glibc -> La contre mesure est dans des paquets plus anciens et il est dangereux d'installer une version fixe, ou dans sid -> Pas de conter mesure pour stretch
     CVE-2019-9169: glibc -> Pas de contre-mesure
-    CVE-2017-15670: glibc -> Pas de contre-mesure
-    CVE-2017-15804: glibc -> Pas de contre-mesure
-    CVE-2017-1000408: glibc -> Pas de contre mesure
     CVE-2018-6485: glibc -> Pas de contre mesure
-    CVE-2017-9120: php7.0 -> Il n'y a pas de paquet PHP version 7 non vulnérable -> Pas de contre mesure
     CVE-2017-8923: php7.0 -> Pas de contre mesure
-    CVE-2018-1000654: libtasn1-6 -> Pas de contre-mesure
     CVE-2017-12424: shadow -> Pas de contre-mesure
-    CVE-2016-2779: util-linux -> Pas de contre-mesure 
-    CVE-2017-14062: libidn11 -> dépendance directe de wget et indirecte de curl, un des 2 est nécessaire pour le HEALTHCHECK et le téléchargement de Dokuwiki -> Pas de contre-mesure 
\ No newline at end of file
+    CVE-2016-2779: util-linux -> Pas de contre-mesure
+    CVE-2017-14062: libidn11 -> dépendance directe de wget et indirecte de curl, un des 2 est nécessaire pour le HEALTHCHECK et le téléchargement de Dokuwiki -> Pas de contre-mesure
+    CVE-2019-11068: libxslt -> dépendance de PHP, pas de contre-mesure 
\ No newline at end of file

pica-dokuwiki/docker-compose.yml

 version : "2.4"
+
+networks:
+  docker_default:
+    external: true
+
 services:
     dokuwiki-app:
-        image: pica-dokuwiki:2018.05
+        image: pica-dokuwiki:stable
         container_name: dokuwiki-app
         volumes:
           - /DATA/docker/wiki/html:/var/www/html
@@ -15,3 +20,5 @@ services:
           - "traefik.port=80"
           - "traefik.enable=true"
         restart: always
+        networks:
+          - docker_default

pica-dokuwiki/nginx.conf

@@ -31,7 +31,7 @@ http {
             try_files $uri $uri/ @dokuwiki;
         }
 
-    location ~ \.php$ {
+        location ~ \.php$ {
             try_files $uri =404;
             fastcgi_pass unix:/run/php/php7.0-fpm.sock;
             fastcgi_index index.php;

pica-etherpad/clair-whitelist.yml

@@ -20,4 +20,4 @@ generalwhitelist:
     CVE-2017-12424: shadow -> Pas de contre mesure
     CVE-2018-6954: systemd -> Pas de contre mesure
     CVE-2018-15686: systemd -> Pas de contre mesure 
-    CVE-2018-6797: Perl est une dépendance du client mysql et la version non vulnérable dans stretch n'a pas été backportée -> Pas de contre-mesure 
\ No newline at end of file
+    CVE-2018-6797: Perl est une dépendance du client mysql et la version non vulnérable dans stretch n'a pas été backportée -> Pas de contre-mesure
\ No newline at end of file