Merge branch 'dev-ci' into 'master'

Gestion des secrets

See merge request !35

.gitlab-ci.yml

 image: docker:stable
 services:
     - docker:dind
- 
+
 stages:
     - metabuild
     - build
-    - static_tests   
-    - dynamic_tests     
+    - static_tests
+    - dynamic_tests
     - deployment
-
+    
 # build the container that further steps will run in in order to avoid duplicating instructions between steps
 metabuild:
     stage: metabuild
@@ -23,10 +23,11 @@ metabuild:
     only:
         changes:
             - "pica-dokuwiki/*"
+            - "pica-etherpad/*"
         refs:
             - master
             - dev-ci
-        
+
 # build the container that was modified
 build:
     stage: build
@@ -38,11 +39,12 @@ build:
         - docker build -f $MODIFIED_IMAGE/Dockerfile $MODIFIED_IMAGE -t $REGISTRY/ci-builds/$CI_COMMIT_SHA
         - docker push $REGISTRY/ci-builds/$CI_COMMIT_SHA
     after_script:
-        - docker logout $REGISTRY        
+        - docker logout $REGISTRY
     tags: [build]
     only:
         changes:
             - "pica-dokuwiki/*"
+            - "pica-etherpad/*"
         refs:
             - master
             - dev-ci
@@ -62,8 +64,8 @@ clair:
         - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
         - mv clair-scanner_linux_amd64 clair-scanner
         - chmod +x clair-scanner
-        - while( ! wget -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; done 
-        - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $REGISTRY/ci-builds/$CI_COMMIT_SHA 
+        - while( ! wget -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; done
+        - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r clair-report.json -l clair.log -w $MODIFIED_IMAGE/clair-whitelist.yml --threshold="High" $REGISTRY/ci-builds/$CI_COMMIT_SHA
     artifacts:
         paths:
             - clair-report.json
@@ -72,6 +74,7 @@ clair:
     only:
         changes:
             - "pica-dokuwiki/*"
+            - "pica-etherpad/*"
         refs:
             - master
             - dev-ci
@@ -86,13 +89,13 @@ docker-bench-security:
         - docker logout $REGISTRY
         - source /etc/profile.d/ci-variables
         - sed -i -e "s/$MODIFIED_IMAGE_FULL/$REGISTRY\/ci-builds\/$CI_COMMIT_SHA:latest/g" $MODIFIED_IMAGE/docker-compose.yml
-        # remove links to external networks to be able to start the container locally 
-        - sed -i -e '/networks/,+3d' $MODIFIED_IMAGE/docker-compose.yml
-    script:
         # if secrets.example files exist, remove the .example extension to be able to start the container
-        - if [[ -d $MODIFIED_IMAGE/secrets ]]; then for i in $MODIFIED_IMAGE/secrets/* ; do cp $i $(echo $i| cut -d '.' -f1,2); done; fi;
+        - if [[ -d $MODIFIED_IMAGE/secrets ]]; then for i in $MODIFIED_IMAGE/secrets/*.example ; do cp $i $(echo $i| cut -d '.' -f1,2); done; fi;
+        # let docker-compose create the required volumes and networks
+        - "sed -i -e 's/external: true/external: false/g' $MODIFIED_IMAGE/docker-compose.yml"
+        - cat $MODIFIED_IMAGE/docker-compose.yml
+    script:
         - cd $MODIFIED_IMAGE
-        - docker volume create $CONTAINER_NAME
         - docker-compose up -d
         - git clone https://github.com/docker/docker-bench-security.git
         - cd docker-bench-security
@@ -101,19 +104,22 @@ docker-bench-security:
         paths:
             - report.txt
     after_script:
-        - docker logout $REGISTRY 
+        - docker logout $REGISTRY
     tags: [build]
     only:
         changes:
             - "pica-dokuwiki/*"
+            - "pica-etherpad/*"
         refs:
             - master
             - dev-ci
 
 # automatically deploy the container on pica01-test
-deployment-test: 
+deployment-test:
     stage: deployment
     image: $REGISTRY/pica-ci:latest
+    variables:
+        PICA_ENVIRONMENT: "TEST"
     before_script:
         - source /etc/profile.d/ci-variables
         - export DOCKER_HOST=tcp://pica01-test.picasoft.net:2376
@@ -123,13 +129,15 @@ deployment-test:
         - echo "$DEV_DOCKER_CA_CERT" > $DOCKER_CERT_PATH/ca.pem
         - echo "$DEV_DOCKER_CLIENT_CERT" > $DOCKER_CERT_PATH/cert.pem
         - echo "$DEV_DOCKER_CLIENT_KEY" > $DOCKER_CERT_PATH/key.pem
-    script:
-        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin     
+        - chmod +x decrypt-secrets.sh
+        - if [[ -d "$MODIFIED_IMAGE/secrets" ]]; then eval $(cat /etc/profile.d/ci-variables) ./decrypt-secrets.sh; fi
+        - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
         - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
         - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest  $MODIFIED_IMAGE_FULL
+    script:        
         - cd $MODIFIED_IMAGE
         - sed -i -e  s/picasoft.net/test.picasoft.net/g docker-compose.yml
-        - if [[ $(docker container ls --format "{{.Names}}" | grep $CONTAINER_NAME) ]]; then docker stop $CONTAINER_NAME | xargs docker rm; fi
+        - for SERVICE in $(docker-compose config --services); do if [[ $(docker container ls --format "{{.Names}}" | grep $SERVICE) ]]; then docker stop $SERVICE | xargs docker rm; fi; done
         - docker-compose up -d --force-recreate --remove-orphans $CONTAINER_NAME
     after_script:
         - rm -rf $DOCKER_CERT_PATH
@@ -138,15 +146,18 @@ deployment-test:
     only:
         changes:
             - "pica-dokuwiki/*"
+            - "pica-etherpad/*"
         refs:
             - master
             - dev-ci
 
 # automatically deploy the container on the production host associated with the modified image
 # this will only happen after manually triggering the deployment
-deployment-prod: 
+deployment-prod:
     stage: deployment
     image: $REGISTRY/pica-ci:latest
+    variables:
+        PICA_ENVIRONMENT: "PRODUCTION"
     before_script:
         - source /etc/profile.d/ci-variables
         - TMP_DOCKER_CA_CERT="${HOST}_DOCKER_CA_CERT" && eval DOCKER_CA_CERT_VARIABLE=\$$TMP_DOCKER_CA_CERT
@@ -159,12 +170,14 @@ deployment-prod:
         - echo "$DOCKER_CA_CERT_VARIABLE" > $DOCKER_CERT_PATH/ca.pem
         - echo "$DOCKER_CLIENT_CERT_VARIABLE" > $DOCKER_CERT_PATH/cert.pem
         - echo "$DOCKER_CLIENT_KEY_VARIABLE" > $DOCKER_CERT_PATH/key.pem
-    script:  
+        - chmod +x decrypt-secrets.sh
+        - if [[ -d "$MODIFIED_IMAGE/secrets" ]]; then eval $(cat /etc/profile.d/ci-variables) ./decrypt-secrets.sh; fi
         - echo $REGISTRY_PASSWORD | docker login $REGISTRY -u $REGISTRY_USERNAME --password-stdin
-        - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest 
+        - docker pull $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest
         - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest  $MODIFIED_IMAGE_FULL
+    script:
         - cd $MODIFIED_IMAGE
-        - if [[ $(docker container ls --format "{{.Names}}" | grep $CONTAINER_NAME) ]]; then docker stop $CONTAINER_NAME | xargs docker rm; fi
+        - for SERVICE in $(docker-compose config --services); do if [[ $(docker container ls --format "{{.Names}}" | grep $SERVICE) ]]; then docker stop $SERVICE | xargs docker rm; fi; done
         - docker-compose up -d --force-recreate --remove-orphans $CONTAINER_NAME
     after_script:
         - docker tag $REGISTRY/ci-builds/$CI_COMMIT_SHA:latest $REGISTRY_PROD/$MODIFIED_IMAGE_FULL
@@ -175,7 +188,8 @@ deployment-prod:
     only:
         changes:
             - "pica-dokuwiki/*"
+            - "pica-etherpad/*"
         refs:
             - master
             - dev-ci
-    when: manual
\ No newline at end of file
+    when: manual

decrypt-secrets.sh

+#!/bin/sh
+
+# import the PGP key for the right environment
+if [ "$PICA_ENVIRONMENT" = "TEST" ];
+then
+    echo "$PRIVATE_GPG_KEY_TEST" > /tmp/pgp-key
+elif [ "$PICA_ENVIRONMENT" = "PRODUCTION" ];
+then
+    echo "$PRIVATE_GPG_KEY_PRODUCTION" > /tmp/pgp-key
+else
+    exit -1
+fi
+cat  /tmp/pgp-key | gpg --import
+
+# decrypt the secrets
+SECRETS_PATH=$MODIFIED_IMAGE/secrets/encrypted-variables-$(echo $PICA_ENVIRONMENT | tr '[:upper:]' '[:lower:]')
+echo $SECRETS_PATH
+
+for container_secrets_folder in $SECRETS_PATH/*;
+do
+    CONTAINER_NAME=$(basename $container_secrets_folder)
+    for encrypted_secret in $container_secrets_folder/*;
+    do
+        SECRET_NAME=$(basename $encrypted_secret) 
+        ( echo -n "$SECRET_NAME=" ; gpg --quiet  --decrypt  $encrypted_secret ) >> $MODIFIED_IMAGE/secrets/$CONTAINER_NAME.secrets
+    done
+done
+
+ls -al pica-etherpad/secrets/
\ No newline at end of file

pica-ci/Dockerfile

@@ -5,6 +5,7 @@ COPY . /workdir/
 RUN apk update && \
     apk add build-base \
             git \
+            gnupg \
             iproute2 \
             libffi-dev \
             openssl-dev \
@@ -13,7 +14,7 @@ RUN apk update && \
             sed \
             wget && \
     pip install docker-compose && \
-    chmod +x get-modified-image.sh get-host-by-image.sh && \
+    chmod +x get-modified-image.sh get-host-by-image.sh decrypt-secrets.sh && \
     export MODIFIED_IMAGE_FULL=$(./get-modified-image.sh) && \
     export MODIFIED_IMAGE=$(echo $MODIFIED_IMAGE_FULL | cut -d ':' -f1) && \
     export CONTAINER_NAME=$(cat $MODIFIED_IMAGE/docker-compose.yml | grep $MODIFIED_IMAGE -B1 | head -n1 | cut -d ':' -f1 | xargs) && \

pica-dokuwiki/clair-whitelist.yml

@@ -8,4 +8,4 @@ generalwhitelist:
     CVE-2017-12424: shadow -> Pas de contre-mesure
     CVE-2016-2779: util-linux -> Pas de contre-mesure
     CVE-2017-14062: libidn11 -> dépendance directe de wget et indirecte de curl, un des 2 est nécessaire pour le HEALTHCHECK et le téléchargement de Dokuwiki -> Pas de contre-mesure
-    CVE-2019-11068: libxslt -> dépendance de PHP, pas de contre-mesure
\ No newline at end of file
+    CVE-2019-11068: libxslt -> dépendance de PHP, pas de contre-mesure 
\ No newline at end of file

pica-etherpad/clair-whitelist.yml

 generalwhitelist:
     CVE-2017-14062: libidn11 -> pas de contre mesure disponible pour stretch
-    CVE-2019-3823: curl -> non affecté, le paquet qui contient la contre mesure est installé à la place de celui qui est vulnérable cf logs
-    CVE-2019-3822: curl -> idem
-    CVE-2018-1000654: libtasn1-6 -> Pas de contre mesure disponible
-    CVE-2016-9841: zlib -> le paquet qui corrige le problème n'est pas backporté -> Pas de contre mesure
-    CVE-2016-2774: isc-dhcp -> Le paquet qui corrige le problème n'est pas backporté, et DHCP n'est probablement même pas utilisé par le conteneur -> Pas de contre mesure
-    CVE-2016-9843: zlib -> le paquet qui corrige le problème n'est pas backporté -> Pas de contre mesure
     CVE-2016-2779: util-linux -> Vulnérabilité Linux
     CVE-2017-10788: libdbd-mysql-perl -> Bug qui semble nécessiter que la base de données soit accessible par le réseau ce qui n'est pas le cas -> Non affecté
     CVE-2018-6485: glibc -> Pas de contre mesure
-    CVE-2017-16997: glibc -> Pas de contre mesure
-    CVE-2017-18269: glibc -> Pas de contre mesure
-    CVE-2017-15670: glibc -> Pas de contre mesure
     CVE-2018-6551: glibc -> Pas de contre mesure
     CVE-2018-1000001: glibc -> Pas de contre mesure
-    CVE-2017-1000408: glibc -> Pas de contre mesure
-    CVE-2017-15804: glibc -> Pas de contre mesure
     CVE-2019-9169: glibc -> Pas de contre mesure
     CVE-2017-12424: shadow -> Pas de contre mesure
-    CVE-2018-6954: systemd -> Pas de contre mesure
-    CVE-2018-15686: systemd -> Pas de contre mesure 
-    CVE-2018-6797: Perl est une dépendance du client mysql et la version non vulnérable dans stretch n'a pas été backportée -> Pas de contre-mesure
\ No newline at end of file
+    CVE-2019-11068: libxslt -> dépendance d'autres paquets, pas de correctif disponible -> Pas de contre-mesure
+    CVE-2019-9631: poppler -> à désinstaller 
\ No newline at end of file

pica-etherpad/docker-compose.yml

 version : "2.4"
+
+volumes:
+  etherpad-db-volume:
+    external: true
+    name: "etherpad-db"
+
+networks:
+  docker_default:
+    external: true
+    name: "docker_default"
+
 services:
     etherpad-app:
         image: pica-etherpad:1.7.5
         container_name: etherpad-app
+        depends_on:
+          - etherpad-db
         links:
           - etherpad-db:mysql
         security_opt:
@@ -19,7 +32,8 @@ services:
           - ETHERPAD_THEME=colibris
           - ETHERPAD_MINIFY=true
         restart: always
-
+        networks:
+          - docker_default
     etherpad-db:
         image: mysql:5.7
         container_name: etherpad-db
@@ -29,7 +43,7 @@ services:
         cpus: "0.20"
         pids_limit: 1024
         volumes:
-          - /DATA/docker/etherpad/etherpad-db/data:/var/lib/mysql
+          - etherpad-db-volume:/var/lib/mysql
         env_file: ./secrets/etherpad-db.secrets
         healthcheck:
           test: "/usr/bin/mysql --user=root --password=$${MYSQL_ROOT_PASSWORD} --execute \"SHOW DATABASES;\""
@@ -37,4 +51,5 @@ services:
           timeout: 20s
           retries: 10
         restart: always
-                                                                                    
+        networks:
+          - docker_default
\ No newline at end of file

pica-etherpad/secrets/encrypted-variables-production/etherpad-db/MYSQL_DATABASE

+kok
}<=5Jo["\iwoA-#	nK:хSΐp;iq9՘7r|G_#uVFrm݀eÕ
oḽ%X)WYx{݁YS5;Lp	X&Z'MS?x8
$qMm~i$pGwa676Q.w[YH1(L&DF#΢W0	D.k)<D|X(#~"
**-?rI\g=.fhT2řV	`rkO_ss49c,s=]##g|@23_My$Qܤz7@ |JP 6ajqS}dg-ݞ9ԕv"
+J	[ȮG
+Ef&e	$	[f\%&$W
Z1GP>GHmJ%~y#'7/f|ݵPB
t>J(J
\ No newline at end of file