Merge branch 'master' of gitlab.utc.fr:picasoft/dockerfiles

mattermost-cleaner/README.md

@@ -10,23 +10,31 @@ https://docs.mattermost.com/deployment/on-boarding.html#do-not-manipulate-the-ma
 
 ## Usage:
 
-    docker run -d \
-        --env POSTGRES_HOST=postgres.host \
-        --env POSTGRES_PORT=27017 \
-        --env POSTGRES_USER=admin \
-        --env POSTGRES_PASS=password \
-	--env MATTERMOST_TEAM=team\
+docker run -d \
+    --name mattermost-cleaner \
+    --env POSTGRES_HOST=mattermost-db \
+    --env POSTGRES_PORT=5432 \
+    --env POSTGRES_USER=mattermost \
+    --env POSTGRES_PASS=password \
+    --env POSTGRES_DB=mattermost \
+    --env MATTERMOST_TEAM=team \
+    --net docker_default \
+    --link mattermost-db:mattermost-db \
+    registry.picasoft.net:5000/mattermost-cleaner:alpha
 
 ## Parameters
 
-    POSTGRES_HOST      the host/ip of your postgres database
-    POSTGRES_PORT      the port number of your postgres database
-    POSTGRES_USER      the username of your postgres database
-    POSTGRES_PASS      the password of your postgres database
-    CRON_TIME       the interval of cron job to run postgresdump. `0 * * * *` by default, which is every hour
+POSTGRES_HOST      the name of the container
+POSTGRES_PORT      the port number of the postgres database
+POSTGRES_USER      the username of the postgres database
+POSTGRES_PASS      the password of the postgres database
+POSTGRES_DB        the name of the postgres database
+CRON_TIME          the interval of cron job to run postgresdump. `0 * * * *` by default, which is every hour
 
 ## Docker-compose
 
+version: "2"
+services:
   mattermost-cleaner:
     image: registry.picasoft.net:5000/mattermost-cleaner:alpha
     container_name: mattermost-cleaner
@@ -36,7 +44,6 @@ https://docs.mattermost.com/deployment/on-boarding.html#do-not-manipulate-the-ma
       - POSTGRES_HOST=mattermost-db
       - POSTGRES_PORT=5432
       - POSTGRES_USER=mattermost
-      - POSTGRES_PASS=...
+      - POSTGRES_PASS=password
       - POSTGRES_DB=mattermost
-      - MATTERMOST_TEAM=name
-
+      - MATTERMOST_TEAM=team

mattermost-cleaner/psqlcmd.sh

@@ -13,5 +13,5 @@ export PGDATABASE=$POSTGRES_DB
 export PGUSER=$POSTGRES_USER
 export PGPASSWORD=$POSTGRES_PASS
 
-SQL_CMD="DELETE FROM posts WHERE id IN (SELECT p.id FROM posts p LEFT JOIN channels c ON p.channelid=c.id LEFT JOIN teams t ON c.teamid=t.id WHERE p.type='system_join_channel' AND t.name = '${MATTERMOST_TEAM}')"
+SQL_CMD="DELETE FROM posts WHERE id IN (SELECT p.id FROM posts p LEFT JOIN channels c ON p.channelid=c.id LEFT JOIN teams t ON c.teamid=t.id WHERE (p.type='system_join_channel' OR p.type='system_join_team') AND t.name = '${MATTERMOST_TEAM}')"
 psql -c "${SQL_CMD}"

pica-backup/Dockerfile

@@ -4,12 +4,15 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
       postgresql-client \
       mysql-client \
+      mongodb-clients \
+      mongo-tools \
       cron && \
     apt-get clean \
     mkdir /scripts/{mysql,postgresql}
 
 COPY mysql-run.sh /scripts/mysql/mysql-run.sh
 COPY postgres-run.sh /scripts/postgresql/postgres-run.sh
+COPY mongo-run.sh /scripts/mongodb/mongo-run.sh
 COPY run.sh /scripts
 
 ENV CRON_TIME="0 0 * * *"

pica-backup/mongo-run.sh

+#!/bin/bash
+# Antoine Barbare 18/02/18 <antoine@barbare.me>
+
+BACKUP_FOLDER=${BACKUP_FOLDER:-"/backup/"}
+
+[ -z "${MONGO_HOST}" ] && { echo "=> MONGO_HOST cannot be empty" && exit 1; }
+[ -z "${MONGO_PORT}" ] && { echo "=> MONGO_PORT cannot be empty" && exit 1; }
+[ -z "${MONGO_DB}" ] && { echo "=> MONGO_DB cannot be empty" && exit 1; }
+
+if [[ -z "${MONGO_USER}" && -z "${MONGO_PASS}" ]]; then
+  BACKUP_CMD="mongodump -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} -o $BACKUP_FOLDER"'${BACKUP_NAME}'
+  RESTORE_CMD="mongorestore -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} "
+else
+  BACKUP_CMD="mongodump -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USER} -p ${MONGO_PASS} -o $BACKUP_FOLDER"'${BACKUP_NAME}'
+  RESTORE_CMD="mongorestore -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USER} -p ${MONGO_PASS} "
+fi
+
+##########################
+# CREATING BACKUP SCRIPT #
+##########################
+backup_script_name="${MONGO_SERVICE_NAME}-backup.sh"
+
+echo "=> ${MONGO_SERVICE_NAME}: Creating backup script"
+rm -f "/$backup_script_name"
+
+cat <<EOF >> "/$backup_script_name"
+#!/bin/bash
+
+BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S)
+
+echo "=> ${MONGO_SERVICE_NAME}: Backup started: \${BACKUP_NAME}"
+if ${BACKUP_CMD} ;then
+    MONGO_BACKUP=$BACKUP_FOLDER\${BACKUP_NAME}
+    echo " => Compress files \$MONGO_BACKUP.tar.gz"
+    tar -czvf \$MONGO_BACKUP.tar.gz \$MONGO_BACKUP && \
+    rm -rf \$MONGO_BACKUP && \
+    echo "${MONGO_SERVICE_NAME}:  Backup succeeded"
+else
+    echo "${MONGO_SERVICE_NAME}: Backup failed"
+    rm -rf $BACKUP_FOLDER\${BACKUP_NAME}
+fi
+EOF
+chmod +x /$backup_script_name
+
+###########################
+# CREATING RESTORE SCRIPT #
+###########################
+restore_script_name="${MONGO_SERVICE_NAME}-restore.sh"
+backup_log="${MONGO_SERVICE_NAME}_mongo_backup.log"
+
+echo "=> ${MONGO_SERVICE_NAME}: Creating restore script"
+rm -f "/$restore_script_name"
+
+cat <<EOF >> /$restore_script_name
+#!/bin/bash
+
+echo "=> ${MONGO_SERVICE_NAME}: Restore database from \$1"
+echo "  => Uncompress save \$1"
+tar -xzvf \$1
+output="\$(echo \$1 | awk -F'.tar.gz' '{print \$1}')"
+if mongorestore -d wekan -h wekan-db2:27017 \$output;then
+      echo "wekan: Restore succeeded"
+else
+      echo "wekan: Restore failed"
+fi
+rm -Rf \$output
+EOF
+chmod +x /$restore_script_name
+
+touch /$backup_log
+tail -F /$backup_log &
+
+if [ -n "${INIT_BACKUP}" ]; then
+    echo "=> ${MONGO_SERVICE_NAME}: Create a backup on the startup"
+    /$backup_script_name
+elif [ -n "${INIT_RESTORE_LATEST}" ]; then
+    echo "=> ${MONGO_SERVICE_NAME}: Restore latest backup"
+    until nc -z $MONGO_HOST $MONGO_PORT
+    do
+        echo "waiting database container..."
+        sleep 1
+    done
+    ls -d -1 $BACKUP_FOLDER* | tail -1 | xargs /$restore_script_name
+fi
+
+echo "${CRON_TIME} /$backup_script_name >> /$backup_log 2>&1" >> /crontab.conf
+crontab  /crontab.conf
+echo "=> ${MONGO_SERVICE_NAME}: Running cron job"

pica-backup/mysql-run.sh

@@ -32,10 +32,14 @@ rm -f "/$backup_script_name"
 cat <<EOF >> "/$backup_script_name"
 #!/bin/bash
 
-BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S).sql
+DATE=\$(date +\%Y.\%m.\%d.\%H\%M\%S)
+BACKUP_NAME=\$DATE.sql
 
 echo "=> ${MYSQL_SERVICE_NAME}: Backup started: \${BACKUP_NAME}"
 if ${BACKUP_CMD} ;then
+    echo " => Compress files $BACKUP_FOLDER\$DATE.tar.gz"
+    tar -czvf $BACKUP_FOLDER\$DATE.tar.gz $BACKUP_FOLDER\${BACKUP_NAME} && \
+    rm -rf $BACKUP_FOLDER\$BACKUP_NAME && \
     echo "${MYSQL_SERVICE_NAME}: Backup succeeded"
 else
     echo "${MYSQL_SERVICE_NAME}: Backup failed"
@@ -56,11 +60,14 @@ rm -f "/$restore_script_name"
 cat <<EOF >> /$restore_script_name
 #!/bin/bash
 echo "=> ${MYSQL_SERVICE_NAME}: Restore database from \$1"
-if mysql -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} < \$1 ;then
+tar -xzvf \$1
+output="\$(echo \$1 | awk -F'.tar.gz' '{print \$1".sql"}')"
+if mysql -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} < \$output ;then
     echo "${MYSQL_SERVICE_NAME}: Restore succeeded"
 else
     echo "${MYSQL_SERVICE_NAME}: Restore failed"
 fi
+rm -Rf \$output
 EOF
 chmod +x /$restore_script_name
 

pica-backup/postgres-run.sh

@@ -26,10 +26,14 @@ export PGDATABASE=$POSTGRES_DB
 export PGUSER=$POSTGRES_USER
 export PGPASSWORD=$POSTGRES_PASS
 
-BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S).sql
+DATE=\$(date +\%Y.\%m.\%d.\%H\%M\%S)
+BACKUP_NAME=\$DATE.sql
 
 echo "=> ${POSTGRES_SERVICE_NAME}: Backup started: \${BACKUP_NAME}"
 if ${BACKUP_CMD} ;then
+    echo " => Compress files $BACKUP_FOLDER\$DATE.tar.gz"
+    tar -czvf $BACKUP_FOLDER\$DATE.tar.gz $BACKUP_FOLDER\${BACKUP_NAME} && \
+    rm -rf $BACKUP_FOLDER\$BACKUP_NAME && \
     echo "${POSTGRES_SERVICE_NAME}:  Backup succeeded"
 else
     echo "${POSTGRES_SERVICE_NAME}: Backup failed"
@@ -56,11 +60,14 @@ export PGUSER=$POSTGRES_USER
 export PGPASSWORD=$POSTGRES_PASS
 
 echo "=> ${POSTGRES_SERVICE_NAME}: Restore database from \$1"
-if psql < \$1 ;then
+tar -xzvf \$1 
+output="\$(echo \$1 | awk -F'.tar.gz' '{print \$1".sql"}')"
+if psql < \$output ;then
     echo "${POSTGRES_SERVICE_NAME}: Restore succeeded"
 else
     echo "${POSTGRES_SERVICE_NAME}: Restore failed"
 fi
+rm -Rf \$output
 EOF
 chmod +x /$restore_script_name
 

pica-backup/run.sh

@@ -2,17 +2,17 @@
 
 # DB_TYPE : MYSQL|POSTGRES
 
-# MYSQL				POSTGRESQL
-# MYSQL_SERVICE_NAME_LIST	POSTGRES_SERVICE_NAME_LIST
-# MYSQL_HOST_LIST		POSTGRES_HOST_LIST
-# MYSQL_PORT_LIST		POSTGRES_PORT_LIST
-# MYSQL_USER_LIST		POSTGRES_USER_LIST
-# MYSQL_PASS_LIST		POSTGRES_PASS_LIST
-# MYSQL_DB_LIST			POSTGRES_DB_LIST
-# MYSQL_INIT_BACKUP_LIST	POSTGRES_INIT_BACKUP_LIST
-# MYSQL_CRON_TIME_LIST		POSTGRES_CRON_TIME_LIST
-# MYSQL_EXTRA_OPTS_LIST		POSTGRES_EXTRA_OPTS-LIST
-# MYSQL_BACKUP_FOLDER_LIST	POSTGRESQL_BACKUP_FOLDER_LIST
+# MYSQL				POSTGRESQL                        MONGODB
+# MYSQL_SERVICE_NAME_LIST	POSTGRES_SERVICE_NAME_LIST        MONGO_SERVICE_NAME_LIST
+# MYSQL_HOST_LIST		POSTGRES_HOST_LIST                MONGO_HOST_LIST
+# MYSQL_PORT_LIST		POSTGRES_PORT_LIST                MONGO_PORT_LIST
+# MYSQL_USER_LIST		POSTGRES_USER_LIST                MONGO_USER_LIST
+# MYSQL_PASS_LIST		POSTGRES_PASS_LIST                MONGO_PASS_LIST
+# MYSQL_DB_LIST			POSTGRES_DB_LIST                  MONGO_DB_LIST
+# MYSQL_INIT_BACKUP_LIST	POSTGRES_INIT_BACKUP_LIST         MONGO_INIT_BACKUP_LIST
+# MYSQL_CRON_TIME_LIST		POSTGRES_CRON_TIME_LIST           MONGO_CRON_TIME_LIST
+# MYSQL_EXTRA_OPTS_LIST		POSTGRES_EXTRA_OPTS-LIST          MONGO_BACKUP_FOLDER_LIST 
+# MYSQL_BACKUP_FOLDER_LIST	POSTGRESQL_BACKUP_FOLDER_LIST     
 
 #####################
 # Create the arrays #
@@ -43,6 +43,17 @@ read -a POSTGRES_CRON_TIME_ARRAY <<< "$POSTGRES_CRON_TIME_LIST"
 read -a POSTGRES_EXTRA_OPTS_ARRAY <<< "$POSTGRES_EXTRA_OPTS_LIST"
 read -a POSTGRES_BACKUP_FOLDER_ARRAY <<< "$POSTGRES_BACKUP_FOLDER_LIST"
 
+#MONGO
+read -a MONGO_HOST_ARRAY <<< "$MONGO_HOST_LIST"
+read -a MONGO_SERVICE_NAME_ARRAY <<< "$MONGO_SERVICE_NAME_LIST"
+read -a MONGO_PORT_ARRAY <<< "$MONGO_PORT_LIST"
+read -a MONGO_USER_ARRAY <<< "$MONGO_USER_LIST"
+read -a MONGO_PASS_ARRAY <<< "$MONGO_PASS_LIST"
+read -a MONGO_DB_ARRAY <<< "$MONGO_DB_LIST"
+read -a MONGO_INIT_BACKUP_ARRAY <<< "$MONGO_INIT_BACKUP_LIST"
+read -a MONGO_CRON_TIME_ARRAY <<< "$MONGO_CRON_TIME_LIST"
+read -a MONGO_BACKUP_FOLDER_ARRAY <<< "$MONGO_BACKUP_FOLDER_LIST"
+
 ###########################
 # Call the backup scripts #
 ###########################
@@ -85,4 +96,21 @@ do
 	/scripts/postgresql/postgres-run.sh
 done
 
+#Execute MongoDB backups
+NB_CALLS=${#MONGO_HOST_ARRAY[@]}
+
+for (( i=0; i < $NB_CALLS; i++ ))
+do
+        MONGO_SERVICE_NAME=${MONGO_SERVICE_NAME_ARRAY[i]} \
+        MONGO_HOST=${MONGO_HOST_ARRAY[i]} \
+        MONGO_PORT=${MONGO_PORT_ARRAY[i]} \
+        MONGO_USER=${MONGO_USER_ARRAY[i]} \
+        MONGO_PASS=${MONGO_PASS_ARRAY[i]} \
+        MONGO_DB=${MONGO_DB_ARRAY[i]} \
+        INIT_BACKUP=${MONGO_INIT_BACKUP_ARRAY[i]} \
+        CRON_TIME=${MONGO_CRON_TIME_ARRAY[i]} \
+        BACKUP_FOLDER="/backup/${MONGO_BACKUP_FOLDER_ARRAY[i]}/" \
+        /scripts/mongodb/mongo-run.sh
+done
+
 exec cron -f