diff --git a/backup-rotation/Dockerfile b/backup-rotation/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..21ab2a0de395da59ab7cf703cc5c7c4931867d6a --- /dev/null +++ b/backup-rotation/Dockerfile @@ -0,0 +1,23 @@ +FROM registry.picasoft.net:5000/pica-debian:stretch + +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get -y install \ + cron \ + python3 \ + python3-pip \ + && pip3 install \ + rotate-backups \ + python-crontab \ + && rm -rf /var/lib/apt/lists/* + +VOLUME ["/backup"] +VOLUME ["/config"] + +COPY start_rotation.py start_rotation.py +COPY backup_data.json /config/backup_data.json + +RUN touch /crontab.conf && \ + chmod +x start_rotation.py + +CMD /start_rotation.py && cron && crontab /crontab.conf && tail -f /etc/crontab diff --git a/backup-rotation/README.txt b/backup-rotation/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..8cbfe9a3e374dad45673a20a27783e3c057924b1 --- /dev/null +++ b/backup-rotation/README.txt @@ -0,0 +1,15 @@ +Le fichier backup_data.json recense les informations sur les différents services mis en place sur la machine. + +Il est structuré ainsi : + +Nom du service : Contient une structure de données contenant les informations relatives au service telles qu'indiquées ci-dessous + "Host" : Indique l'hôte de la base de données + "Port" : Indique le port pour se connecter au service + "Database" : Indique le nom de la base de données + "Folder" : Indique le nom du dossier de backup utilisé par le script de backup et de rotation + "Cron" : Indique la fréquence de temps à laquelle les backups sont effectués par le script de rotation au format cron + "Backup-Rota" : Contient les paramètres pour la rotation des backups dans une structure de données comme indiqué ci-dessous + "Hour" : nombre de backups horaires à conserver + "Day" : nombre de backups quotidiens à conserver + "Week" : nombre de backups hebdomadaires à conserver + "Month": nombre de backups mensuels à conserver diff --git a/backup-rotation/backup_data.json b/backup-rotation/backup_data.json new file mode 100644 index 0000000000000000000000000000000000000000..6b29ca21e6d47062806147e791672cb6c0efdd73 --- /dev/null +++ b/backup-rotation/backup_data.json @@ -0,0 +1,40 @@ +{ + "wekan": + { + "Host": "wekan-db", + "Port": "27017", + "Database": "wekan", + "Type": "mongo", + "Folder": "wekan", + "Cron" : "0 * * * *", + "Init-Backup" : "0", + "Backup-rota": + { + "Hour" : 24, + "Day" : 7, + "Week" : 4, + "Month" : 12 + } + }, + "etherpad": + { + "Host": "etherpad-db", + "Port": "3306", + "User": "root", + "Password": "lolilolilol", + "Database": "--all-databases", + "Type": "mysql", + "Folder": "etherpad", + "Cron" : "0 * * * *", + "Options" : "--single-transaction", + "Init-Backup" : "0", + "Backup-rota": + { + "Hour" : 24, + "Day" : 7, + "Week" : 4, + "Month" : 12 + } + } +} + diff --git a/backup-rotation/fake_backups.py b/backup-rotation/fake_backups.py new file mode 100755 index 0000000000000000000000000000000000000000..f5be76517de34361b6fe4473725da2fbb2ac04fe --- /dev/null +++ b/backup-rotation/fake_backups.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 +from datetime import datetime, timedelta +import getopt +import os +import sys +import time + +def main(argv): + + folder = False + numberOfBackups = False + + # On récupère les paramètres passés et on vérifie que les paramètres -f + # et -n existent bien. + try: + opts, args = getopt.getopt(argv, "f:n:") + except getopt.GetoptError: + print('usage: fake_backups.py -f <folder> -n <number of backups>') + sys.exit(2) + for opt, arg in opts: + if opt in ('-f'): + # -f doit contenir le chemin d'un dossier existant + if os.path.isdir(arg): + folder = arg + else: + print('Please specify a valid path for argument -f <folder>.') + sys.exit(2) + elif opt in ('-n'): + # -n doit être un entier + try: + numberOfBackups = int(arg) + except ValueError: + print('Please specify a valid number for argument -n <number of backups>.') + if folder and numberOfBackups: + i = 0 + # On génère des fichiers vides comportant comme nom la date actuellement moins la valeur de i en heures + start = time.time() + while(i < numberOfBackups): + d = datetime.today() - timedelta(hours=i) + filename = d.strftime('%Y.%m.%d.%H%M%S') + file = open(folder + '/' + filename + '.tar.gz', 'w') + file.close() + i += 1 + end = time.time() + time_elapsed = end - start + + print('[' + folder + '/' + ']') + # Affichage du temps de génération du script en secondes + print(str(i) + ' files generated in ' + str(time_elapsed) + ' seconds.') + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/backup-rotation/start_rotation.py b/backup-rotation/start_rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..612bcb136f99b560668b99a3e8fcf4649888e9e1 --- /dev/null +++ b/backup-rotation/start_rotation.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 +import os +import sys +import json +from crontab import CronTab + +# Définition des paramètres + +BACKUP_FOLDER = '/backup' +CONFIG_FOLDER = '/config' +CONFIG_FILE = 'backup_data.json' +PATH_TO_CONFIG = CONFIG_FOLDER + '/' + CONFIG_FILE + +# +# Garde-fou : Test d'existence des dossiers à manipuler +# + +if not(os.path.exists(BACKUP_FOLDER) and os.path.exists(CONFIG_FOLDER)): + sys.stderr.write('Err: BACKUP_FOLDER or CONFIG_FOLDER doesn\'t exist. \n') + sys.exit() + +if not(os.path.isfile(PATH_TO_CONFIG)): + sys.stderr.write('Err: ' + PATH_TO_CONFIG + ' not found. \n') + sys.exit() + +# Création d'une table Cron pour enregistrer les tâches à effectuer + +cron = CronTab(user="root") +cron.remove_all() + +# Récupération des informations de configuration depuis un fichier .json + +services_list = json.load(open(PATH_TO_CONFIG)) + +#Traitement de chaque service : ajout des informations +#pour la rotation spécfiques à chaque service + +for service in services_list: + backup_rota = services_list[service]["Backup-rota"] + rotation_cmd = "/usr/local/bin/rotate-backups -H " +str(backup_rota["Hour"]) + " -d " + \ + str(backup_rota["Day"]) + " -w " + str(backup_rota["Week"]) + " -m " + str(backup_rota["Month"]) + " " + \ + BACKUP_FOLDER + "/" + services_list[service]["Folder"] + job = cron.new(command=rotation_cmd, user = 'root') + job.setall(services_list[service]["Cron"]) + if job.is_valid(): + job.enable() + else: + sys.stderr.write('Err: syntax error in ' + CONFIG_FILE) + sys.exit() + +for job in cron: + print (cron) + +#Ecriture des jobs cron dans la crontab système + +cron.write("/crontab.conf") + diff --git a/pica-backupv2/Dockerfile b/pica-backupv2/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..97d676f216ce90f913968009264168bcc2ac90d5 --- /dev/null +++ b/pica-backupv2/Dockerfile @@ -0,0 +1,24 @@ +FROM registry.picasoft.net:5000/pica-debian + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + postgresql-client \ + mysql-client \ + mongodb-clients \ + mongo-tools \ + python3 \ + cron && \ + apt-get clean && \ + mkdir -p /scripts/config + +COPY backup_env_var.py /scripts +COPY mysql-run.sh /scripts/mysql/mysql-run.sh +COPY postgres-run.sh /scripts/postgresql/postgres-run.sh +COPY mongo-run.sh /scripts/mongodb/mongo-run.sh +COPY run.sh /scripts + +ENV CRON_TIME="0 0 * * *" + +VOLUME ["/backup"] + +CMD ["/scripts/backup_env_var.py"] diff --git a/pica-backupv2/backup_env_var.py b/pica-backupv2/backup_env_var.py new file mode 100755 index 0000000000000000000000000000000000000000..01f222c9b5a2cfc6a3865bfe3111473dcd0c122a --- /dev/null +++ b/pica-backupv2/backup_env_var.py @@ -0,0 +1,87 @@ +#!/usr/bin/python3 +import os +import sys +import json + +#Ajoute la variable correctement dans une variable d'environnement +def add_value_to_key(value, key): + if "MONGO" in key: + if(flag_mongo): + os.environ[key] = os.environ[key] + "," + value + else: + os.environ[key] = value + if "MYSQL" in key: + if(flag_mysql): + os.environ[key] = os.environ[key] + "," + value + else: + os.environ[key] = value + if "POSTGRES" in key: + if(flag_postgres): + os.environ[key] = os.environ[key] + "," + value + else: + os.environ[key] = value + +CONFIG_FOLDER = '/config' +CONFIG_FILE = 'backup_data.json' +PATH_TO_CONFIG = CONFIG_FOLDER + '/' + CONFIG_FILE + +#Garde-fou : test d'existence des fichiers et dossiers +if not(os.path.exists(CONFIG_FOLDER)): + sys.stderr.write('Err: BACKUP_FOLDER or CONFIG_FOLDER doesn\'t exist. \n') + sys.exit() + +if not(os.path.isfile(PATH_TO_CONFIG)): + sys.stderr.write('Err: ' + PATH_TO_CONFIG + ' not found. \n') + sys.exit() + +#Initialisation des variables +services_list = json.load(open(PATH_TO_CONFIG)) +flag_mongo = False +flag_mysql = False +flag_prostgres = False + +for service in services_list: + if services_list[service]["Type"] == "mongo": + #Ajout des clés pour service à DDB Mongo + add_value_to_key(service,"MONGO_SERVICE_NAME_LIST") + add_value_to_key(services_list[service]["Host"],"MONGO_HOST_LIST") + add_value_to_key(services_list[service]["Port"],"MONGO_PORT_LIST") + add_value_to_key(services_list[service]["Database"],"MONGO_DB_LIST") + add_value_to_key(services_list[service]["Init-Backup"],"MONGO_INIT_BACKUP_LIST") + add_value_to_key(services_list[service]["Cron"],"MONGO_CRON_TIME_LIST") + add_value_to_key(services_list[service]["Folder"],"MONGO_BACKUP_FOLDER_LIST") + + flag_mongo = True + + + if services_list[service]["Type"] == "mysql": + #Ajout des clés pour service à DDB MySQL + add_value_to_key(service,"MYSQL_SERVICE_NAME_LIST") + add_value_to_key(services_list[service]["Host"],"MYSQL_HOST_LIST") + add_value_to_key(services_list[service]["Port"],"MYSQL_PORT_LIST") + add_value_to_key(services_list[service]["User"],"MYSQL_USER_LIST") + add_value_to_key(services_list[service]["Password"],"MYSQL_PASS_LIST") + add_value_to_key(services_list[service]["Database"],"MYSQL_DB_LIST") + add_value_to_key(services_list[service]["Init-Backup"],"MYSQL_INIT_BACKUP_LIST") + add_value_to_key(services_list[service]["Cron"],"MYSQL_CRON_TIME_LIST") + add_value_to_key(services_list[service]["Folder"],"MYSQL_BACKUP_FOLDER_LIST") + add_value_to_key(services_list[service]["Options"],"MYSQL_EXTRA_OPTS_LIST") + + flag_mysql = True + + + if services_list[service]["Type"] == "postgres": + #Ajout des clés pour service à DDB Postgre + add_value_to_key(service,"POSTGRES_SERVICE_NAME_LIST") + add_value_to_key(services_list[service]["Host"],"POSTGRES_HOST_LIST") + add_value_to_key(services_list[service]["Port"],"POSTGRES_PORT_LIST") + add_value_to_key(services_list[service]["User"],"POSTGRES_USER_LIST") + add_value_to_key(services_list[service]["Password"],"POSTGRES_PASS_LIST") + add_value_to_key(services_list[service]["Database"],"POSTGRES_DB_LIST") + add_value_to_key(services_list[service]["Init-Backup"],"POSTGRES_INIT_BACKUP_LIST") + add_value_to_key(services_list[service]["Cron"],"POSTGRES_CRON_TIME_LIST") + add_value_to_key(services_list[service]["Folder"],"POSTGRES_BACKUP_FOLDER_LIST") + + flag_postgres = True +#Exécution du script run.sh après avoir enregistré toutes les variables du fichier .json dans des variables d'environnement +os.execv("/scripts/run.sh",['']) diff --git a/pica-backupv2/config/backup_data.json b/pica-backupv2/config/backup_data.json new file mode 100644 index 0000000000000000000000000000000000000000..1771bd3fc67a3773ae03eac17ca62cf8eef6cb19 --- /dev/null +++ b/pica-backupv2/config/backup_data.json @@ -0,0 +1,42 @@ + { + "wekan": + { + "Host": "wekan-db", + "Port": "27017", + "Database": "wekan", + "Type": "mongo", + "Folder": "wekan", + "Cron" : "0 * * * *", + "Init-Backup" : "0", + "Backup-rota": + { + "Hour" : 24, + "Day" : 7, + "Week" : 4, + "Month" : 12 + } + }, + "etherpad": + { + "Host": "etherpad-db", + "Port": "3306", + "User": "root", + "Password": "EPx4fY93ed", + "Database": "--all-databases", + "Type": "mysql", + "Folder": "etherpad", + "Cron" : "0 * * * *", + "Options" : "--single-transaction", + "Init-Backup" : "0", + "Backup-rota": + { + "Hour" : 24, + "Day" : 7, + "Week" : 4, + "Month" : 12 + } + } + } + + + diff --git a/pica-backupv2/mongo-run.sh b/pica-backupv2/mongo-run.sh new file mode 100755 index 0000000000000000000000000000000000000000..96023aeb644d421f1608a3f19d15cc283e9d209a --- /dev/null +++ b/pica-backupv2/mongo-run.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Antoine Barbare 18/02/18 <antoine@barbare.me> + +BACKUP_FOLDER=${BACKUP_FOLDER:-"/backup/"} + +[ -z "${MONGO_HOST}" ] && { echo "=> MONGO_HOST cannot be empty" && exit 1; } +[ -z "${MONGO_PORT}" ] && { echo "=> MONGO_PORT cannot be empty" && exit 1; } +[ -z "${MONGO_DB}" ] && { echo "=> MONGO_DB cannot be empty" && exit 1; } + +if [[ -z "${MONGO_USER}" && -z "${MONGO_PASS}" ]]; then + BACKUP_CMD="mongodump -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} -o $BACKUP_FOLDER"'${BACKUP_NAME}' + RESTORE_CMD="mongorestore -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} " +else + BACKUP_CMD="mongodump -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USER} -p ${MONGO_PASS} -o $BACKUP_FOLDER"'${BACKUP_NAME}' + RESTORE_CMD="mongorestore -d ${MONGO_DB} -h ${MONGO_HOST}:${MONGO_PORT} -u ${MONGO_USER} -p ${MONGO_PASS} " +fi + +########################## +# CREATING BACKUP SCRIPT # +########################## +backup_script_name="${MONGO_SERVICE_NAME}-backup.sh" + +echo "=> ${MONGO_SERVICE_NAME}: Creating backup script" +rm -f "/$backup_script_name" + +cat <<EOF >> "/$backup_script_name" +#!/bin/bash + +BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S) + +echo "=> ${MONGO_SERVICE_NAME}: Backup started: \${BACKUP_NAME}" +if ${BACKUP_CMD} ;then + MONGO_BACKUP=$BACKUP_FOLDER\${BACKUP_NAME} + echo " => Compress files \$MONGO_BACKUP.tar.gz" + tar -czvf \$MONGO_BACKUP.tar.gz \$MONGO_BACKUP && \ + rm -rf \$MONGO_BACKUP && \ + echo "${MONGO_SERVICE_NAME}: Backup succeeded" +else + echo "${MONGO_SERVICE_NAME}: Backup failed" + rm -rf $BACKUP_FOLDER\${BACKUP_NAME} +fi +EOF +chmod +x /$backup_script_name + +########################### +# CREATING RESTORE SCRIPT # +########################### +restore_script_name="${MONGO_SERVICE_NAME}-restore.sh" +backup_log="${MONGO_SERVICE_NAME}_mongo_backup.log" + +echo "=> ${MONGO_SERVICE_NAME}: Creating restore script" +rm -f "/$restore_script_name" + +cat <<EOF >> /$restore_script_name +#!/bin/bash + +echo "=> ${MONGO_SERVICE_NAME}: Restore database from \$1" +echo " => Uncompress save \$1" +tar -xzvf \$1 +output="\$(echo \$1 | awk -F'.tar.gz' '{print \$1}')" +if mongorestore -d ${MONGO_SERVICE_NAME} -h ${MONGO_HOST}:${MONGO_PORT} \$output;then + echo "${MONGO_SERVICE_NAME}: Restore succeeded" +else + echo "${MONGO_SERVICE_NAME}: Restore failed" +fi +rm -Rf \$output +EOF +chmod +x /$restore_script_name + +touch /$backup_log +tail -F /$backup_log & + +if [ -n "${INIT_BACKUP}" ]; then + echo "=> ${MONGO_SERVICE_NAME}: Create a backup on the startup" + /$backup_script_name +elif [ -n "${INIT_RESTORE_LATEST}" ]; then + echo "=> ${MONGO_SERVICE_NAME}: Restore latest backup" + until nc -z $MONGO_HOST $MONGO_PORT + do + echo "waiting database container..." + sleep 1 + done + ls -d -1 $BACKUP_FOLDER* | tail -1 | xargs /$restore_script_name +fi + +echo "${CRON_TIME} /$backup_script_name >> /$backup_log 2>&1" >> /crontab.conf +crontab /crontab.conf +echo "=> ${MONGO_SERVICE_NAME}: Running cron job" diff --git a/pica-backupv2/mysql-run.sh b/pica-backupv2/mysql-run.sh new file mode 100755 index 0000000000000000000000000000000000000000..950feecdb87dd90716f3ff0d732b3cf6fff209b2 --- /dev/null +++ b/pica-backupv2/mysql-run.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +if [ "${MYSQL_ENV_MYSQL_PASS}" == "**Random**" ]; then + unset MYSQL_ENV_MYSQL_PASS +fi + +BACKUP_FOLDER=${BACKUP_FOLDER:-"/backup/"} + +MYSQL_HOST=${MYSQL_PORT_3306_TCP_ADDR:-${MYSQL_HOST}} +MYSQL_HOST=${MYSQL_PORT_1_3306_TCP_ADDR:-${MYSQL_HOST}} +MYSQL_PORT=${MYSQL_PORT_3306_TCP_PORT:-${MYSQL_PORT}} +MYSQL_PORT=${MYSQL_PORT_1_3306_TCP_PORT:-${MYSQL_PORT}} +MYSQL_USER=${MYSQL_USER:-${MYSQL_ENV_MYSQL_USER}} +MYSQL_PASS=${MYSQL_PASS:-${MYSQL_ENV_MYSQL_PASS}} + +[ -z "${MYSQL_HOST}" ] && { echo "=> MYSQL_HOST cannot be empty" && exit 1; } +[ -z "${MYSQL_PORT}" ] && { echo "=> MYSQL_PORT cannot be empty" && exit 1; } +[ -z "${MYSQL_USER}" ] && { echo "=> MYSQL_USER cannot be empty" && exit 1; } +[ -z "${MYSQL_PASS}" ] && { echo "=> MYSQL_PASS cannot be empty" && exit 1; } + +BACKUP_CMD="mysqldump -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} ${EXTRA_OPTS} ${MYSQL_DB} > $BACKUP_FOLDER"'${BACKUP_NAME}' + +########################## +# CREATING BACKUP SCRIPT # +########################## + +backup_script_name="${MYSQL_SERVICE_NAME}-backup.sh" + +echo "=> ${MYSQL_SERVICE_NAME}: Creating backup script" +rm -f "/$backup_script_name" + +cat <<EOF >> "/$backup_script_name" +#!/bin/bash + +DATE=\$(date +\%Y.\%m.\%d.\%H\%M\%S) +BACKUP_NAME=\$DATE.sql + +echo "=> ${MYSQL_SERVICE_NAME}: Backup started: \${BACKUP_NAME}" +if ${BACKUP_CMD} ;then + echo " => Compress files $BACKUP_FOLDER\$DATE.tar.gz" + tar -czvf $BACKUP_FOLDER\$DATE.tar.gz $BACKUP_FOLDER\${BACKUP_NAME} && \ + rm -rf $BACKUP_FOLDER\$BACKUP_NAME && \ + echo "${MYSQL_SERVICE_NAME}: Backup succeeded" +else + echo "${MYSQL_SERVICE_NAME}: Backup failed" + rm -rf ${BACKUP_FOLDER}\${BACKUP_NAME} +fi +EOF +chmod +x /$backup_script_name + +########################### +# CREATING RESTORE SCRIPT # +########################### + +restore_script_name="${MYSQL_SERVICE_NAME}-restore.sh" + +echo "=> ${MYSQL_SERVICE_NAME}: Creating restore script" +rm -f "/$restore_script_name" + +cat <<EOF >> /$restore_script_name +#!/bin/bash +echo "=> ${MYSQL_SERVICE_NAME}: Restore database from \$1" +tar -xzvf \$1 +output="\$(echo \$1 | awk -F'.tar.gz' '{print \$1".sql"}')" +if mysql -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} < \$output ;then + echo "${MYSQL_SERVICE_NAME}: Restore succeeded" +else + echo "${MYSQL_SERVICE_NAME}: Restore failed" +fi +rm -Rf \$output +EOF +chmod +x /$restore_script_name + +backup_log="${MYSQL_SERVICE_NAME}_mysql_backup.log" + +touch /$backup_log +tail -F /$backup_log & + +if [ -n "${INIT_BACKUP}" ]; then + echo "=> ${MYSQL_SERVICE_NAME}: Create a backup on the startup" + /${backup_script_name} +elif [ -n "${INIT_RESTORE_LATEST}" ]; then + echo "=> ${MYSQL_SERVICE_NAME}: Restore latest backup" + until nc -z $MYSQL_HOST $MYSQL_PORT + do + echo "waiting database container..." + sleep 1 + done + ls -d -1 $BACKUP_FOLDER* | tail -1 | xargs /$restore_script_name +fi + +##################### +# SET THE CRON RULE # +##################### +echo "${CRON_TIME} /$backup_script_name >> /$backup_log 2>&1" >> /crontab.conf +crontab /crontab.conf +echo "=> ${MYSQL_SERVICE_NAME}: Running cron job" diff --git a/pica-backupv2/postgres-run.sh b/pica-backupv2/postgres-run.sh new file mode 100755 index 0000000000000000000000000000000000000000..4d57837370e0a4eb2e38df56060a77373cfd16ff --- /dev/null +++ b/pica-backupv2/postgres-run.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +BACKUP_FOLDER=${BACKUP_FOLDER:-"/backup/"} + +[ -z "${POSTGRES_HOST}" ] && { echo "=> POSTGRES_HOST cannot be empty" && exit 1; } +[ -z "${POSTGRES_PORT}" ] && { echo "=> POSTGRES_PORT cannot be empty" && exit 1; } +[ -z "${POSTGRES_USER}" ] && { echo "=> POSTGRES_USER cannot be empty" && exit 1; } +[ -z "${POSTGRES_PASS}" ] && { echo "=> POSTGRES_PASS cannot be empty" && exit 1; } +[ -z "${POSTGRES_DB}" ] && { echo "=> POSTGRES_DB cannot be empty" && exit 1; } + +BACKUP_CMD="pg_dump -w -c > $BACKUP_FOLDER"'${BACKUP_NAME}' + +########################## +# CREATING BACKUP SCRIPT # +########################## +backup_script_name="${POSTGRES_SERVICE_NAME}-backup.sh" + +echo "=> ${POSTGRES_SERVICE_NAME}: Creating backup script" +rm -f "/$backup_script_name" + +cat <<EOF >> "/$backup_script_name" +#!/bin/bash +export PGHOST=$POSTGRES_HOST +export PGPORT=$POSTGRES_PORT +export PGDATABASE=$POSTGRES_DB +export PGUSER=$POSTGRES_USER +export PGPASSWORD=$POSTGRES_PASS + +DATE=\$(date +\%Y.\%m.\%d.\%H\%M\%S) +BACKUP_NAME=\$DATE.sql + +echo "=> ${POSTGRES_SERVICE_NAME}: Backup started: \${BACKUP_NAME}" +if ${BACKUP_CMD} ;then + echo " => Compress files $BACKUP_FOLDER\$DATE.tar.gz" + tar -czvf $BACKUP_FOLDER\$DATE.tar.gz $BACKUP_FOLDER\${BACKUP_NAME} && \ + rm -rf $BACKUP_FOLDER\$BACKUP_NAME && \ + echo "${POSTGRES_SERVICE_NAME}: Backup succeeded" +else + echo "${POSTGRES_SERVICE_NAME}: Backup failed" + rm -rf $BACKUP_FOLDER\${BACKUP_NAME} +fi +EOF +chmod +x /$backup_script_name + +########################### +# CREATING RESTORE SCRIPT # +########################### +restore_script_name="${POSTGRES_SERVICE_NAME}-restore.sh" +backup_log="${POSTGRES_SERVICE_NAME}_postgres_backup.log" + +echo "=> ${POSTGRES_SERVICE_NAME}: Creating restore script" +rm -f "/$restore_script_name" + +cat <<EOF >> /$restore_script_name +#!/bin/bash +export PGHOST=$POSTGRES_HOST +export PGPORT=$POSTGRES_PORT +export PGDATABASE=$POSTGRES_DB +export PGUSER=$POSTGRES_USER +export PGPASSWORD=$POSTGRES_PASS + +echo "=> ${POSTGRES_SERVICE_NAME}: Restore database from \$1" +tar -xzvf \$1 +output="\$(echo \$1 | awk -F'.tar.gz' '{print \$1".sql"}')" +if psql < \$output ;then + echo "${POSTGRES_SERVICE_NAME}: Restore succeeded" +else + echo "${POSTGRES_SERVICE_NAME}: Restore failed" +fi +rm -Rf \$output +EOF +chmod +x /$restore_script_name + +touch /$backup_log +tail -F /$backup_log & + +if [ -n "${INIT_BACKUP}" ]; then + echo "=> ${POSTGRES_SERVICE_NAME}: Create a backup on the startup" + /$backup_script_name +elif [ -n "${INIT_RESTORE_LATEST}" ]; then + echo "=> ${POSTGRES_SERVICE_NAME}: Restore latest backup" + until nc -z $POSTGRES_HOST $POSTGRES_PORT + do + echo "waiting database container..." + sleep 1 + done + ls -d -1 $BACKUP_FOLDER* | tail -1 | xargs /$restore_script_name +fi + +echo "${CRON_TIME} /$backup_script_name >> /$backup_log 2>&1" >> /crontab.conf +crontab /crontab.conf +echo "=> ${POSTGRES_SERVICE_NAME}: Running cron job" diff --git a/pica-backupv2/run.sh b/pica-backupv2/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a0f586017d7bcc3144e66379a0d04b5893cc61f --- /dev/null +++ b/pica-backupv2/run.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# DB_TYPE : MYSQL|POSTGRES + +# MYSQL POSTGRESQL MONGODB +# MYSQL_SERVICE_NAME_LIST POSTGRES_SERVICE_NAME_LIST MONGO_SERVICE_NAME_LIST +# MYSQL_HOST_LIST POSTGRES_HOST_LIST MONGO_HOST_LIST +# MYSQL_PORT_LIST POSTGRES_PORT_LIST MONGO_PORT_LIST +# MYSQL_USER_LIST POSTGRES_USER_LIST MONGO_USER_LIST +# MYSQL_PASS_LIST POSTGRES_PASS_LIST MONGO_PASS_LIST +# MYSQL_DB_LIST POSTGRES_DB_LIST MONGO_DB_LIST +# MYSQL_INIT_BACKUP_LIST POSTGRES_INIT_BACKUP_LIST MONGO_INIT_BACKUP_LIST +# MYSQL_CRON_TIME_LIST POSTGRES_CRON_TIME_LIST MONGO_CRON_TIME_LIST +# MYSQL_EXTRA_OPTS_LIST POSTGRES_EXTRA_OPTS-LIST MONGO_BACKUP_FOLDER_LIST +# MYSQL_BACKUP_FOLDER_LIST POSTGRESQL_BACKUP_FOLDER_LIST + +##################### +# Create the arrays # +##################### + +IFS=, +#MYSQL +read -a MYSQL_HOST_ARRAY <<< "$MYSQL_HOST_LIST" +read -a MYSQL_SERVICE_NAME_ARRAY <<< "$MYSQL_SERVICE_NAME_LIST" +read -a MYSQL_PORT_ARRAY <<< "$MYSQL_PORT_LIST" +read -a MYSQL_USER_ARRAY <<< "$MYSQL_USER_LIST" +read -a MYSQL_PASS_ARRAY <<< "$MYSQL_PASS_LIST" +read -a MYSQL_DB_ARRAY <<< "$MYSQL_DB_LIST" +read -a MYSQL_INIT_BACKUP_ARRAY <<< "$MYSQL_INIT_BACKUP_LIST" +read -a MYSQL_CRON_TIME_ARRAY <<< "$MYSQL_CRON_TIME_LIST" +read -a MYSQL_EXTRA_OPTS_ARRAY <<< "$MYSQL_EXTRA_OPTS_LIST" +read -a MYSQL_BACKUP_FOLDER_ARRAY <<< "$MYSQL_BACKUP_FOLDER_LIST" + +#POSTGRES +read -a POSTGRES_HOST_ARRAY <<< "$POSTGRES_HOST_LIST" +read -a POSTGRES_SERVICE_NAME_ARRAY <<< "$POSTGRES_SERVICE_NAME_LIST" +read -a POSTGRES_PORT_ARRAY <<< "$POSTGRES_PORT_LIST" +read -a POSTGRES_USER_ARRAY <<< "$POSTGRES_USER_LIST" +read -a POSTGRES_PASS_ARRAY <<< "$POSTGRES_PASS_LIST" +read -a POSTGRES_DB_ARRAY <<< "$POSTGRES_DB_LIST" +read -a POSTGRES_INIT_BACKUP_ARRAY <<< "$POSTGRES_INIT_BACKUP_LIST" +read -a POSTGRES_CRON_TIME_ARRAY <<< "$POSTGRES_CRON_TIME_LIST" +read -a POSTGRES_EXTRA_OPTS_ARRAY <<< "$POSTGRES_EXTRA_OPTS_LIST" +read -a POSTGRES_BACKUP_FOLDER_ARRAY <<< "$POSTGRES_BACKUP_FOLDER_LIST" + +#MONGO +read -a MONGO_HOST_ARRAY <<< "$MONGO_HOST_LIST" +read -a MONGO_SERVICE_NAME_ARRAY <<< "$MONGO_SERVICE_NAME_LIST" +read -a MONGO_PORT_ARRAY <<< "$MONGO_PORT_LIST" +read -a MONGO_USER_ARRAY <<< "$MONGO_USER_LIST" +read -a MONGO_PASS_ARRAY <<< "$MONGO_PASS_LIST" +read -a MONGO_DB_ARRAY <<< "$MONGO_DB_LIST" +read -a MONGO_INIT_BACKUP_ARRAY <<< "$MONGO_INIT_BACKUP_LIST" +read -a MONGO_CRON_TIME_ARRAY <<< "$MONGO_CRON_TIME_LIST" +read -a MONGO_BACKUP_FOLDER_ARRAY <<< "$MONGO_BACKUP_FOLDER_LIST" + +########################### +# Call the backup scripts # +########################### + +# Define the scripts location + +#Execute MySQL backups +NB_CALLS=${#MYSQL_HOST_ARRAY[@]} + +for (( i=0; i < $NB_CALLS; i++ )) +do + MYSQL_SERVICE_NAME=${MYSQL_SERVICE_NAME_ARRAY[i]} \ + MYSQL_HOST=${MYSQL_HOST_ARRAY[i]} \ + MYSQL_PORT=${MYSQL_PORT_ARRAY[i]} \ + MYSQL_USER=${MYSQL_USER_ARRAY[i]} \ + MYSQL_PASS=${MYSQL_PASS_ARRAY[i]} \ + MYSQL_DB=${MYSQL_DB_ARRAY[i]} \ + INIT_BACKUP=${MYSQL_INIT_BACKUP_ARRAY[i]} \ + CRON_TIME=${MYSQL_CRON_TIME_ARRAY[i]} \ + EXTRA_OPTS=${MYSQL_EXTRA_OPTS_ARRAY[i]} \ + BACKUP_FOLDER="/backup/${MYSQL_BACKUP_FOLDER_ARRAY[i]}/" \ + /scripts/mysql/mysql-run.sh +done + +#Execute PostgreSQL backups +NB_CALLS=${#POSTGRES_HOST_ARRAY[@]} + +for (( i=0; i < $NB_CALLS; i++ )) +do + POSTGRES_SERVICE_NAME=${POSTGRES_SERVICE_NAME_ARRAY[i]} \ + POSTGRES_HOST=${POSTGRES_HOST_ARRAY[i]} \ + POSTGRES_PORT=${POSTGRES_PORT_ARRAY[i]} \ + POSTGRES_USER=${POSTGRES_USER_ARRAY[i]} \ + POSTGRES_PASS=${POSTGRES_PASS_ARRAY[i]} \ + POSTGRES_DB=${POSTGRES_DB_ARRAY[i]} \ + INIT_BACKUP=${POSTGRES_INIT_BACKUP_ARRAY[i]} \ + CRON_TIME=${POSTGRES_CRON_TIME_ARRAY[i]} \ + EXTRA_OPTS=${POSTGRES_EXTRA_OPTS_ARRAY[i]} \ + BACKUP_FOLDER="/backup/${POSTGRES_BACKUP_FOLDER_ARRAY[i]}/" \ + /scripts/postgresql/postgres-run.sh +done + +#Execute MongoDB backups +NB_CALLS=${#MONGO_HOST_ARRAY[@]} + +for (( i=0; i < $NB_CALLS; i++ )) +do + MONGO_SERVICE_NAME=${MONGO_SERVICE_NAME_ARRAY[i]} \ + MONGO_HOST=${MONGO_HOST_ARRAY[i]} \ + MONGO_PORT=${MONGO_PORT_ARRAY[i]} \ + MONGO_USER=${MONGO_USER_ARRAY[i]} \ + MONGO_PASS=${MONGO_PASS_ARRAY[i]} \ + MONGO_DB=${MONGO_DB_ARRAY[i]} \ + INIT_BACKUP=${MONGO_INIT_BACKUP_ARRAY[i]} \ + CRON_TIME=${MONGO_CRON_TIME_ARRAY[i]} \ + BACKUP_FOLDER="/backup/${MONGO_BACKUP_FOLDER_ARRAY[i]}/" \ + /scripts/mongodb/mongo-run.sh +done + +exec cron -f