Redirect STDERR to logfile too.

.env.example

@@ -0,0 +1,23 @@
+# mysql
+MYSQL_ROOT_PASSWORD=justfortesting
+
+# phpmyadmin
+PMA_HOST=mysql
+
+# backup-s3
+CRON_EXPR="* * * * *"
+S3_ACCESS_KEY=your_access_key
+S3_SECRET_KEY=your_secret_key
+S3_BUCKET_NAME=your_bucket_name
+S3_REGION=eu-west-3
+S3_ROOT_PATH=@MYSQL_HOST_ALIAS/%Y%m/@databaseName
+S3_DATABASE_PATH=%Y-%m-%d
+S3_DATABASE_FILENAME=%Y-%m-%dT%H-%M-@databaseName
+S3_TABLE_PATH=%Y-%m-%d/%H-%M-tables
+S3_TABLE_FILENAME=%Y-%m-%dT%H-%M-@tableName
+# Possible values: TABLES, DATABASE, ALL
+S3_BACKUP_TYPE=ALL
+MYSQL_HOST=mysql
+MYSQL_HOST_ALIAS=mysql-local-test
+MYSQL_BACKUP_USER=root
+MYSQL_BACKUP_PASS=justfortesting

.github/workflows/release.yml

@@ -24,33 +24,15 @@ jobs:
           file-to-check: version.txt
           only-return-version: true
 
-      - name: push-image
-        env:
-          PROJECT_VERSION: ${{ steps.get_project_version.outputs.version }}
-        run: |
-          docker login ${{ secrets.DOCKER_REGISTRY }} -u ${{ secrets.DOCKER_REGISTRY_USER }} -p ${{ secrets.DOCKER_REGISTRY_TOKEN }}
-          docker build -t ${{ secrets.DOCKER_REGISTRY }}/${GITHUB_REPOSITORY}:${PROJECT_VERSION} -t ${{ secrets.DOCKER_REGISTRY }}/${GITHUB_REPOSITORY}:latest .
-          docker push ${{ secrets.DOCKER_REGISTRY }}/${GITHUB_REPOSITORY}:${PROJECT_VERSION}
-          if [[ "$PROJECT_VERSION" != *"SNAPSHOT"* || "$PROJECT_VERSION" != *"RC"* ]]; then
-            docker push ${{ secrets.DOCKER_REGISTRY }}/${GITHUB_REPOSITORY}:latest
-          fi
-
-      - name: setup-github-release
-        id: setup_github_release
-        env:
-          PROJECT_VERSION: ${{ steps.get_project_version.outputs.version }}
-        run: |
-          echo ::set-output name=gitcommitmessage::$(git log --no-merges -1 --oneline)
-          if [[ "$PROJECT_VERSION" == *"SNAPSHOT"* || "$PROJECT_VERSION" == *"RC"* ]]; then
-            echo ::set-output name=isprerelease::true
-          fi
+      - name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_REGISTRY_USER }}
+          password: ${{ secrets.DOCKER_REGISTRY_TOKEN }}
 
-      - name: github-release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GPR_TOKEN }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
         with:
-          tag_name: ${{ steps.get_project_version.outputs.version }}
-          release_name: ${{ steps.get_project_version.outputs.version }}
-          body: ${{ steps.setup_github_release.outputs.gitcommitmessage }}
-          prerelease: ${{ steps.setup_github_release.outputs.isprerelease }}
+          push: true
+          tags: computablefacts/mysql-s3-backup:${{ steps.get_project_version.outputs.version }},computablefacts/mysql-s3-backup:latest

.gitignore

@@ -1,3 +1,4 @@
 /.project
 /.idea
 /*.iml
+.env

README.md

@@ -1,10 +1,25 @@
 # mysql-s3-backup
-[![Build](https://github.com/avides/mysql-s3-backup/workflows/release/badge.svg)](https://github.com/avides/mysql-s3-backup/actions)
-[![Nightly build](https://github.com/avides/mysql-s3-backup/workflows/nightly/badge.svg)](https://github.com/avides/mysql-s3-backup/actions)
-[![Docker Pulls](https://img.shields.io/docker/pulls/avides/mysql-s3-backup.svg)](https://hub.docker.com/r/avides/mysql-s3-backup)
+[![Build](https://github.com/computablefacts/mysql-s3-backup/workflows/release/badge.svg)](https://github.com/computablefacts/mysql-s3-backup/actions)
+[![Docker Pulls](https://img.shields.io/docker/pulls/computablefacts/mysql-s3-backup.svg)](https://hub.docker.com/r/computablefacts/mysql-s3-backup)
 
 This script is used to backup a MySQL Server, ZIP the data and upload it to Amazon S3. It will dump, zip and upload each table in a separate "sql.gz"-file. This script will be executed by a specified cron expression.
 
+## TODOS
+
+I fork this repo to:
+
+- [x] add STDERR output to logfile
+- [x] allow `/` in cron expression like in "*/15 * * * *"
+- [x] allow to set AWS credential from a Docker secret (linked to /root/.aws/credentials)
+- [x] add an environment variable to set the S3 path
+- [x] add an environment variable to set the table dump filename
+- [x] add an environment variable to make a dump of the whole database
+- [x] add an environment variable to set the database dump filename
+- [x] add an environment variable to set the database dump path
+- [x] add an environment variable to set the table dump path
+- [x] add `--single-transaction --quick` options to mysqldump 
+- [x] add `--force` options to mysqldump to dump views (and not stopping when something goes wrong)
+
 ## Requirements
 
 - Docker
@@ -28,7 +43,7 @@ docker run \
 --env MYSQL_BACKUP_USER="root" \
 --env MYSQL_BACKUP_PASS="MYSQL_BACKUP_PASS" \
 -p 9300:9300 \
-avides/mysql-s3-backup:2.6.0
+computablefacts/mysql-s3-backup:2.7.0
 ```
 
 ### Build & Test
@@ -59,6 +74,33 @@ docker logs -f mysql-s3-backup
 docker exec -it mysql-s3-backup /bin/bash
 ```
 
+### Build & Test with Docker compose
+
+Make an `.env` file from `.env.example`:
+
+```bash
+cp .env.example .env
+```
+
+Change your AWS settings:
+```
+S3_ACCESS_KEY=your_access_key
+S3_SECRET_KEY=your_secret_key
+S3_BUCKET_NAME=your_bucket_name
+S3_REGION=eu-west-3
+```
+
+Start the containers:
+```bash
+docker-compose up
+```
+
+This will start a MySQL server, a phpMyAdmin and the mysql-s3-backup containers.
+You can use phpMyAdmin, found at http://localhost:8099/, to create some databases and tables.
+Then you could see the backups done on your S3 bucket. 
+
+Happy testing!
+
 ## Metrics
 
 This container delivers some metrics about the MySQL backups. The metrics are available to the HTTP endpoint "/metrics" on port 9300. Following metrics are collected:

backup.sh

@@ -5,13 +5,17 @@ source functions.sh
 
 ## Check if variables are given
 if [ -z ${S3_ACCESS_KEY+x} ]; then
-    echo "ERROR: Please set the S3_ACCESS_KEY!"
-    exit 1
+    echo "WARNING: Please set the S3_ACCESS_KEY or create /root/.aws/credentials file!"
+else
+    ## Set S3 environment variables
+    export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY
 fi
 
 if [ -z ${S3_SECRET_KEY+x} ]; then
-    echo "ERROR: Please set the S3_SECRET_KEY!"
-    exit 1
+    echo "WARNING: Please set the S3_SECRET_KEY or create /root/.aws/credentials file!"
+else
+    ## Set S3 environment variables
+    export AWS_SECRET_ACCESS_KEY=$S3_SECRET_KEY
 fi
 
 if [ -z ${S3_REGION+x} ]; then
@@ -24,6 +28,31 @@ if [ -z ${S3_BUCKET_NAME+x} ]; then
     exit 1
 fi
 
+if [ -z ${S3_ROOT_PATH+x} ]; then
+    echo "Use default S3_ROOT_PATH..."
+    export S3_ROOT_PATH=@MYSQL_HOST_ALIAS/@databaseName
+fi
+
+if [ -z ${S3_DATABASE_PATH+x} ]; then
+    echo "Use default S3_DATABASE_PATH..."
+    export S3_DATABASE_PATH=/
+fi
+
+if [ -z ${S3_DATABASE_FILENAME+x} ]; then
+    echo "Use default S3_DATABASE_FILENAME..."
+    export S3_DATABASE_FILENAME=%Y-%m-%d-%H-%M-@databaseName
+fi
+
+if [ -z ${S3_TABLE_PATH+x} ]; then
+    echo "Use default S3_TABLE_PATH..."
+    export S3_TABLE_PATH=%Y-%m-%d-%H-%M-tables
+fi
+
+if [ -z ${S3_TABLE_FILENAME+x} ]; then
+    echo "Use default S3_TABLE_FILENAME..."
+    export S3_TABLE_FILENAME=@tableName
+fi
+
 if [ -z ${MYSQL_HOST_ALIAS+x} ]; then
     echo "Use default MySQL host alias: $MYSQL_HOST"
     MYSQL_HOST_ALIAS=$MYSQL_HOST
@@ -54,8 +83,6 @@ BACKUP_PATH="/root/backups"
 BUCKET_NAME=${S3_BUCKET_NAME}
 
 ## Set S3 environment variables
-export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY
-export AWS_SECRET_ACCESS_KEY=$S3_SECRET_KEY
 export AWS_DEFAULT_REGION=$S3_REGION
 
 ## MySQL-Servers

crontab

@@ -1,2 +1,2 @@
-CRON_EXPR	root . /root/project_env.sh; /root/backup.sh >> /var/log/backup-cron/cron.log
+CRON_EXPR	root . /root/project_env.sh; /root/backup.sh >> /var/log/backup-cron/cron.log 2>&1
 # An empty line is required at the end of this file for a valid cron file.

docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3.7'
+
+services:
+  mysql:
+    image: mysql:8.0
+    env_file:
+      - .env
+
+  phpmyadmin:
+    depends_on:
+      - mysql
+    image: phpmyadmin
+    env_file:
+      - .env
+    ports:
+      - "8099:80"
+
+  backup-s3:
+    depends_on:
+      - mysql
+    build: .
+    env_file:
+      - .env

entrypoint.sh

@@ -1,6 +1,6 @@
 ## Setup
-printenv | grep -E 'PATH|S3_BUCKET_NAME|S3_ACCESS_KEY|S3_SECRET_KEY|S3_REGION|MYSQL_HOST|MYSQL_PORT|MYSQL_BACKUP_USER|MYSQL_BACKUP_PASS' | sed 's/^\(.*\)$/export \1"/g' | sed '/=/s//="/' > /root/project_env.sh
-sed -i -e "s/CRON_EXPR/${CRON_EXPR}/g" /etc/cron.d/backup-cron
+printenv | grep -E 'PATH|S3_|MYSQL_' | sed 's/^\(.*\)$/export \1"/g' | sed '/=/s//="/' > /root/project_env.sh
+sed -i -e "s#CRON_EXPR#${CRON_EXPR}#g" /etc/cron.d/backup-cron
 
 ## Start metrics endpoint
 mkdir /var/log/webserver