add master db container

Author: hmil

README.md

@@ -11,22 +11,34 @@ backend as well as replication slaves, backup and deployment scripts.
 
 This repository assumes a regular linux distro with the following programs accessible on the cli:
 - docker
-- mysql
+- mysql (+ mysqldump)
 
 ## Usage
 
 TBD. The general idea would be to have frontend.sh expose all functionnalities although
 sub-projects should be as disconnected as possible.
 
-For now, one can create a slave instance like this:
 
-`./frontend.sh slave setup # Creates and starts a replication slave`
+### Replication slave
+
+Set up a replication slave like this:
+
+`sudo ./frontend.sh slave setup`
 
 And remove it with
+
 `./frontend.sh slave remove`
 
-The container can be started/stopped using the start and stop commands
-```sh
-./frontend.sh slave stop
-./frontend.sh slave start
-```
+Additionnal commands available: `stop`, `start`
+
+(Notice: you must take care of `start`ing the instance after a reboot yourself)
+
+
+### Master database
+
+Master DB is deployed like this:
+
+`sudo ./frontend.sh master setup -b backup_db.sql` where `backup_db.sql` is a sql
+file containing a backup of the production data.
+
+Additionnal commands: `stop`, `start`, `remove`

db/master/conf/master_config.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+server-id	= 1
+expire_logs_days	= 10
+binlog_do_db	= master
+log_bin			= /var/lib/mysql/mysql-bin.log

db/master/master.sh

@@ -0,0 +1,87 @@
+#!/bin/sh -e
+
+. ../utils.sh
+
+check_env() {
+  if [ -z "$MASTER_ROOT_PASSWORD" ] || [ -z "$MASTER_DB_PASSWORD" ]; then
+    echo "An environment variable is missing. Run --help for usage info." 1>&2
+    return 1
+  fi
+  if [ -z "$MASTER_EXPORTED_PORT" ]; then
+    export MASTER_EXPORTED_PORT=3306
+  fi
+  if [ -z "$MASTER_DB_CONTAINER_NAME" ]; then
+    export MASTER_DB_CONTAINER_NAME="courseadvisor-master"
+  fi
+}
+
+setup() {
+  # Create mysql docker image
+  docker run --name "$MASTER_DB_CONTAINER_NAME" -p $MASTER_EXPORTED_PORT:3306 -e "MYSQL_ROOT_PASSWORD=$MASTER_ROOT_PASSWORD" \
+    -e MYSQL_DATABASE=master -e MYSQL_USER=master -e "MYSQL_PASSWORD=$MASTER_DB_PASSWORD" -v `pwd`/conf:/etc/mysql/conf.d -d mysql:5.5
+  echo "Waiting for mysql deamon to start"
+  wait_for_mysql $MASTER_EXPORTED_PORT "$MASTER_ROOT_PASSWORD" 1>/dev/null
+  # Creating slave user
+  mysql -P$MASTER_EXPORTED_PORT -uroot "-p$MASTER_ROOT_PASSWORD" "-h127.0.0.1" <<EOF
+GRANT RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO '$SLAVE_REPLICATION_USER'@'%' IDENTIFIED BY '$SLAVE_REPLICATION_PASSWORD';
+GRANT SELECT, LOCK TABLES, SHOW VIEW ON \`master\`.* TO '$SLAVE_REPLICATION_USER'@'%';
+EOF
+  echo "Master database is running"
+}
+
+stop() {
+  docker stop "$MASTER_DB_CONTAINER_NAME"
+}
+
+remove() {
+  stop
+  docker rm "$MASTER_DB_CONTAINER_NAME"
+}
+
+start() {
+  docker start "$MASTER_DB_CONTAINER_NAME"
+}
+
+usage() {
+  cat 1>&2 <<EOF
+  usage: master (setup [-b <sqlfile>]|remove)
+
+Sets up or removes the master db container. Sensible arguments are passed via env variables:
+
+mandatory:
+MASTER_DB_PASSWORD          : Password for master user
+MASTER_ROOT_PASSWORD        : Password for root user
+
+optionnal:
+MASTER_EXPORTED_PORT : Which host port to bind to master's SQL (default 3306)
+MASTER_DB_CONTAINER_NAME: Docker container name for the master (default courseadvisor-master)
+
+options:
+-b <sqlfile> : executes the sql statements in sqlfile using master database
+EOF
+}
+
+check_env
+
+case "$1" in
+  "setup")
+     setup
+     shift
+     if [ "$#" -ge 2 ] && [ "$1" = "-b" ]; then
+       echo "Seeding database with $2"
+       echo "use master;" | cat - "$2" | mysql -P$MASTER_EXPORTED_PORT -umaster "-p$MASTER_DB_PASSWORD" "-h127.0.0.1"
+     fi
+  ;;
+  "remove")
+    remove
+  ;;
+  "stop")
+    stop
+  ;;
+  "start")
+    start
+  ;;
+  *)
+    usage
+  ;;
+esac

db/slave/slave.sh

@@ -1,74 +1,53 @@
 #!/bin/sh -e
-# server-id   = 2
-# relay-log   = /var/log/mysql/mysql-relay-bin.log
-# log_bin     = /var/log/mysql/mysql-bin.log
-# replicate-do-db   = courseadvisor
-# report-host = hmil.fr
-#
-# position: 107
+
+. ../utils.sh
 
 check_env() {
-  if [ -z "$SLAVE_ROOT_PASSWORD" ] || [ -z "$SLAVE_REPLICATION_PASSWORD" ] || [ -z "$SLAVE_REPLICATION_USER" ]; then
+  if [ -z "$SLAVE_ROOT_PASSWORD" ] || [ -z "$SLAVE_REPLICATION_PASSWORD" ] || [ -z "$SLAVE_REPLICATION_USER" ] \
+    || [ -z "$MASTER_HOST" ]; then
     echo "An environment variable is missing. Run --help for usage info." 1>&2
     return 1
   fi
   if [ -z "$SLAVE_EXPORTED_PORT" ]; then
     export SLAVE_EXPORTED_PORT=3300
   fi
-  if [ -z "$SLAVE_CONTAINER_NAME" ]; then
-    export SLAVE_CONTAINER_NAME="courseadvisor-slave"
+  if [ -z "$SLAVE_DB_CONTAINER_NAME" ]; then
+    export SLAVE_DB_CONTAINER_NAME="courseadvisor-slave"
   fi
 }
 
-wait_for_mysql() {
-  set +e
-  failure=1
-  tries=30
-  while [ "$failure" -ne 0 ]; do
-    tries=`expr $tries - 1`
-    if [ $tries -eq 0 ]; then
-      echo "Could not connect to mysql server" 1>&2
-      return 1
-    fi
-    sleep 1
-    echo 'select 1;' | mysql -P$SLAVE_EXPORTED_PORT -h"127.0.0.1" -uroot -p$SLAVE_ROOT_PASSWORD 2>/dev/null
-    failure=$?
-  done
-  return 0
-}
-
 setup() {
   # prepare configuration with random slave number
   server_id=`tr -cd 0-9 </dev/urandom | head -c 6`
   server_name=`hostname`
   cat slave_config.cnf | sed "s/__SERVER_ID__/$server_id/" | sed "s/__SERVER_NAME__/$server_name/" > conf/slave_config.cnf
   # Create mysql docker image
-  docker run --name ${SLAVE_CONTAINER_NAME} -p ${SLAVE_EXPORTED_PORT}:3306 -e MYSQL_ROOT_PASSWORD=${SLAVE_ROOT_PASSWORD} -e MYSQL_DATABASE=master -v `pwd`/conf:/etc/mysql/conf.d -d mysql:5.5
+  docker run --name "$SLAVE_DB_CONTAINER_NAME" -p $SLAVE_EXPORTED_PORT:3306 -e MYSQL_ROOT_PASSWORD="$SLAVE_ROOT_PASSWORD" -e MYSQL_DATABASE=master -v `pwd`/conf:/etc/mysql/conf.d -d mysql:5.5
   echo "Waiting for mysql deamon to start"
-  wait_for_mysql 1>/dev/null
+  wait_for_mysql $SLAVE_EXPORTED_PORT "$SLAVE_ROOT_PASSWORD" 1>/dev/null
   echo "Dumping master state (warning: locks master DB)"
   # Seeds database with snapshot data, replication log position is set to take over right after this snapshot
-  mysqldump -P5467 -u${SLAVE_REPLICATION_USER} -h"courseadvisor.ch" -p${SLAVE_REPLICATION_PASSWORD} --master-data -B master |
-    sed "s/CHANGE MASTER TO/CHANGE MASTER TO MASTER_HOST='courseadvisor.ch',MASTER_PORT=5467,MASTER_USER='${SLAVE_REPLICATION_USER}',MASTER_PASSWORD='${SLAVE_REPLICATION_PASSWORD}',/" |
-    mysql -P${SLAVE_EXPORTED_PORT} -h"127.0.0.1" -uroot -p${SLAVE_ROOT_PASSWORD}
+  mysqldump -P$MASTER_EXPORTED_PORT "-u$SLAVE_REPLICATION_USER" "-h$MASTER_HOST" "-p$SLAVE_REPLICATION_PASSWORD" --master-data -B master |
+    sed "s/CHANGE MASTER TO /CHANGE MASTER TO MASTER_HOST='$MASTER_HOST',MASTER_PORT=$MASTER_EXPORTED_PORT,MASTER_USER='$SLAVE_REPLICATION_USER',MASTER_PASSWORD='$SLAVE_REPLICATION_PASSWORD',/" |
+    mysql -P$SLAVE_EXPORTED_PORT -h"127.0.0.1" -uroot "-p$SLAVE_ROOT_PASSWORD"
 
   echo "Starting slave replication"
   # Sets up remote master
-  echo "START SLAVE;" | mysql -P${SLAVE_EXPORTED_PORT} -h"127.0.0.1" -uroot -p${SLAVE_ROOT_PASSWORD}
+  echo "START SLAVE;" | mysql -P$SLAVE_EXPORTED_PORT "-h127.0.0.1" -uroot "-p$SLAVE_ROOT_PASSWORD"
   echo "Slave instance is running"
 }
 
 stop() {
-  docker stop $SLAVE_CONTAINER_NAME
+  docker stop "$SLAVE_DB_CONTAINER_NAME"
 }
 
 remove() {
   stop
-  docker rm $SLAVE_CONTAINER_NAME
+  docker rm "$SLAVE_DB_CONTAINER_NAME"
 }
 
 start() {
-  docker start $SLAVE_CONTAINER_NAME
+  docker start "$SLAVE_DB_CONTAINER_NAME"
 }
 
 usage() {
@@ -81,10 +60,11 @@ mandatory:
 SLAVE_REPLICATION_USER     : Replication user on master host
 SLAVE_REPLICATION_PASSWORD : Replication user password on master host
 SLAVE_ROOT_PASSWORD        : Password for root user on local slave instance
+MASTER_HOST                : Host of the master server
 
 optionnal:
 SLAVE_EXPORTED_PORT : Which host port to bind to slave's SQL (default 3300)
-SLAVE_CONTAINER_NAME: Docker container name for the slave (default courseadvisor-slave)
+SLAVE_DB_CONTAINER_NAME: Docker container name for the slave (default courseadvisor-slave)
 EOF
 }
 

db/slave/slave_config.cnf

@@ -1,6 +1,5 @@
 [mysqld]
 server-id   = __SERVER_ID__
-#relay-log   = /var/log/mysql/mysql-relay-bin.log
-#log_bin     = /var/log/mysql/mysql-bin.log
+relay-log   = /var/lib/mysql/mysql-relay-bin.log
 replicate-do-db   = master
 report_host = __SERVER_NAME__

db/utils.sh

@@ -0,0 +1,22 @@
+wait_for_mysql() {
+  set +e
+  if [ $# -ne 2 ]; then
+    echo "usage: wait_for_mysql <port> <root_pwd>" 1>&2
+    return 1
+  fi
+  port="$1"
+  root_pwd="$2"
+  failure=1
+  tries=30
+  while [ "$failure" -ne 0 ]; do
+    tries=`expr "$tries" - 1`
+    if [ "$tries" -eq 0 ]; then
+      echo "Could not connect to mysql server" 1>&2
+      return 1
+    fi
+    sleep 1
+    echo 'select 1;' | mysql -P$port "-h127.0.0.1" -uroot "-p$root_pwd" 2>/dev/null
+    failure=$?
+  done
+  return 0
+}

frontend.sh

@@ -12,6 +12,18 @@ For command-specific usage instructions, run frontend.sh command --help
 EOF
 }
 
+process_args() {
+  # Poor man's path expansion in argument 
+  while [ $# -gt 0 ]; do
+    if [ -e "$1" ]; then
+      readlink -f "$1"
+    else
+      echo "$1"
+    fi
+    shift
+  done
+}
+
 
 if [ $# -lt 2 ]; then
   usage
@@ -28,14 +40,20 @@ fi
 prog="$1"
 shift
 
+args=`process_args $@`
+
 case "$prog" in
   help|--help|-h)
     usage
     exit 0
   ;;
   slave)
     cd db/slave
-    ./slave.sh $@
+    ./slave.sh $args
+  ;;
+  master)
+    cd db/master
+    ./master.sh $args
   ;;
   *)
     usage