Merge pull request #15 from lucj/master

Use ContainerPilot 3.3.0 and Consul 0.9.0

Author: geek

Dockerfile

@@ -21,18 +21,18 @@ RUN curl -Ls -o get-pip.py https://bootstrap.pypa.io/get-pip.py && \
 		mock==2.0.0
 
 # Add consul agent
-RUN export CONSUL_VERSION=0.7.5 \
-    && export CONSUL_CHECKSUM=40ce7175535551882ecdff21fdd276cef6eaab96be8a8260e0599fadb6f1f5b8 \
+RUN export CONSUL_VERSION=1.0.6 \
+    && export CONSUL_CHECKSUM=bcc504f658cef2944d1cd703eda90045e084a15752d23c038400cf98c716ea01 \
     && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \
     && echo "${CONSUL_CHECKSUM}  /tmp/consul.zip" | sha256sum -c \
     && unzip /tmp/consul -d /usr/local/bin \
     && rm /tmp/consul.zip \
     && mkdir -p /opt/consul/config
 
 # Add ContainerPilot and set its configuration file path
-ENV CONTAINERPILOT_VER 2.7.2
-ENV CONTAINERPILOT file:///etc/containerpilot.json
-RUN export CONTAINERPILOT_CHECKSUM=e886899467ced6d7c76027d58c7f7554c2fb2bcc \
+ENV CONTAINERPILOT_VER 3.7.0
+ENV CONTAINERPILOT /etc/containerpilot.json5
+RUN export CONTAINERPILOT_CHECKSUM=b10b30851de1ae1c095d5f253d12ce8fe8e7be17 \
     && curl -Lso /tmp/containerpilot.tar.gz \
         "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \
     && echo "${CONTAINERPILOT_CHECKSUM}  /tmp/containerpilot.tar.gz" | sha1sum -c \
@@ -44,11 +44,8 @@ ENV MONGO_SECONDARY_CATCHUP_PERIOD 8
 ENV MONGO_STEPDOWN_TIME 60
 ENV MONGO_ELECTION_TIMEOUT 30
 
-# configure ContainerPilot and MySQL
+# Configure ContainerPilot and Mongo
 COPY etc/* /etc/
 COPY bin/* /usr/local/bin/
 
-ENTRYPOINT ["containerpilot", "mongod"]
-
-# Define CMD so the name of the replicaset can be overridden in the compose file
-CMD ["--replSet=joyent"]
+ENTRYPOINT ["containerpilot"]

bin/manage.py

@@ -8,6 +8,9 @@
 import struct
 import sys
 import time
+import datetime
+import subprocess
+import manta
 
 from functools import wraps
 
@@ -84,6 +87,12 @@ def get_environ(key, default):
 MONGO_SECONDARY_CATCHUP_PERIOD=int(get_environ('MONGO_SECONDARY_CATCHUP_PERIOD', 8))
 MONGO_ELECTION_TIMEOUT=int(get_environ('MONGO_ELECTION_TIMEOUT', 30))
 
+# Manta related information
+MANTA_URL=get_environ('MANTA_URL','https://us-east.manta.joyent.com')
+MANTA_USER=get_environ('MANTA_USER','')
+MANTA_KEY_ID=get_environ('MANTA_KEY_ID','')
+MANTA_TLS_INSECURE=bool(get_environ('MANTA_TLS_INSECURE',True))
+
 # ---------------------------------------------------------
 # Top-level functions called by ContainerPilot or forked by this program
 
@@ -155,14 +164,58 @@ def pre_stop():
 
     return True
 
+@debug
+def upload(filename):
+    try:
+        log.debug(filename)
+        client = manta.MantaClient(url=MANTA_URL,
+                                   account=MANTA_USER,
+                                   signer=manta.SSHAgentSigner([MANTA_KEY_ID]),
+                                   disable_ssl_certificate_validation=MANTA_TLS_INSECURE
+                                   )
+        client.put_object('/%s/stor/%s' % (MANTA_USER, filename), path='/tmp/%s' % filename)
+    except Exception as e:
+        log.debug(e)
+        return False
+    return True
+
+@debug
+def backup():
+    """
+    Run periodic mongodump and save backup to MANTA
+    """
+
+    hostname = socket.gethostname()
+    ip = get_ip()
+    local_mongo = MongoClient(ip, connect=False)
+
+    timestamp = datetime.datetime.utcnow().isoformat()
+    dump = "dump-%s.gz" % timestamp
+
+    try:
+        backup_output = subprocess.check_output([ 'mongodump', '--archive=/tmp/%s' % dump, '--gzip' ])
+        log.debug(backup_output)
+
+        # Upload dump onto Manta
+        #TODO: do not manage to upload to Manta
+        # upload(dump)
+
+        # Delete local dump once uploaded
+        # os.remove('/tmp/%s' % dump)
+    except Exception as e:
+        log.debug('Error during backup process');
+        log.debug(e);
+        return False
+
+    return True
+
 @debug
 def health():
     """
     Run a simple health check. Also acts as a check for whether the
     ContainerPilot configuration needs to be reloaded (if it's been
     changed externally).
     """
-    # TODO periodic mongodumps to Manta
 
     hostname = socket.gethostname()
     ip = get_ip()

etc/containerpilot.json

@@ -0,0 +1,58 @@
+{
+  consul: "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}{{ end }}:8500",
+  logging: {
+    level: "DEBUG",
+    format: "text"
+  },
+  jobs: [
+    {
+      name: "preStart",
+      exec: "python /usr/local/bin/manage.py"
+    },
+    {
+      name: "preStop",
+      exec: "python /usr/local/bin/manage.py pre_stop"
+    },
+    {{ if .CONSUL_AGENT }}
+    {
+      name: "consul-agent",
+      exec: ["/usr/local/bin/consul", "agent",
+                "-data-dir=/opt/consul/data",
+                "-config-dir=/opt/consul/config",
+                "-rejoin",
+                "-retry-join", "{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}",
+                "-retry-max", "10",
+                "-retry-interval", "10s"],
+       restarts: "unlimited"
+    },
+    {{ end }}
+    {
+      name: "mongodb-replicaset",
+      port: 27017,
+      exec: "mongod --replSet={{ if .REPLICASET }}{{ .REPLICASET }}{{ else }}joyent{{ end }}",
+      when: {
+        source: "preStart",
+        once: "exitSuccess"
+      },
+      health: {
+        exec: "python /usr/local/bin/manage.py health",
+        interval: 10,
+        ttl: 25
+      },
+    },
+    {
+      name: "onchange-mongo",
+      exec: "python /usr/local/bin/manage.py on_change",
+      when: {
+        source: "watch.mongodb-replicaset",
+        each: "changed"
+      }
+    },
+  ],
+  watches: [
+    {
+      name: "mongodb-replicaset",
+      interval: 2
+    }
+  ]
+}