Python autoscaling library init

Summary: With Mesos example

Test Plan: Scale a Mesos cluster

Reviewers: tberman, peter, mark, merkkila

Reviewed By: merkkila

Subscribers: tberman, justin, stephan, serdar

Differential Revision: http://phabricator.thefactory.com/D3238

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 The Factory <http://www.thefactory.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,25 @@
+## Usage
+See `examples/`
+
+## Examples
+
+### mesos_ec2.py
+
+```bash
+mesos_ec2.py \
+    --mesos-url http://mesos_master.local:5050 \
+    --cpus 1,3 \
+    --region us-west-2 \
+    --asg mesos-MesosSlaveStack-1AB12345ABC-ServerGroup-789XYZ789 \
+    --log-level info
+```
+
+```console
+INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): mesos_master.local
+INFO:autoscale:State: {'mem_free': 6353, 'disk_free': 35703, 'cpus_free': 8.5}
+INFO:autoscale:Thresholds: {'cpus': {'upper': 3, 'lower': 1}}
+INFO:autoscale:Should scale by -1
+Scaling mesos-MesosSlaveStack-1AB12345ABC-ServerGroup-789XYZ789 in us-west-2 by -1
+INFO:autoscale:Current scale: 9
+INFO:autoscale:Scaling to 8
+```

autoscale.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+import logging
+import sys
+
+import boto.ec2.autoscale
+import requests
+
+
+logger = logging.getLogger(__name__)
+
+
+class MesosReporter():
+    def __init__(self, mesos_url):
+        self.mesos_url = mesos_url.rstrip('/')
+        stats_url = '/'.join([self.mesos_url, '/stats.json'])
+        self.state = requests.get(stats_url).json()
+
+    @property
+    def state(self):
+        return self.state
+
+
+class MesosDecider():
+    def __init__(self, thresholds):
+        self.thresholds = thresholds
+
+    def should_scale(self, cluster):
+        increment = 1
+        decrement = -1
+
+        cpus_free = cluster.state['cpus_total'] - cluster.state['cpus_used']
+        disk_free = cluster.state['disk_total'] - cluster.state['disk_used']
+        mem_free = cluster.state['mem_total'] - cluster.state['mem_used']
+        logger.info('State: %s', dict(cpus_free=cpus_free, disk_free=disk_free, mem_free=mem_free))
+        logger.info('Thresholds: %s', self.thresholds)
+
+        if   (('cpus' in self.thresholds and cpus_free < self.thresholds['cpus']['lower']) or
+              ('disk' in self.thresholds and disk_free < self.thresholds['disk']['lower']) or
+              ('mem' in self.thresholds and mem_free < self.thresholds['mem']['lower'])):
+            scale_by = increment
+        elif (('cpus' in self.thresholds and cpus_free > self.thresholds['cpus']['upper']) or
+              ('disk' in self.thresholds and disk_free > self.thresholds['disk']['upper']) or
+              ('mem' in self.thresholds and mem_free > self.thresholds['mem']['upper'])):
+            scale_by = decrement
+        else:
+            scale_by = 0
+
+        logger.info('Should scale by %s', scale_by)
+        return scale_by
+
+
+class AwsAsgScaler():
+    def __init__(self, region, asg_name, min_instances=1, max_instances=None, 
+                 aws_access_key_id=None, aws_secret_access_key=None):
+        self.region = region
+        self.asg_name = asg_name
+        self.min_instances = min_instances
+        self.max_instances = max_instances
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+
+    def _get_connection(self):
+        if self.aws_access_key_id and self.aws_secret_access_key:
+            return boto.ec2.autoscale.connect_to_region(
+                self.region, 
+                aws_access_key_id=self.aws_access_key_id,
+                aws_secret_access_key=self.aws_secret_access_key)
+        else:
+            return boto.ec2.autoscale.connect_to_region(self.region)
+
+    def scale(self, delta):
+        c = self._get_connection()
+        current_count = c.get_all_groups(names=[self.asg_name])[0].desired_capacity
+        logger.info("Current scale: %s", current_count)
+        new_count = current_count + delta
+
+        if self.min_instances and new_count < self.min_instances:
+            new_count = self.min_instances
+        elif self.max_instances and new_count > self.max_instances:
+            new_count = self.max_instances
+
+        if new_count != current_count:
+            logger.info("Scaling to %s", new_count)
+            c.set_desired_capacity(self.asg_name, new_count)

examples/mesos_ec2.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+import argparse
+import logging
+import sys
+
+from autoscale import MesosReporter, MesosDecider, AwsAsgScaler
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-l', '--log-level', default="warn", help='Log level (debug, [default] info, warn, error)')
+    parser.add_argument('-u', '--mesos-url', help='Mesos cluster URL', required=True)
+    parser.add_argument('-c', '--cpus', help='Comma-delimited CPU thresholds (lower,upper)')    
+    parser.add_argument('-d', '--disk', help='Comma-delimited disk thresholds (lower,upper)')
+    parser.add_argument('-m', '--mem', help='Comma-delimited memory thresholds (lower,upper)')
+    parser.add_argument('-r', '--region', help='AWS region', required=True)
+    parser.add_argument('-a', '--asg', help='AWS auto scaling group name', required=True)
+
+    args = parser.parse_args()
+
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(stream=sys.stderr, level=getattr(logging, args.log_level.upper()))
+
+    thresholds = {}
+    if args.cpus:
+        lower, upper = args.cpus.split(',')
+        thresholds['cpus'] = dict(lower=int(lower), upper=int(upper))
+    if args.disk:
+        lower, upper = args.disk.split(',')
+        thresholds['disk'] = dict(lower=int(lower), upper=int(upper))
+    if args.mem:
+        lower, upper = args.mem.split(',')
+        thresholds['mem'] = dict(lower=int(lower), upper=int(upper))
+
+    reporter = MesosReporter(args.mesos_url)
+    decider = MesosDecider(thresholds)
+    scaler = AwsAsgScaler(args.region, args.asg) 
+
+    delta = decider.should_scale(reporter)
+    if delta:
+        print 'Scaling {asg} in {region} by {delta}'.format(asg=args.asg, region=args.region, delta=delta)
+        scaler.scale(delta)
+    else:
+        print 'No change needed for {asg} in {region}'.format(asg=args.asg, region=args.region)
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -0,0 +1,24 @@
+import sys
+from setuptools import setup
+
+extra = {}
+if sys.version_info >= (3,):
+    extra['use_2to3'] = True
+
+setup(name='autoscale',
+      version="0.0.1",
+      description='Extensible library to manage scaling logic and actions',
+      author='Mike Babineau',
+      author_email='[email protected]',
+      install_requires=[ 'requests>=2.0.0', 'boto>=2.1.0' ],
+      url='https://github.com/thefactory/autoscale-python',
+      license='MIT',
+      platforms='Posix; MacOS X; Windows',
+      classifiers=['Development Status :: 3 - Alpha',
+                   'Intended Audience :: Developers',
+                   'Intended Audience :: System Administrators',
+                   'License :: OSI Approved :: MIT License',
+                   'Operating System :: OS Independent',
+                   'Topic :: Software Development :: Libraries :: Python Modules'],
+      **extra
+)