diff --git a/cloudlift/config/environment_configuration.py b/cloudlift/config/environment_configuration.py index 9dda5c61..1c241ab5 100644 --- a/cloudlift/config/environment_configuration.py +++ b/cloudlift/config/environment_configuration.py @@ -106,6 +106,20 @@ def _env_config_exists(self): ) return response.get('Item') is not None + def _allocate_eip(self): + client = boto3.client('ec2') + try: + # try to get an unassociated ip address first + allocated_addresses = client.describe_addresses() + response = list(filter(lambda x: 'AssociationId' not in x, allocated_addresses['Addresses']))[0] + except (KeyError, IndexError): + # Allocate a new ip address + response = client.allocate_address( + Domain='vpc' + ) + return response['AllocationId'] + + def _create_config(self): log_warning( "\nConfiguration for this environment was not found in DynamoDB.\ @@ -115,8 +129,12 @@ def _create_config(self): \nthe same configuration.\n" ) region = prompt("AWS region for environment", default='ap-south-1') - vpc_cidr = ipaddress.IPv4Network(prompt("VPC CIDR", default='10.10.10.10/16')) - nat_eip = prompt("Allocation ID Elastic IP for NAT") + vpc_cidr = ipaddress.IPv4Network(prompt("VPC CIDR", default='10.5.0.0/16')) + + nat_eip = prompt("Allocation ID Elastic IP for NAT (Enter existing EIP Allocation Id or \ + press enter to create a new one, if none are available", default='') + if not nat_eip: + nat_eip = self._allocate_eip() public_subnet_1_cidr = prompt( "Public Subnet 1 CIDR", default=list(vpc_cidr.subnets(new_prefix=22))[0]) public_subnet_2_cidr = prompt( @@ -127,10 +145,10 @@ def _create_config(self): "Private Subnet 2 CIDR", default=list(vpc_cidr.subnets(new_prefix=22))[3]) cluster_min_instances = prompt("Min instances in cluster", default=1) cluster_max_instances = prompt("Max instances in cluster", default=5) - cluster_instance_type = prompt("Instance type", default='m5.xlarge') - key_name = prompt("SSH key name") - notifications_arn = prompt("Notification SNS ARN") - ssl_certificate_arn = prompt("SSL certificate ARN") + cluster_instance_type = prompt("Instance type", default='t2.micro') + key_name = prompt("SSH key name", default='shoan') + notifications_arn = prompt("Notification SNS ARN", default='arn:aws:sns:ap-south-1:259042324395:shoan') + ssl_certificate_arn = prompt("SSL certificate ARN", default='arn:aws:acm:ap-south-1:259042324395:certificate/09d771d0-24d3-45d2-8e40-2237f12bea6a') environment_configuration = { self.environment: { "region": region, diff --git a/cloudlift/config/service_configuration.py b/cloudlift/config/service_configuration.py index 20379ba8..bca2d44e 100644 --- a/cloudlift/config/service_configuration.py +++ b/cloudlift/config/service_configuration.py @@ -5,6 +5,7 @@ import json +import boto3 import dictdiffer from botocore.exceptions import ClientError from click import confirm, edit @@ -37,10 +38,52 @@ def __init__(self, service_name, environment): # mfa_region = get_region_for_environment(environment) # mfa_session = mfa.get_mfa_session(mfa_region) # ssm_client = mfa_session.client('ssm') - self.table = get_resource_for( - 'dynamodb', - environment - ).Table(SERVICE_CONFIGURATION_TABLE) + # self.table = get_resource_for( + # 'dynamodb', + # environment + # ).Table(SERVICE_CONFIGURATION_TABLE) + session = boto3.session.Session() + self.dynamodb = session.resource('dynamodb') + self.table = self._get_table() + + def _get_table(self): + dynamodb_client = boto3.session.Session().client('dynamodb') + table_names = dynamodb_client.list_tables()['TableNames'] + if SERVICE_CONFIGURATION_TABLE not in table_names: + log_warning("Could not find configuration table, creating one..") + self._create_configuration_table() + return self.dynamodb.Table(SERVICE_CONFIGURATION_TABLE) + + def _create_configuration_table(self): + self.dynamodb.create_table( + TableName=SERVICE_CONFIGURATION_TABLE, + KeySchema=[ + { + 'AttributeName': 'service_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'environment', + 'KeyType': 'RANGE' + } + ], + AttributeDefinitions=[ + { + 'AttributeName': 'service_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'environment', + 'AttributeType': 'S' + }, + + ], + BillingMode='PAY_PER_REQUEST' + ) + + log_bold("Service Configuration table created!") + + def edit_config(self): ''' @@ -237,6 +280,7 @@ def _default_service_configuration(self): u'health_check_path': u'/elb-check' }, u'memory_reservation': 1000, + u'interruptable': False, u'command': None } } diff --git a/cloudlift/deployment/cluster_template_generator.py b/cloudlift/deployment/cluster_template_generator.py index 327f93d0..e4083878 100644 --- a/cloudlift/deployment/cluster_template_generator.py +++ b/cloudlift/deployment/cluster_template_generator.py @@ -25,6 +25,8 @@ from cloudlift.version import VERSION +INSTANCE_PURCHASE_OPTIONS = ['Spot', 'Ondemand'] + class ClusterTemplateGenerator(TemplateGenerator): """ This class generates CloudFormation template for a environment cluster @@ -439,107 +441,141 @@ def _add_ec2_auto_scaling(self): GroupDescription=Sub("${AWS::StackName}-databases") ) self.template.add_resource(database_security_group) - user_data = Base64(Sub('\n'.join([ - "#!/bin/bash", - "yum update -y", - "yum install -y aws-cfn-bootstrap", - "/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration", - "/opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource AutoScalingGroup", - "yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm", - "systemctl enable amazon-ssm-agent", - "systemctl start amazon-ssm-agent", - ""]))) - lc_metadata = cloudformation.Init({ - "config": cloudformation.InitConfig( - files=cloudformation.InitFiles({ - "/etc/cfn/cfn-hup.conf": cloudformation.InitFile( - content=Sub( - '\n'.join([ - '[main]', - 'stack=${AWS::StackId}', - 'region=${AWS::Region}', + + for asg_type in INSTANCE_PURCHASE_OPTIONS: + lc_metadata_override = '' + if asg_type == 'Spot': + lc_metadata_override = '\n'.join([ + 'echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true >> /etc/ecs/ecs.config', + ]) + user_data = Base64(Sub('\n'.join([ + "#!/bin/bash", + "yum update -y", + "yum install -y aws-cfn-bootstrap", + "/opt/aws/bin/cfn-init -v --region ${{AWS::Region}} --stack ${{AWS::StackName}} --resource LaunchConfiguration{}".format(asg_type), + "/opt/aws/bin/cfn-signal -e $? --region ${{AWS::Region}} --stack ${{AWS::StackName}} --resource AutoScalingGroup{}".format(asg_type), + "yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm", + "systemctl enable amazon-ssm-agent", + "systemctl start amazon-ssm-agent", + ""]))) + + + lc_metadata = cloudformation.Init({ + "config": cloudformation.InitConfig( + files=cloudformation.InitFiles({ + "/etc/cfn/cfn-hup.conf": cloudformation.InitFile( + content=Sub( + '\n'.join([ + '[main]', + 'stack=${AWS::StackId}', + 'region=${AWS::Region}', + '' + ]) + ), + mode='256', # TODO: Why 256 + owner="root", + group="root" + ), + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": cloudformation.InitFile( + content=Sub( + '\n'.join([ + '[cfn-auto-reloader-hook]', + 'triggers=post.update', + 'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init', + 'action=/opt/aws/bin/cfn-init -v --region ${{AWS::Region}} --stack ${{AWS::StackName}} --resource LaunchConfiguration{}'.format(asg_type), '' ]) ), - mode='256', # TODO: Why 256 - owner="root", - group="root" - ), - "/etc/cfn/hooks.d/cfn-auto-reloader.conf": cloudformation.InitFile( - content=Sub( - '\n'.join([ - '[cfn-auto-reloader-hook]', - 'triggers=post.update', - 'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init', - 'action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration', - '' - ]) - ), - ) - }), - services={ - "sysvinit": cloudformation.InitServices({ - "cfn-hup": cloudformation.InitService( - enabled=True, - ensureRunning=True, - files=['/etc/cfn/cfn-hup.conf', - '/etc/cfn/hooks.d/cfn-auto-reloader.conf'] - ) - }) - }, - commands={ - '01_add_instance_to_cluster': { - 'command': Sub( - 'echo "ECS_CLUSTER=${Cluster}\nECS_RESERVED_MEMORY=256" > /etc/ecs/ecs.config' ) + }), + services={ + "sysvinit": cloudformation.InitServices({ + "cfn-hup": cloudformation.InitService( + enabled=True, + ensureRunning=True, + files=['/etc/cfn/cfn-hup.conf', + '/etc/cfn/hooks.d/cfn-auto-reloader.conf'] + ) + }) + }, + commands={ + '01_add_instance_to_cluster': { + 'command': Sub( + '\n'.join([ + 'echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config', + 'echo ECS_RESERVED_MEMORY=256 >> /etc/ecs/ecs.config', + # 'echo ECS_INSTANCE_ATTRIBUTES={{\\\"instance-purchase-option\\\":\\\"{}\\\"}} >> /etc/ecs/ecs.config'.format(asg_type), + 'echo ECS_INSTANCE_ATTRIBUTES=\'{\"instance-purchase-option\":\"' + asg_type + '\"}\' >> /etc/ecs/ecs.config', + lc_metadata_override, + ]).strip() + ) + } } - } + ) + }) + if asg_type is 'Spot': + launch_configuration = LaunchConfiguration( + 'LaunchConfiguration{}'.format(asg_type), + UserData=user_data, + IamInstanceProfile=Ref(instance_profile), + SecurityGroups=[Ref(sg_hosts)], + InstanceType=Ref('InstanceType'), + ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"), + Metadata=lc_metadata, + KeyName=Ref(self.key_pair), + SpotPrice=self._get_spot_price(), + + ) + else: + launch_configuration = LaunchConfiguration( + 'LaunchConfiguration{}'.format(asg_type), + UserData=user_data, + IamInstanceProfile=Ref(instance_profile), + SecurityGroups=[Ref(sg_hosts)], + InstanceType=Ref('InstanceType'), + ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"), + Metadata=lc_metadata, + KeyName=Ref(self.key_pair), + ) + self.template.add_resource(launch_configuration) + # , PauseTime='PT15M', WaitOnResourceSignals=True, MaxBatchSize=1, MinInstancesInService=1) + up = AutoScalingRollingUpdate('AutoScalingRollingUpdate') + # TODO: clean up + subnets = list(self.private_subnets) + self.auto_scaling_group = AutoScalingGroup( + "AutoScalingGroup{}".format(asg_type), + UpdatePolicy=up, + DesiredCapacity=self.desired_instances, + Tags=[ + { + 'PropagateAtLaunch': True, + 'Value': Sub('${AWS::StackName} - ECS Host'), + 'Key': 'Name' + } + ], + MinSize=Ref('MinSize'), + MaxSize=Ref('MaxSize'), + VPCZoneIdentifier=[Ref(subnets.pop()), Ref(subnets.pop())], + LaunchConfigurationName=Ref(launch_configuration), + CreationPolicy=CreationPolicy( + ResourceSignal=ResourceSignal(Timeout='PT15M') + ) ) - }) - launch_configuration = LaunchConfiguration( - 'LaunchConfiguration', - UserData=user_data, - IamInstanceProfile=Ref(instance_profile), - SecurityGroups=[Ref(sg_hosts)], - InstanceType=Ref('InstanceType'), - ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"), - Metadata=lc_metadata, - KeyName=Ref(self.key_pair) - ) - self.template.add_resource(launch_configuration) - # , PauseTime='PT15M', WaitOnResourceSignals=True, MaxBatchSize=1, MinInstancesInService=1) - up = AutoScalingRollingUpdate('AutoScalingRollingUpdate') - # TODO: clean up - subnets = list(self.private_subnets) - self.auto_scaling_group = AutoScalingGroup( - "AutoScalingGroup", - UpdatePolicy=up, - DesiredCapacity=self.desired_instances, - Tags=[ - { - 'PropagateAtLaunch': True, - 'Value': Sub('${AWS::StackName} - ECS Host'), - 'Key': 'Name' - } - ], - MinSize=Ref('MinSize'), - MaxSize=Ref('MaxSize'), - VPCZoneIdentifier=[Ref(subnets.pop()), Ref(subnets.pop())], - LaunchConfigurationName=Ref(launch_configuration), - CreationPolicy=CreationPolicy( - ResourceSignal=ResourceSignal(Timeout='PT15M') + self.template.add_resource(self.auto_scaling_group) + self.cluster_scaling_policy = ScalingPolicy( + 'AutoScalingPolicy{}'.format(asg_type), + AdjustmentType='ChangeInCapacity', + AutoScalingGroupName=Ref(self.auto_scaling_group), + Cooldown=300, + PolicyType='SimpleScaling', + ScalingAdjustment=1 ) - ) - self.template.add_resource(self.auto_scaling_group) - self.cluster_scaling_policy = ScalingPolicy( - 'AutoScalingPolicy', - AdjustmentType='ChangeInCapacity', - AutoScalingGroupName=Ref(self.auto_scaling_group), - Cooldown=300, - PolicyType='SimpleScaling', - ScalingAdjustment=1 - ) - self.template.add_resource(self.cluster_scaling_policy) + self.template.add_resource(self.cluster_scaling_policy) + + def _get_spot_price(self): + # TODO: Use boto/aws apis to get the on-demand price for the instance selected + return "1.00" + def _add_cluster_parameters(self): self.template.add_parameter(Parameter( @@ -616,11 +652,12 @@ def _add_cluster_outputs(self): Description="ID of the 2nd subnet", Value=Ref(public_subnets.pop())) ) - self.template.add_output(Output( - "AutoScalingGroup", - Description="AutoScaling group for ECS container instances", - Value=Ref('AutoScalingGroup')) - ) + for asg_type in INSTANCE_PURCHASE_OPTIONS: + self.template.add_output(Output( + "AutoScalingGroup{}".format(asg_type), + Description="AutoScaling group for ECS container instances", + Value=Ref('AutoScalingGroup{}'.format(asg_type))) + ) self.template.add_output(Output( "SecurityGroupAlb", Description="Security group ID for ALB", diff --git a/cloudlift/deployment/service_template_generator.py b/cloudlift/deployment/service_template_generator.py index ac31d725..ede57c21 100644 --- a/cloudlift/deployment/service_template_generator.py +++ b/cloudlift/deployment/service_template_generator.py @@ -12,7 +12,7 @@ from troposphere.ecs import (AwsvpcConfiguration, ContainerDefinition, DeploymentConfiguration, Environment, LoadBalancer, LogConfiguration, - NetworkConfiguration, PlacementStrategy, + NetworkConfiguration, PlacementStrategy,PlacementConstraint, PortMapping, Service, TaskDefinition) from troposphere.elasticloadbalancingv2 import Action, Certificate, Listener from troposphere.elasticloadbalancingv2 import LoadBalancer as ALBLoadBalancer @@ -30,6 +30,7 @@ from cloudlift.config.logging import log, log_bold from cloudlift.deployment.service_information_fetcher import ServiceInformationFetcher from cloudlift.deployment.template_generator import TemplateGenerator +from cloudlift.deployment.cluster_template_generator import INSTANCE_PURCHASE_OPTIONS class ServiceTemplateGenerator(TemplateGenerator): @@ -220,6 +221,15 @@ def _add_service(self, service_name, config): MinimumHealthyPercent=100, MaximumPercent=200 ) + log_bold("Interruptable flag: {}".format(str(config['interruptable']))) + service_config = {} + if 'interruptable' in config and config['interruptable']: + desired_ec2_target_lifecycle = 'Spot' + else: + desired_ec2_target_lifecycle = 'Ondemand' + service_config['PlacementConstraints']=[PlacementConstraint(Type='memberOf', Expression='attribute:instance-purchase-option == {}'.format(desired_ec2_target_lifecycle))] + + if 'http_interface' in config: alb, lb, service_listener, alb_sg = self._add_alb(cd, service_name, config, launch_type) @@ -256,7 +266,8 @@ def _add_service(self, service_name, config): else: launch_type_svc = { 'Role': Ref(self.ecs_service_role), - 'PlacementStrategies': self.PLACEMENT_STRATEGIES + 'PlacementStrategies': self.PLACEMENT_STRATEGIES, + **service_config } svc = Service( service_name, @@ -312,6 +323,7 @@ def _add_service(self, service_name, config): else: launch_type_svc = { 'PlacementStrategies': self.PLACEMENT_STRATEGIES + **service_config } svc = Service( service_name, @@ -396,11 +408,12 @@ def _add_alb(self, cd, service_name, config, launch_type): ) self.template.add_resource(alb) - - target_group_name = "TargetGroup" + service_name - health_check_path = config['http_interface']['health_check_path'] if 'health_check_path' in config['http_interface'] else "/elb-check" - if config['http_interface']['internal']: - target_group_name = target_group_name + 'Internal' + for asg_name in INSTANCE_PURCHASE_OPTIONS: + target_group_name = "TargetGroup" + asg_name + service_name + health_check_path = config['http_interface']['health_check_path'] if 'health_check_path' in config[ + 'http_interface'] else "/elb-check" + if config['http_interface']['internal']: + target_group_name = target_group_name + 'Internal' target_group_config = {} if launch_type == self.LAUNCH_TYPE_FARGATE: @@ -446,7 +459,7 @@ def _add_alb(self, cd, service_name, config, launch_type): config['http_interface']['internal'] ) self._add_alb_alarms(service_name, alb) - return alb, lb, service_listener, svc_alb_sg + return alb, lb, service_listener, svc_alb_sg #TODO check against git history def _add_service_listener(self, service_name, target_group_action, alb, internal): diff --git a/cloudlift/version/__init__.py b/cloudlift/version/__init__.py index 7bd82a81..e946359b 100644 --- a/cloudlift/version/__init__.py +++ b/cloudlift/version/__init__.py @@ -1 +1 @@ -VERSION = '1.4.3' \ No newline at end of file +VERSION = '1.5.0-dev' \ No newline at end of file