Skip to content

Spot instances - WIP #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
shoan wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Spot instances - WIP #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions cloudlift/config/environment_configuration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,20 @@ def _env_config_exists(self):
)
return response.get('Item') is not None

def _allocate_eip(self):
client = boto3.client('ec2')
try:
# try to get an unassociated ip address first
allocated_addresses = client.describe_addresses()
response = list(filter(lambda x: 'AssociationId' not in x, allocated_addresses['Addresses']))[0]
except (KeyError, IndexError):
# Allocate a new ip address
response = client.allocate_address(
Domain='vpc'
)
return response['AllocationId']


def _create_config(self):
log_warning(
"\nConfiguration for this environment was not found in DynamoDB.\
Expand All @@ -115,8 +129,12 @@ def _create_config(self):
\nthe same configuration.\n"
)
region = prompt("AWS region for environment", default='ap-south-1')
vpc_cidr = ipaddress.IPv4Network(prompt("VPC CIDR", default='10.10.10.10/16'))
nat_eip = prompt("Allocation ID Elastic IP for NAT")
vpc_cidr = ipaddress.IPv4Network(prompt("VPC CIDR", default='10.5.0.0/16'))

nat_eip = prompt("Allocation ID Elastic IP for NAT (Enter existing EIP Allocation Id or \
press enter to create a new one, if none are available", default='')
if not nat_eip:
nat_eip = self._allocate_eip()
public_subnet_1_cidr = prompt(
"Public Subnet 1 CIDR", default=list(vpc_cidr.subnets(new_prefix=22))[0])
public_subnet_2_cidr = prompt(
Expand All @@ -127,10 +145,10 @@ def _create_config(self):
"Private Subnet 2 CIDR", default=list(vpc_cidr.subnets(new_prefix=22))[3])
cluster_min_instances = prompt("Min instances in cluster", default=1)
cluster_max_instances = prompt("Max instances in cluster", default=5)
cluster_instance_type = prompt("Instance type", default='m5.xlarge')
key_name = prompt("SSH key name")
notifications_arn = prompt("Notification SNS ARN")
ssl_certificate_arn = prompt("SSL certificate ARN")
cluster_instance_type = prompt("Instance type", default='t2.micro')
key_name = prompt("SSH key name", default='shoan')
notifications_arn = prompt("Notification SNS ARN", default='arn:aws:sns:ap-south-1:259042324395:shoan')
ssl_certificate_arn = prompt("SSL certificate ARN", default='arn:aws:acm:ap-south-1:259042324395:certificate/09d771d0-24d3-45d2-8e40-2237f12bea6a')
environment_configuration = {
self.environment: {
"region": region,
Expand Down
52 changes: 48 additions & 4 deletions cloudlift/config/service_configuration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import json

import boto3
import dictdiffer
from botocore.exceptions import ClientError
from click import confirm, edit
Expand Down Expand Up @@ -37,10 +38,52 @@ def __init__(self, service_name, environment):
# mfa_region = get_region_for_environment(environment)
# mfa_session = mfa.get_mfa_session(mfa_region)
# ssm_client = mfa_session.client('ssm')
self.table = get_resource_for(
'dynamodb',
environment
).Table(SERVICE_CONFIGURATION_TABLE)
# self.table = get_resource_for(
# 'dynamodb',
# environment
# ).Table(SERVICE_CONFIGURATION_TABLE)
session = boto3.session.Session()
self.dynamodb = session.resource('dynamodb')
self.table = self._get_table()

def _get_table(self):
dynamodb_client = boto3.session.Session().client('dynamodb')
table_names = dynamodb_client.list_tables()['TableNames']
if SERVICE_CONFIGURATION_TABLE not in table_names:
log_warning("Could not find configuration table, creating one..")
self._create_configuration_table()
return self.dynamodb.Table(SERVICE_CONFIGURATION_TABLE)

def _create_configuration_table(self):
self.dynamodb.create_table(
TableName=SERVICE_CONFIGURATION_TABLE,
KeySchema=[
{
'AttributeName': 'service_name',
'KeyType': 'HASH'
},
{
'AttributeName': 'environment',
'KeyType': 'RANGE'
}
],
AttributeDefinitions=[
{
'AttributeName': 'service_name',
'AttributeType': 'S'
},
{
'AttributeName': 'environment',
'AttributeType': 'S'
},

],
BillingMode='PAY_PER_REQUEST'
)

log_bold("Service Configuration table created!")



def edit_config(self):
'''
Expand Down Expand Up @@ -237,6 +280,7 @@ def _default_service_configuration(self):
u'health_check_path': u'/elb-check'
},
u'memory_reservation': 1000,
u'interruptable': False,
u'command': None
}
}
Expand Down
235 changes: 136 additions & 99 deletions cloudlift/deployment/cluster_template_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
from cloudlift.version import VERSION


INSTANCE_PURCHASE_OPTIONS = ['Spot', 'Ondemand']

class ClusterTemplateGenerator(TemplateGenerator):
"""
This class generates CloudFormation template for a environment cluster
Expand Down Expand Up @@ -439,107 +441,141 @@ def _add_ec2_auto_scaling(self):
GroupDescription=Sub("${AWS::StackName}-databases")
)
self.template.add_resource(database_security_group)
user_data = Base64(Sub('\n'.join([
"#!/bin/bash",
"yum update -y",
"yum install -y aws-cfn-bootstrap",
"/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration",
"/opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource AutoScalingGroup",
"yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm",
"systemctl enable amazon-ssm-agent",
"systemctl start amazon-ssm-agent",
""])))
lc_metadata = cloudformation.Init({
"config": cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/etc/cfn/cfn-hup.conf": cloudformation.InitFile(
content=Sub(
'\n'.join([
'[main]',
'stack=${AWS::StackId}',
'region=${AWS::Region}',

for asg_type in INSTANCE_PURCHASE_OPTIONS:
lc_metadata_override = ''
if asg_type == 'Spot':
lc_metadata_override = '\n'.join([
'echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true >> /etc/ecs/ecs.config',
])
user_data = Base64(Sub('\n'.join([
"#!/bin/bash",
"yum update -y",
"yum install -y aws-cfn-bootstrap",
"/opt/aws/bin/cfn-init -v --region ${{AWS::Region}} --stack ${{AWS::StackName}} --resource LaunchConfiguration{}".format(asg_type),
"/opt/aws/bin/cfn-signal -e $? --region ${{AWS::Region}} --stack ${{AWS::StackName}} --resource AutoScalingGroup{}".format(asg_type),
"yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm",
"systemctl enable amazon-ssm-agent",
"systemctl start amazon-ssm-agent",
""])))


lc_metadata = cloudformation.Init({
"config": cloudformation.InitConfig(
files=cloudformation.InitFiles({
"/etc/cfn/cfn-hup.conf": cloudformation.InitFile(
content=Sub(
'\n'.join([
'[main]',
'stack=${AWS::StackId}',
'region=${AWS::Region}',
''
])
),
mode='256', # TODO: Why 256
owner="root",
group="root"
),
"/etc/cfn/hooks.d/cfn-auto-reloader.conf": cloudformation.InitFile(
content=Sub(
'\n'.join([
'[cfn-auto-reloader-hook]',
'triggers=post.update',
'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init',
'action=/opt/aws/bin/cfn-init -v --region ${{AWS::Region}} --stack ${{AWS::StackName}} --resource LaunchConfiguration{}'.format(asg_type),
''
])
),
mode='256', # TODO: Why 256
owner="root",
group="root"
),
"/etc/cfn/hooks.d/cfn-auto-reloader.conf": cloudformation.InitFile(
content=Sub(
'\n'.join([
'[cfn-auto-reloader-hook]',
'triggers=post.update',
'path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init',
'action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource LaunchConfiguration',
''
])
),
)
}),
services={
"sysvinit": cloudformation.InitServices({
"cfn-hup": cloudformation.InitService(
enabled=True,
ensureRunning=True,
files=['/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf']
)
})
},
commands={
'01_add_instance_to_cluster': {
'command': Sub(
'echo "ECS_CLUSTER=${Cluster}\nECS_RESERVED_MEMORY=256" > /etc/ecs/ecs.config'
)
}),
services={
"sysvinit": cloudformation.InitServices({
"cfn-hup": cloudformation.InitService(
enabled=True,
ensureRunning=True,
files=['/etc/cfn/cfn-hup.conf',
'/etc/cfn/hooks.d/cfn-auto-reloader.conf']
)
})
},
commands={
'01_add_instance_to_cluster': {
'command': Sub(
'\n'.join([
'echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config',
'echo ECS_RESERVED_MEMORY=256 >> /etc/ecs/ecs.config',
# 'echo ECS_INSTANCE_ATTRIBUTES={{\\\"instance-purchase-option\\\":\\\"{}\\\"}} >> /etc/ecs/ecs.config'.format(asg_type),
'echo ECS_INSTANCE_ATTRIBUTES=\'{\"instance-purchase-option\":\"' + asg_type + '\"}\' >> /etc/ecs/ecs.config',
lc_metadata_override,
]).strip()
)
}
}
}
)
})
if asg_type is 'Spot':
launch_configuration = LaunchConfiguration(
'LaunchConfiguration{}'.format(asg_type),
UserData=user_data,
IamInstanceProfile=Ref(instance_profile),
SecurityGroups=[Ref(sg_hosts)],
InstanceType=Ref('InstanceType'),
ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"),
Metadata=lc_metadata,
KeyName=Ref(self.key_pair),
SpotPrice=self._get_spot_price(),

)
else:
launch_configuration = LaunchConfiguration(
'LaunchConfiguration{}'.format(asg_type),
UserData=user_data,
IamInstanceProfile=Ref(instance_profile),
SecurityGroups=[Ref(sg_hosts)],
InstanceType=Ref('InstanceType'),
ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"),
Metadata=lc_metadata,
KeyName=Ref(self.key_pair),
)
self.template.add_resource(launch_configuration)
# , PauseTime='PT15M', WaitOnResourceSignals=True, MaxBatchSize=1, MinInstancesInService=1)
up = AutoScalingRollingUpdate('AutoScalingRollingUpdate')
# TODO: clean up
subnets = list(self.private_subnets)
self.auto_scaling_group = AutoScalingGroup(
"AutoScalingGroup{}".format(asg_type),
UpdatePolicy=up,
DesiredCapacity=self.desired_instances,
Tags=[
{
'PropagateAtLaunch': True,
'Value': Sub('${AWS::StackName} - ECS Host'),
'Key': 'Name'
}
],
MinSize=Ref('MinSize'),
MaxSize=Ref('MaxSize'),
VPCZoneIdentifier=[Ref(subnets.pop()), Ref(subnets.pop())],
LaunchConfigurationName=Ref(launch_configuration),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(Timeout='PT15M')
)
)
})
launch_configuration = LaunchConfiguration(
'LaunchConfiguration',
UserData=user_data,
IamInstanceProfile=Ref(instance_profile),
SecurityGroups=[Ref(sg_hosts)],
InstanceType=Ref('InstanceType'),
ImageId=FindInMap("AWSRegionToAMI", Ref("AWS::Region"), "AMI"),
Metadata=lc_metadata,
KeyName=Ref(self.key_pair)
)
self.template.add_resource(launch_configuration)
# , PauseTime='PT15M', WaitOnResourceSignals=True, MaxBatchSize=1, MinInstancesInService=1)
up = AutoScalingRollingUpdate('AutoScalingRollingUpdate')
# TODO: clean up
subnets = list(self.private_subnets)
self.auto_scaling_group = AutoScalingGroup(
"AutoScalingGroup",
UpdatePolicy=up,
DesiredCapacity=self.desired_instances,
Tags=[
{
'PropagateAtLaunch': True,
'Value': Sub('${AWS::StackName} - ECS Host'),
'Key': 'Name'
}
],
MinSize=Ref('MinSize'),
MaxSize=Ref('MaxSize'),
VPCZoneIdentifier=[Ref(subnets.pop()), Ref(subnets.pop())],
LaunchConfigurationName=Ref(launch_configuration),
CreationPolicy=CreationPolicy(
ResourceSignal=ResourceSignal(Timeout='PT15M')
self.template.add_resource(self.auto_scaling_group)
self.cluster_scaling_policy = ScalingPolicy(
'AutoScalingPolicy{}'.format(asg_type),
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(self.auto_scaling_group),
Cooldown=300,
PolicyType='SimpleScaling',
ScalingAdjustment=1
)
)
self.template.add_resource(self.auto_scaling_group)
self.cluster_scaling_policy = ScalingPolicy(
'AutoScalingPolicy',
AdjustmentType='ChangeInCapacity',
AutoScalingGroupName=Ref(self.auto_scaling_group),
Cooldown=300,
PolicyType='SimpleScaling',
ScalingAdjustment=1
)
self.template.add_resource(self.cluster_scaling_policy)
self.template.add_resource(self.cluster_scaling_policy)

def _get_spot_price(self):
# TODO: Use boto/aws apis to get the on-demand price for the instance selected
return "1.00"


def _add_cluster_parameters(self):
self.template.add_parameter(Parameter(
Expand Down Expand Up @@ -616,11 +652,12 @@ def _add_cluster_outputs(self):
Description="ID of the 2nd subnet",
Value=Ref(public_subnets.pop()))
)
self.template.add_output(Output(
"AutoScalingGroup",
Description="AutoScaling group for ECS container instances",
Value=Ref('AutoScalingGroup'))
)
for asg_type in INSTANCE_PURCHASE_OPTIONS:
self.template.add_output(Output(
"AutoScalingGroup{}".format(asg_type),
Description="AutoScaling group for ECS container instances",
Value=Ref('AutoScalingGroup{}'.format(asg_type)))
)
self.template.add_output(Output(
"SecurityGroupAlb",
Description="Security group ID for ALB",
Expand Down
Loading