fix(update_service): fixes TaskDefinition inactive exception

cloudlift/config/pre_flight.py

@@ -46,4 +46,12 @@ def check_aws_instance_type(instance_type):
             continue
         else:
             return False, i
-    return True, ""
+    return True, ""
+
+def service_update_preflight_checks(current_version, service_name, environment, ecr_client):
+    # If the current deployment is considered dirty, make sure that an image tagged as 'master' is uploaded to ECR otherwise on service update, the service will try to use an image tagged as 'master' which does not exist
+    if current_version == 'dirty':
+        repo_name = service_name + '-repo'
+        res = ecr_client.batch_get_image(repositoryName=repo_name, imageIds=[{'imageTag': 'master'}])
+        if res['images'] == []:
+            raise UnrecoverableException("Current deployment is dirty. Please push an image tagged as 'master' to ECR.")

cloudlift/deployment/ecs.py

@@ -38,7 +38,8 @@ def describe_services(self, cluster_name, service_name):
     def describe_task_definition(self, task_definition_arn):
         try:
             return self.boto.describe_task_definition(
-                taskDefinition=task_definition_arn
+                taskDefinition=task_definition_arn,
+                include=['TAGS']
             )
         except ClientError:
             raise UnknownTaskDefinitionError(
@@ -59,21 +60,26 @@ def describe_tasks(self, cluster_name, task_arns):
         return self.boto.describe_tasks(cluster=cluster_name, tasks=task_arns)
 
     def register_task_definition(self, family, containers, volumes, role_arn, cpu=False, memory=False, execution_role_arn=None,
-                                 requires_compatibilities=[], network_mode='bridge'):
+                                 requires_compatibilities=[], network_mode='bridge', tags=[]):
         fargate_td = {}
         if 'FARGATE' in requires_compatibilities:
             fargate_td = {
                 'requiresCompatibilities': requires_compatibilities or [],
                 'cpu': cpu,
                 'memory': memory,
             }
+
+        if not any(tag['key'] == 'task_definition_source' for tag in tags):
+            tags.append({'key': 'task_definition_source', 'value': 'boto3'})
+
         return self.boto.register_task_definition(
             family=family,
             containerDefinitions=containers,
             volumes=volumes,
             executionRoleArn=execution_role_arn,
             taskRoleArn=role_arn or u'',
             networkMode=network_mode,
+            tags=tags,
             **fargate_td
         )
 
@@ -223,6 +229,10 @@ def revision(self):
     @property
     def family_revision(self):
         return '%s:%d' % (self.get(u'family'), self.get(u'revision'))
+
+    @property
+    def tags(self):
+        return self.get(u'tags')
 
     @property
     def diff(self):
@@ -407,7 +417,8 @@ def get_current_task_definition(self, service):
             task_definition_arn=service.task_definition
         )
         task_definition = EcsTaskDefinition(
-            task_definition=task_definition_payload[u'taskDefinition']
+            task_definition=task_definition_payload[u'taskDefinition'],
+            tags=task_definition_payload[u'tags']
         )
         return task_definition
 
@@ -429,17 +440,36 @@ def update_task_definition(self, task_definition):
                 'memory' : task_definition.memory or u'',
 
             }
+        new_tags = []
+        tags = task_definition.get('tags', [])
+        if not tags:
+            new_tags.append({'key': 'task_definition_source', 'value': 'boto3'})
+
+        for tag in tags:
+            if tag['key'] == 'task_definition_source':
+                new_tags.append({'key': tag['key'], 'value': 'boto3'})
+            else:
+                new_tags.append(tag)
+
         response = self._client.register_task_definition(
             family=task_definition.family,
             containers=task_definition.containers,
             volumes=task_definition.volumes,
             role_arn=task_definition.role_arn,
             execution_role_arn=task_definition.execution_role_arn if task_definition.execution_role_arn else boto3.resource('iam').Role('ecsTaskExecutionRole').arn,
             network_mode=task_definition.network_mode or u'bridge',
+            tags=new_tags,
             **fargate_td
         )
+        # deregister old task definition only if it was created manually (not by cloudformation)
+        if task_definition.tags:
+            for tag in task_definition.tags:
+                if tag['key'] == 'task_definition_source':
+                    task_definition_source = tag['value']
+                    if task_definition_source != 'cloudformation':
+                        self._client.deregister_task_definition(task_definition.arn)
+                    break
         new_task_definition = EcsTaskDefinition(response[u'taskDefinition'])
-        self._client.deregister_task_definition(task_definition.arn)
         return new_task_definition
 
     def update_service(self, service):

cloudlift/deployment/service_creator.py

@@ -15,7 +15,8 @@
 from cloudlift.config.logging import log, log_bold, log_err
 from cloudlift.deployment.progress import get_stack_events, print_new_events
 from cloudlift.deployment.service_template_generator import ServiceTemplateGenerator
-
+from cloudlift.deployment.service_information_fetcher import ServiceInformationFetcher
+from cloudlift.config.pre_flight import service_update_preflight_checks
 
 class ServiceCreator(object):
     '''
@@ -29,6 +30,7 @@ def __init__(self, name, environment):
         self.stack_name = get_service_stack_name(environment, name)
         self.client = get_client_for('cloudformation', self.environment)
         self.s3client = get_client_for('s3', self.environment)
+        self.ecrClient = get_client_for('ecr', self.environment)
         self.bucket_name = 'cloudlift-service-template'
         self.environment_stack = self._get_environment_stack()
         self.existing_events = get_stack_events(self.client, self.stack_name)
@@ -106,6 +108,9 @@ def update(self):
         '''
 
         log_bold("Starting to update service")
+        current_version = ServiceInformationFetcher(
+            self.name, self.environment).get_current_version(skip_master_reset=True)
+        service_update_preflight_checks(current_version=current_version, service_name=self.name, environment=self.environment, ecr_client=self.ecrClient)
         self.service_configuration.edit_config()
         try:
             template_generator = ServiceTemplateGenerator(
@@ -164,4 +169,4 @@ def _print_progress(self):
         if "FAIL" in final_status:
             log_err("Finished with status: %s" % (final_status))
         else:
-            log_bold("Finished with status: %s" % (final_status))
+            log_bold("Finished with status: %s" % (final_status))

cloudlift/deployment/service_information_fetcher.py

@@ -43,9 +43,9 @@ def init_stack_info(self):
             self.ecs_service_names = []
             log_warning("Could not determine services.")
 
-    def get_current_version(self):
+    def get_current_version(self, skip_master_reset=False):
         commit_sha = self._fetch_current_task_definition_tag()
-        if commit_sha is None or commit_sha == 'dirty':
+        if commit_sha is None or commit_sha == 'dirty' and not skip_master_reset:
             log_warning("Currently deployed tag could not be found or is dirty,\
 resetting to master")
             commit_sha = "master"

cloudlift/deployment/service_template_generator.py

@@ -309,7 +309,7 @@ def _add_service(self, service_name, config):
             ContainerDefinitions=[cd],
             ExecutionRoleArn=boto3.resource('iam').Role('ecsTaskExecutionRole').arn,
             TaskRoleArn=Ref(task_role),
-            Tags=Tags(Team=self.team_name, environment=self.env),
+            Tags=Tags(Team=self.team_name, environment=self.env, task_definition_source="cloudformation"),
             **launch_type_td
 
         )