Merge branch 'master' into 'develop'

Master

See merge request verbose-equals-true/django-postgres-vue-gitlab-ecs!11

Author: briancaffey

.env.template

@@ -1,11 +1,11 @@
 SECRET_KEY=secret
 DEBUG=True
 
-RDS_DB_NAME=postgres
-RDS_USERNAME=postgres
-RDS_PASSWORD=postgres
-RDS_HOSTNAME=postgres
-RDS_PORT=5432
+POSTGRES_NAME=postgres
+POSTGRES_USERNAME=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_SERVICE_HOST=postgres
+POSTGRES_SERVICE_PORT=5432
 
 GITHUB_KEY=yourkey
 GITHUB_SECRET=yoursecret

.gitlab-ci.yml

@@ -11,8 +11,6 @@ variables:
   REPOSITORY_URL: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${AppUrl}/backend
   ECRBackendRepositoryName: ${AppUrl}/backend
   GitSHA: $CI_COMMIT_SHORT_SHA
-  AWSAccessKeyId: $AWS_ACCESS_KEY_ID
-  AWSSecretAccessKey: $AWS_SECRET_ACCESS_KEY
 
 pages:
   image: node:latest
@@ -145,13 +143,11 @@ Build Quasar PWA Assets:
   only:
     - master
   variables:
-    VUE_APP_CI_COMMIT_SHORT_SHA: "$CI_COMMIT_SHORT_SHA"
-    VUE_APP_CI_JOB_URL: "$CI_JOB_URL"
-    VUE_APP_API_BASE_URL: "https://api.${AppUrl}"
-    VUE_APP_BASE_URL: "https://${AppUrl}"
-    VUE_APP_WS_PROTOCOL: wss://
-    VUE_APP_BASE_HOST: api.${AppUrl}
-    VUE_APP_SITE_DOMAIN: ${AppUrl}
+    DOMAIN_NAME: "api.${AppUrl}"
+    GOOGLE_OAUTH2_KEY: google123
+    GITHUB_KEY: github123
+    WS_PROTOCOL: wss
+    HTTP_PROTOCOL: https
   artifacts:
     paths:
       - quasar/dist/pwa
@@ -172,7 +168,6 @@ Release backend:
   services:
     - docker:dind
   before_script:
-    - cd backend
     - apk add --no-cache curl jq python py-pip
     - pip install awscli
   script:
@@ -181,17 +176,17 @@ Release backend:
         -u $CI_REGISTRY_USER \
         -p $CI_REGISTRY_PASSWORD \
         $CI_REGISTRY
-    - docker pull $CI_REGISTRY_IMAGE:latest || true
+    - docker pull $CI_REGISTRY_IMAGE/backend:latest || true
     - |
       docker build \
-        --cache-from $CI_REGISTRY_IMAGE:latest \
+        --cache-from $CI_REGISTRY_IMAGE/backend:latest \
         --target production \
         --tag $REPOSITORY_URL:${GitSHA} \
-        --file scripts/prod/Dockerfile .
+        --file backend/scripts/prod/Dockerfile .
     - $(aws ecr get-login --no-include-email --region us-east-1)
     - docker push $REPOSITORY_URL:${GitSHA}
   after_script:
-    - echo "The production image has been released from the GitLab to ECR"
+    - echo "The production image has been released from GitLab to ECR"
 
 Sync Quasar PWA Assets:
   image: python:3.7
@@ -213,7 +208,7 @@ Sync Quasar PWA Assets:
         ./dist/pwa/ s3://${AppUrl}/
     - |
       aws s3 cp \
-        --cache-control 'max-age=0'
+        --cache-control 'max-age=0' \
         ./dist/pwa/index.html s3://${AppUrl}/
     - |
       aws s3 cp \
@@ -224,9 +219,7 @@ Sync Quasar PWA Assets:
 
 .Create stack:
   image: python:3.7
-  stage: cloudformation
-  only:
-    - master
+  stage: deploy
   variables:
     EnvironmentName: staging
   before_script:
@@ -246,7 +239,7 @@ Sync Quasar PWA Assets:
 
 Update stack:
   image: python:3.7
-  stage: cloudformation
+  stage: deploy
   only:
     - master
   variables:
@@ -265,3 +258,29 @@ Update stack:
         --parameters file://./parameters.json
   after_script:
     - echo "Update stack complete"
+
+collectstatic: &task
+  image: python:3.7
+  stage: deploy
+  only:
+    - master
+  variables:
+    EnvironmentName: staging
+  before_script:
+    - pip install awscli
+  when: manual
+  script:
+    - |
+      aws ecs run-task \
+        --cluster ${EnvironmentName}-cluster \
+        --task-definition collectstatic
+
+migrate:
+  <<: *task
+  variables:
+    EnvironmentName: staging
+  script:
+    - |
+      aws ecs run-task \
+        --cluster ${EnvironmentName}-cluster \
+        --task-definition migrate

backend/backend/settings/base.py

@@ -36,6 +36,7 @@
 CORS_ALLOW_HEADERS = default_headers + (
     'access-control-allow-headers',
     'access-control-allow-methods',
+    'access-control-allow-origin'
 )
 
 # Application definition
@@ -102,6 +103,7 @@
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'social_django.middleware.SocialAuthExceptionMiddleware',
+    'corsheaders.middleware.CorsMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
@@ -136,33 +138,33 @@
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': os.environ.get('RDS_DB_NAME', 'postgres'),
-        'USER': os.environ.get('RDS_USERNAME', 'postgres'),
-        'PASSWORD': os.environ.get('RDS_PASSWORD', 'postgres'),
-        'HOST': os.environ.get('RDS_HOSTNAME', 'postgres'),
-        'PORT': os.environ.get('RDS_PORT', 5432),
+        'NAME': os.environ.get('POSTGRES_NAME', 'postgres'),
+        'USER': os.environ.get('POSTGRES_USERNAME', 'postgres'),
+        'PASSWORD': os.environ.get('POSTGRES_PASSWORD', 'postgres'),
+        'HOST': os.environ.get('POSTGRES_SERVICE_HOST', 'postgres'),
+        'PORT': os.environ.get('POSTGRES_SERVICE_PORT', 5432),
     }
 }
 
 ASGI_APPLICATION = 'backend.routing.application'
 
-ELASTICACHE_REDIS_HOST_NAME = \
+REDIS_SERVICE_HOST = \
     os.environ.get(
-        'CELERY_BROKER_URL',
+        'REDIS_SERVICE_HOST',
         'redis://redis:6379'
-    )[8:].split(':')[0]
+    )
 
 CHANNEL_LAYERS = {
     'default': {
         'BACKEND': 'channels_redis.core.RedisChannelLayer',
         'CONFIG': {
-            "hosts": [(ELASTICACHE_REDIS_HOST_NAME, 6379)],
+            "hosts": [(REDIS_SERVICE_HOST, 6379)],
         },
     },
 }
 
 REDIS = redis.Redis(
-    host=ELASTICACHE_REDIS_HOST_NAME,
+    host=REDIS_SERVICE_HOST,
     port=6379,
     db=3,
     charset="utf-8",
@@ -186,10 +188,6 @@
 
 # Celery Configuration
 
-CELERY_BROKER_URL = \
-    os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379')
-CELERY_RESULT_BACKEND = \
-    os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379')
 CELERY_ACCEPT_CONTENT = ['application/json']
 CELERY_TASK_SERIALIZER = 'json'
 CELERY_RESULT_SERIALIZER = 'json'
@@ -234,8 +232,7 @@
 # https://docs.djangoproject.com/en/2.1/howto/static-files/
 
 AWS_DEFAULT_ACL = None
-AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', 'key_id')
-AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', 'key')
+
 AWS_STORAGE_BUCKET_NAME = os.environ.get(
     'AWS_STORAGE_BUCKET_NAME', 'bucketname')
 AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"

backend/backend/settings/development.py

@@ -2,16 +2,12 @@
 
 SECRET_KEY = "my-secret-key"
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': os.environ.get('RDS_DB_NAME', 'postgres'), # noqa
-        'USER': os.environ.get('RDS_USERNAME', 'postgres'), # noqa
-        'PASSWORD': os.environ.get('RDS_PASSWORD', 'postgres'), # noqa
-        'HOST': os.environ.get('RDS_HOSTNAME', 'postgres'), # noqa
-        'PORT': os.environ.get('RDS_PORT', 5432), # noqa
-    }
-}
+# Celery
+
+CELERY_BROKER_URL = \
+    os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379')  # noqa
+CELERY_RESULT_BACKEND = \
+    os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379')  # noqa
 
 DEBUG_APPS = [
     'django_extensions',
@@ -65,6 +61,9 @@ def show_toolbar(request):
     '--no-browser',
 ]
 
+AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', 'key_id')  # noqa
+AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', 'key')  # noqa
+
 STATIC_URL = '/static/'
 
 STATIC_ROOT = '/static/'

backend/backend/settings/production.py

@@ -36,3 +36,8 @@
         },
     },
 }
+
+# Celery
+
+CELERY_BROKER_URL = f"redis://{REDIS_SERVICE_HOST}:6379/0"  # noqa
+CELERY_RESULT_BACKEND = f"redis://{REDIS_SERVICE_HOST}:6379/0"  # noqa

backend/core/views.py

@@ -45,7 +45,9 @@ def hello_world(request):
     response = JsonResponse(
         {
             'message': 'Hello, World!',
-            'git_sha': os.environ.get('GIT_SHA', '<git sha>')
+            'git_sha': os.environ.get('GIT_SHA', '<git SHA>'),
+            'debug': settings.DEBUG,
+            'format': 'JSON'
         }
     )
     return response

backend/scripts/prod/start_prod.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
 python3 manage.py collectstatic --no-input
-python3 manage.py makemigrations
 python3 manage.py migrate --no-input
 gunicorn -t 300 -b 0.0.0.0:8000 backend.wsgi

cloudformation/infrastructure/ecr-repository.yaml

@@ -2,6 +2,11 @@ Description: >
   This template deploys an ECR Repository that we will use when pushing our backend container images.
 
 Parameters:
+
+  StackName:
+    Type: String
+    Description: The name of the stack
+
   ECRBackendRepositoryName:
     Type: "String"
     Description: "The name of the ECR Repository for our backend conainer."
@@ -13,5 +18,6 @@ Resources:
       RepositoryName: !Ref ECRBackendRepositoryName
 
 Outputs:
-  ECRBackendRepositoryURL:
+  EcrBackendRepo:
+    Description: The ECR Repository for the main backend container
     Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ECRBackendRepositoryName}"

cloudformation/infrastructure/ecs-cluster.yaml

@@ -3,6 +3,10 @@ Description: >
 
 Parameters:
 
+  StackName:
+    Type: String
+    Description: The name of the stack
+
   EnvironmentName:
     Type: String
     Description: An environemnt name to namespace cluster resources (e.g. qa, staging, production)
@@ -17,10 +21,6 @@ Parameters:
      r5.large, r5.xlarge, r5.2xlarge, r5.4xlarge, r5.12xlarge, r5.24xlarge ]
     ConstraintDescription: Please choose a valid instance type.
 
-  VPC:
-    Description: Main VPC
-    Type: AWS::EC2::VPC::Id
-
   LoadBalancerSecurityGroup:
     Type: AWS::EC2::SecurityGroup::Id
     Description: Security group for the load balancer
@@ -57,12 +57,16 @@ Resources:
 
   ECSCluster:
     Type: AWS::ECS::Cluster
+    Properties:
+      ClusterName: !Sub ${EnvironmentName}-cluster
 
   ContainerInstanceSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
       GroupDescription: Access to the ECS hosts that run containers.
-      VpcId: !Ref 'VPC'
+      VpcId:
+        Fn::ImportValue:
+          !Sub ${StackName}:VpcId
       SecurityGroupIngress:
         - SourceSecurityGroupId: !Ref LoadBalancerSecurityGroup
           IpProtocol: -1
@@ -173,6 +177,29 @@ Resources:
         - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM
         - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy
       Policies:
+        - PolicyName: s3-access
+          PolicyDocument: !Sub |
+            {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "VisualEditor0",
+                        "Effect": "Allow",
+                        "Action": [
+                            "s3:PutObject",
+                            "s3:GetObjectAcl",
+                            "s3:GetObject",
+                            "s3:ListBucket",
+                            "s3:DeleteObject",
+                            "s3:PutObjectAcl"
+                        ],
+                        "Resource": [
+                            "arn:aws:s3:::${StackName}-assets/*",
+                            "arn:aws:s3:::${StackName}-assets"
+                        ]
+                    }
+                ]
+            }
         - PolicyName: ecs-service
           PolicyDocument: |
             {
@@ -318,7 +345,7 @@ Outputs:
     Description: The name of the ECS cluster
     Value: !Ref 'ECSCluster'
     Export:
-      Name: !Sub ${EnvironmentName}:ClusterName
+      Name: !Sub ${StackName}:ECSCluster
 
   AutoScalingRole:
     Description: The ARN of the role used for auto-scaling