[IMPROVEMENT] Deprecate mod_deploy module and update Deployment pipeline workflow (#842)

* Update build commit and runCI script files

Signed-off-by: Tarun Arora <[email protected]>

* Remove mod_deploy module, update CD Github workflow

Signed-off-by: Tarun Arora <[email protected]>

* Add build_commit in flask globals

Signed-off-by: Tarun Arora <[email protected]>

* Add unittests to check when github response is invalid

Signed-off-by: Tarun Arora <[email protected]>

* Add unittests for request_from_github

Signed-off-by: Tarun Arora <[email protected]>

* Apply suggestions from code review

Co-authored-by: Shivam Kumar Jha <[email protected]>

---------

Signed-off-by: Tarun Arora <[email protected]>
Co-authored-by: Willem <[email protected]>
Co-authored-by: Shivam Kumar Jha <[email protected]>

Author: 3 people

.github/workflows/sp-deployment-pipeline.yml

@@ -29,8 +29,12 @@ jobs:
           script_stop: true
           command_timeout: 10m
           script: |
+            echo "defining directories"
+            INSTALL_FOLDER="/var/www/sample-platform"
+            SAMPLE_REPOSITORY="/repository"
+
             echo "jump to app folder"
-            cd /var/www/sample-platform
+            cd $INSTALL_FOLDER
 
             echo "checkout branch"
             sudo git restore .
@@ -50,5 +54,10 @@ jobs:
             echo "run migrations"
             sudo FLASK_APP=./run.py flask db upgrade
 
+            echo "update runCI script files"
+            sudo cp "install/ci-vm/ci-linux/ci/bootstrap" "${SAMPLE_REPOSITORY}/TestData/ci-linux/bootstrap"
+            sudo cp "install/ci-vm/ci-linux/ci/runCI" "${SAMPLE_REPOSITORY}/TestData/ci-linux/runCI"
+            sudo cp "install/ci-vm/ci-windows/ci/runCI.bat" "${SAMPLE_REPOSITORY}/TestData/ci-windows/runCI.bat"
+
             echo "reload server"
             sudo systemctl reload platform

.pycodestylerc

@@ -1,5 +1,5 @@
 [pycodestyle]
 count = True
 max-line-length = 120
-exclude=test_diff.py,migrations,venv*,parse.py
+exclude=test_diff.py,migrations,venv*,parse.py,config.py
 ignore = E701

config_sample.py

@@ -13,7 +13,6 @@
 EMAIL_DOMAIN = ''
 EMAIL_API_KEY = ''
 HMAC_KEY = ''
-GITHUB_DEPLOY_KEY = ''
 GITHUB_CI_KEY = ''
 GITHUB_CLIENT_ID = ''
 GITHUB_CLIENT_KEY = ''

decorators.py

@@ -4,6 +4,7 @@
 from functools import wraps
 from typing import Any, Callable, Dict, List, Optional, Union
 
+import git
 from flask import g, render_template, request
 
 from database import EnumSymbol
@@ -93,11 +94,7 @@ def decorated_function(*args, **kwargs):
             ctx['applicationName'] = 'CCExtractor CI platform'
             ctx['applicationVersion'] = getattr(g, 'version', 'Unknown')
             ctx['currentYear'] = date.today().strftime('%Y')
-            try:
-                from build_commit import build_commit
-            except ImportError:
-                build_commit = 'Unknown'
-            ctx['build_commit'] = build_commit
+            ctx['build_commit'] = getattr(g, 'build_commit', "Unknown")
             user = getattr(g, 'user', None)
             ctx['user'] = user
             # Create menu entries

install/install.sh

@@ -137,8 +137,6 @@ userInput email_domain "Email Domain:" $config_server_name "" 1
 echo "You can generate your own Email API key here, https://www.mailgun.com/)"
 userInput email_api_key "Email API key:" "" "Authentication, Email notification related functions" 0 
 hmac_key=$(head -80 /dev/urandom | LC_ALL=c tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
-echo "You can get details about creating deployment secrets here, https://developer.github.com/webhooks/"
-userInput github_deploy_key "GitHub Automated Deploy Webhook Secret:" "" "Automated deployment, mod_deploy module functions" 0 
 echo "You can get details about creating WEBHOOK_SECRET here, https://developer.github.com/webhooks/"
 userInput github_ci_key "GitHub CI Webhook Secret:" "" "CI related functions (mod_ci module)" 0
 echo "Get the GitHub client ID and client secret for the platform, by registering the platform here, https://github.com/settings/applications/new"
@@ -236,7 +234,6 @@ SERVER_NAME = '${server_name}'
 EMAIL_DOMAIN = '${email_domain}'
 EMAIL_API_KEY = '${email_api_key}'
 HMAC_KEY = '${hmac_key}'
-GITHUB_DEPLOY_KEY = '${github_deploy_key}'
 GITHUB_CI_KEY = '${github_ci_key}'
 GITHUB_CLIENT_ID = '${github_client_id}'
 GITHUB_CLIENT_KEY = '${github_client_secret_key}'

mod_ci/controllers.py

@@ -35,7 +35,6 @@
 from mod_ci.models import (BlockedUsers, GcpInstance, MaintenanceMode,
                            PrCommentInfo)
 from mod_customized.models import CustomizedTest
-from mod_deploy.controllers import is_valid_signature, request_from_github
 from mod_home.models import CCExtractorVersion, GeneralData
 from mod_regression.models import (Category, RegressionTest,
                                    RegressionTestOutput,
@@ -44,6 +43,7 @@
 from mod_sample.models import Issue
 from mod_test.models import (Fork, Test, TestPlatform, TestProgress,
                              TestResult, TestResultFile, TestStatus, TestType)
+from utility import is_valid_signature, request_from_github
 
 mod_ci = Blueprint('ci', __name__)
 

mod_deploy/__init__.py

@@ -7,6 +7,7 @@
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
+import git
 from flask import Flask, g
 from flask_migrate import Migrate
 from google.cloud.storage import Client
@@ -26,7 +27,6 @@
 from mod_auth.controllers import mod_auth
 from mod_ci.controllers import mod_ci
 from mod_customized.controllers import mod_customized
-from mod_deploy.controllers import mod_deploy
 from mod_home.controllers import mod_home
 from mod_regression.controllers import mod_regression
 from mod_sample.controllers import mod_sample
@@ -43,10 +43,7 @@
     raise MissingConfigError()
 
 app.config.from_mapping(config)
-try:
-    app.config['DEBUG'] = os.environ['DEBUG']
-except KeyError:
-    app.config['DEBUG'] = False
+app.config['DEBUG'] = os.environ.get('DEBUG', False)
 
 # embed flask-migrate in the app itself
 try:
@@ -67,6 +64,10 @@
 storage_client = Client.from_service_account_json(sa_file)
 storage_client_bucket = storage_client.bucket(app.config.get('GCS_BUCKET_NAME', ''))
 
+# Save build commit
+repo = git.Repo(search_parent_directories=True)
+app.config['BUILD_COMMIT'] = repo.head.object.hexsha
+
 
 def load_secret_keys(application: Flask, secret_session: str = 'secret_key',
                      secret_csrf: str = 'secret_csrf') -> None:
@@ -233,6 +234,7 @@ def before_request() -> None:
     g.version = "0.1"
     g.log = log
     g.github = get_github_config(app.config)
+    g.build_commit = app.config['BUILD_COMMIT']
 
 
 def get_github_config(config: Dict[str, str]) -> Dict[str, str]:
@@ -245,7 +247,6 @@ def get_github_config(config: Dict[str, str]) -> Dict[str, str]:
     :rtype: dict
     """
     return {
-        'deploy_key': config.get('GITHUB_DEPLOY_KEY', ''),
         'ci_key': config.get('GITHUB_CI_KEY', ''),
         'bot_token': config.get('GITHUB_TOKEN', ''),
         'bot_name': config.get('GITHUB_BOT', ''),
@@ -268,7 +269,6 @@ def teardown(exception: Optional[Exception]):
 app.register_blueprint(mod_regression, url_prefix='/regression')
 app.register_blueprint(mod_sample, url_prefix='/sample')
 app.register_blueprint(mod_home)
-app.register_blueprint(mod_deploy)
 app.register_blueprint(mod_test, url_prefix="/test")
 app.register_blueprint(mod_ci)
 app.register_blueprint(mod_customized, url_prefix='/custom')

mod_deploy/controllers.py

@@ -104,7 +104,6 @@ def load_config(file):
         'DATABASE_URI': 'sqlite:///:memory:',
         'WTF_CSRF_ENABLED': False,
         'SQLALCHEMY_POOL_SIZE': 1,
-        'GITHUB_DEPLOY_KEY': "test_deploy",
         'GITHUB_CI_KEY': "test_ci",
         'GITHUB_TOKEN': "",
         'GITHUB_BOT': "",