Implement the report using the Tern

This commit implements the report using Tern The reports use the ``concurrent.futures``, but instead of building all the implementation for it, we use flask-executer that makes the implementation on top of Flask. All the report requests go to the ``tern-tasks`` (Futures) and are handled such as a Celery implementation. All tasks start with the status 'UNKNOWN', which means that the task has no defined status until the ``tern-tasks`` start processing it. Once the task is in picked up by the ``tern-tasks`` it is moved to 'RUNNING'. If the processing finishes, the status is moved to 'FINISH' independently of the task content/result. Here the status is for the task itself. The FAIL status is given when the task cannot finish its routine. The Tern, for now, is called using ``subprocess``, and the output is handled by the stdout and stderr. The initial API specification was changed a bit for more clearness. How to run as Docker container was added to the README.md. Signed-off-by: Kairo de Araujo <[email protected]>
tern-tools · Mar 9, 2022 · 6233661 · 6233661
1 parent 4086cba
commit 6233661
Show file tree

Hide file tree

Showing 16 changed files with 456 additions and 167 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,21 @@
+tests/
+docs/
+cached/
+.github
+.gitignore
+Dockerfile
+Makefile
+Pipfile
+Pipfile.lock
+README.md
+requirements-dev.txt
+setup.py
+tox.ini
+.coverage
+.dockerignore
+.git
+.mypy_cache
+.pytest_cache
+.tox
+__pycache__
+tern_rest_api.egg-info
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
+# tern-rest-api specifics
+cached
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,29 @@
+# Copyright (c) 2022 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+FROM python:3.9-slim-buster as base
+
+RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \
+    && echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences.d/bullseye \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    attr \
+    findutils \
+    fuse-overlayfs/bullseye \
+    fuse3/bullseye \
+    git \
+    jq \
+    skopeo \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /opt/tern-rest-api
+
+ADD . /opt/tern-rest-api
+WORKDIR /opt/tern-rest-api
+RUN pip install --no-cache -r ./requirements.txt
+
+ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cached
+ENV TERN_DEFAULT_REGISTRY="registry.hub.docker.com"
+
+ENV FLASK_APP=/opt/tern-rest-api/app.py
+CMD ["bash", "docker_start.sh"]
diff --git a/Makefile b/Makefile
@@ -0,0 +1,6 @@
+build-dev:
+	docker build -t tern-rest-api:dev .
+
+
+serve-dev: build-dev
+	docker run --rm --name tern-rest-api -e ENVIRONMENT=DEVELOPMENT --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(PWD):/opt/tern-rest-api -p 5001:80 tern-rest-api:dev
diff --git a/Pipfile b/Pipfile
@@ -7,6 +7,8 @@ name = "pypi"
 flask = "*"
 flask-restx = "*"
 tern = "*"
+flask-executor = "*"
+gunicorn = "*"
 
 [dev-packages]
 black = "*"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -59,7 +59,13 @@ $ pipenv install -d
 
 ### Running the development Tern REST API
 
+## As a Docker Container
+```shell
+$ make server-dev
+```
+Open http://localhost/ in your browser.
 
+## On your local machine
 Runing the API locally
 
 ```shell

diff --git a/app.py b/app.py
@@ -8,7 +8,7 @@
 
 from flask_restx import Api
 
-from tern_api import __version__, tern_api
+from tern_api import __version__, tern_app
 from tern_api.api.v1.common_models import api_models_namespace
 from tern_api.api.v1.reports import ns as report_v1
 from tern_api.api.v1.version import ns as version_v1
@@ -26,20 +26,21 @@
 
 
 api = Api(
-    tern_api,
+    tern_app,
     version=__version__.version,
     title="Tern REST API",
     description="Tern Project REST API",
 )
 
+
 api.add_namespace(api_models_namespace)
 api.add_namespace(version_v1, path="/api/v1/version")
 api.add_namespace(report_v1, path="/api/v1/report")
 
 
 def export_swagger_json(filepath):
-    tern_api.config["SERVER_NAME"] = "localhost"
-    with tern_api.app_context().__enter__():
+    tern_app.config["SERVER_NAME"] = "localhost"
+    with tern_app.app_context().__enter__():
         with open(filepath, "w") as f:
             swagger_json = json.dumps(api.__schema__, indent=4)
             f.write(swagger_json)
diff --git a/docker_start.sh b/docker_start.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ ${ENVIRONMENT^^} == "DEVELOPMENT" ]]; then
+    flask run --reload --debugger -h 0.0.0.0 -p 80
+else
+    gunicorn --workers=1 -b 0.0.0.0:80 app:tern_api
+fi
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -15,32 +15,34 @@ black==22.1.0
 certifi==2021.10.8
 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 charset-normalizer==2.0.10; python_version >= '3'
-click==8.0.3; python_version >= '3.6'
-coverage==6.3.1
+click==8.0.4; python_version >= '3.6'
+coverage==6.3.2
 debian-inspector==30.0.0; python_version >= '3.6' and python_version < '4'
 distlib==0.3.4
 docker==5.0.3; python_version >= '3.6'
 dockerfile-parse==1.2.0
-filelock==3.5.1; python_version >= '3.7'
+filelock==3.6.0; python_version >= '3.7'
 flake8==4.0.1
+flask-executor==0.10.0
 flask-restx==0.5.1
 flask==2.0.3
 gitdb==4.0.9; python_version >= '3.6'
 gitpython==3.1.26; python_version >= '3.7'
+gunicorn==20.1.0
 idna==3.3; python_version >= '3'
 iniconfig==1.1.1
 isort==5.10.1
-itsdangerous==2.0.1; python_version >= '3.6'
+itsdangerous==2.1.0; python_version >= '3.7'
 jinja2==3.0.3; python_version >= '3.6'
 jsonschema==4.4.0; python_version >= '3.7'
-markupsafe==2.0.1; python_version >= '3.6'
+markupsafe==2.1.0; python_version >= '3.7'
 mccabe==0.6.1
 mypy-extensions==0.4.3
 packageurl-python==0.9.6; python_version >= '3.6'
 packaging==21.3; python_version >= '3.6'
 pathspec==0.9.0
 pbr==5.8.0; python_version >= '2.6'
-platformdirs==2.5.0; python_version >= '3.7'
+platformdirs==2.5.1; python_version >= '3.7'
 pluggy==1.0.0; python_version >= '3.6'
 prettytable==3.0.0; python_version >= '3.7'
 py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
@@ -62,7 +64,7 @@ tomli==2.0.1; python_version >= '3.7'
 tox==3.24.5
 typing-extensions==4.1.1; python_version < '3.10'
 urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-virtualenv==20.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+virtualenv==20.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 wcwidth==0.2.5
 websocket-client==1.2.3; python_version >= '3.6'
 werkzeug==2.0.3; python_version >= '3.6'
diff --git a/requirements.txt b/requirements.txt
@@ -11,19 +11,21 @@ attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2,
 certifi==2021.10.8
 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 charset-normalizer==2.0.10; python_version >= '3'
-click==8.0.3; python_version >= '3.6'
+click==8.0.4; python_version >= '3.6'
 debian-inspector==30.0.0; python_version >= '3.6' and python_version < '4'
 docker==5.0.3; python_version >= '3.6'
 dockerfile-parse==1.2.0
+flask-executor==0.10.0
 flask-restx==0.5.1
 flask==2.0.3
 gitdb==4.0.9; python_version >= '3.6'
 gitpython==3.1.26; python_version >= '3.7'
+gunicorn==20.1.0
 idna==3.3; python_version >= '3'
-itsdangerous==2.0.1; python_version >= '3.6'
+itsdangerous==2.1.0; python_version >= '3.7'
 jinja2==3.0.3; python_version >= '3.6'
 jsonschema==4.4.0; python_version >= '3.7'
-markupsafe==2.0.1; python_version >= '3.6'
+markupsafe==2.1.0; python_version >= '3.7'
 packageurl-python==0.9.6; python_version >= '3.6'
 pbr==5.8.0; python_version >= '2.6'
 prettytable==3.0.0; python_version >= '3.7'

diff --git a/tern_api/__init__.py b/tern_api/__init__.py
@@ -1,7 +1,15 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
+import os
+
 from flask import Flask
+from flask_executor import Executor
+
+tern_app = Flask(__name__)
+tern_app.config["TERN_API_CACHE_DIR"] = os.getenv("TERN_API_CACHE_DIR")
+tern_app.config["TERN_DEFAULT_REGISTRY"] = os.getenv("TERN_DEFAULT_REGISTRY")
 
-tern_api = Flask(__name__)
+tern_tasks = Executor(tern_app)
diff --git a/tern_api/api/v1/common_models.py b/tern_api/api/v1/common_models.py
@@ -18,21 +18,6 @@
     },
 )
 
-async_response_model = api_models_namespace.model(
-    "async_response_model",
-    {
-        "message": fields.String(
-            description="Status message",
-            requored=True,
-            example="Request submitted.",
-        ),
-        "id": fields.String(
-            description="Unique Identification for request",
-            required=True,
-            example="19f035a711644eab84ef5a38ceb5572e",
-        ),
-    },
-)
 
 image_report_data = api_models_namespace.model(
     "image_report_data",
@@ -52,6 +37,12 @@
             example="3.0",
             required=True,
         ),
+        "cache": fields.String(
+            description="Use cache if available?",
+            exampple=True,
+            required=True,
+            default=True,
+        ),
     },
 )
 image_report_model = api_models_namespace.model(
@@ -60,3 +51,24 @@
 report_model = api_models_namespace.model(
     "report_mode", {"images": fields.List(fields.Nested(image_report_model))}
 )
+
+async_response_model = api_models_namespace.model(
+    "async_response_model",
+    {
+        "message": fields.String(
+            description="Status message",
+            requored=True,
+            example="Request submitted.",
+        ),
+        "id": fields.String(
+            description="Unique Identification for request",
+            required=True,
+            example="19f035a711644eab84ef5a38ceb5572e",
+        ),
+        "cache": fields.Boolean(
+            description="Request uses cache?",
+            required=True,
+            example=True,
+        ),
+    },
+)
diff --git a/tern_api/api/v1/reports.py b/tern_api/api/v1/reports.py
@@ -3,8 +3,10 @@
 #
 # Copyright (c) 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
+from flask import request
 from flask_restx import Namespace, Resource, fields
 
+from tern_api import reports, tern_app
 from tern_api.api.v1.common_models import (
     async_response_model,
     error_model,
@@ -22,8 +24,8 @@ class Report(Resource):
             "registry": fields.String(
                 description="Registry Server",
                 required=False,
-                default="https://registry.hub.docker.com",
-                example="http://registry.example.com",
+                default=tern_app.config["TERN_DEFAULT_REGISTRY"],
+                example=tern_app.config["TERN_DEFAULT_REGISTRY"],
             ),
             "image": fields.String(
                 description="Image name",
@@ -35,6 +37,11 @@ class Report(Resource):
                 required=True,
                 example="3.0",
             ),
+            "cache": fields.Boolean(
+                description="Use cache data if available?",
+                required=True,
+                example=True,
+            ),
         },
     )
     report_response_request = ns.model(
@@ -52,6 +59,9 @@ def post(self):
 
         **Note**: This request will be processed assynchronous.
         """
+        payload = request.json
+        response = reports.request(payload)
+        return response.to_response()
 
 
 @ns.route("/status")
@@ -72,8 +82,8 @@ class ReportStatus(Resource):
             "status": fields.String(
                 description="Status of request",
                 required=True,
-                example="DONE",
-                enum=["UNKNOWN", "FAILED", "DONE"],
+                example="FINISH",
+                enum=["UNKNOWN", "FINISH", "RUNNING", "FAIL"],
             ),
             "result": fields.Nested(report_model),
         },
@@ -90,3 +100,7 @@ class ReportStatus(Resource):
     @ns.expect(report_status_parameters)
     def post(self):
         """Request Tern BoM report status/result"""
+
+        payload = request.json
+        response = reports.status(payload.get("id"))
+        return response.to_response()
diff --git a/tern_api/constants.py b/tern_api/constants.py
@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class task_status(Enum):
+    UNKNOWN = "UNKNOWN"
+    RUNNING = "RUNNING"
+    FINISH = "FINISH"
+    FAIL = "FAIL"