support json/xml according to accept header + add job status links (#58…

…) + extra job routes (#86) + compliance with ogc api

.dockerignore

@@ -25,6 +25,7 @@ env
 *.egg[s]
 *.egg-info
 *egg[s]__pycache__
+.python_history
 
 # Unit test / Coverage reports
 .cache

.gitignore

@@ -17,6 +17,7 @@ downloads
 *.egg-info
 *egg[s]
 __pycache__
+.python_history
 
 # Unit test / Coverage reports
 *.log

CHANGES.rst

@@ -352,14 +352,12 @@ Changes:
 Changes:
 --------
 
-- Add wps languages for other wps requests types: ``DescribeProcess`` and ``GetCapabilities``.
+- Add `WPS` languages for other wps requests types: ``DescribeProcess`` and ``GetCapabilities``.
 
 Fixes:
 ------
 
 - Fix a bug where the validation of ``OneOf`` items was casting the value to the first valid possibility.
-  Now, it doesn't change the value if it's valid without casting it (and still casts it if it's
-  necessary to make it valid).
 
 `1.1.0 <https://github.com/crim-ca/weaver/tree/1.1.0>`_ (2020-02-17)
 ========================================================================

config/weaver.ini.example

@@ -19,6 +19,12 @@ mongodb.host = mongodb
 mongodb.port = 27017
 mongodb.db_name = weaver
 
+# caching
+cache.regions = result
+cache.type = memory
+cache.result.expire = 3600
+cache.result.enabled = false
+
 # NOTE:
 #   For all below parameters, settings suffixed by `_url` are automatically generated from their corresponding `_path`
 #   settings using `weaver.url` if they are not provided. Otherwise, the explicit definition provided by `_url` suffixed

requirements.txt

@@ -1,4 +1,5 @@
 alembic
+beaker
 # AWS support (S3 buckets)
 boto3
 # celery 5 to be released in near future
@@ -27,9 +28,11 @@ oauthlib
 owslib>=0.19.2; python_version >= "3"
 pymongo
 pyramid>=1.7.3
+pyramid_beaker>=0.8
 pyramid_celery
 pyramid_mako
 python-dateutil
+pyramid_rewrite
 pytz
 # no viable pywps version with Python>3.5 dependencies
 # use '4.2.4' plus a few commits that provide fix, but not yet released

tests/wps_restapi/test_api.py

@@ -17,35 +17,35 @@ def setUpClass(cls):
         cls.json_headers = {"Accept": CONTENT_TYPE_APP_JSON, "Content-Type": CONTENT_TYPE_APP_JSON}
 
     def test_frontpage_format(self):
-        resp = self.testapp.get(sd.api_frontpage_uri, headers=self.json_headers)
+        resp = self.testapp.get(sd.api_frontpage_service.path, headers=self.json_headers)
         assert resp.status_code == 200
         try:
             sd.FrontpageSchema().deserialize(resp.json)
         except colander.Invalid as ex:
             self.fail("expected valid response format as defined in schema [{!s}]".format(ex))
 
     def test_version_format(self):
-        resp = self.testapp.get(sd.api_versions_uri, headers=self.json_headers)
+        resp = self.testapp.get(sd.api_versions_service.path, headers=self.json_headers)
         assert resp.status_code == 200
         try:
             sd.VersionsSchema().deserialize(resp.json)
         except colander.Invalid as ex:
             self.fail("expected valid response format as defined in schema [{!s}]".format(ex))
 
     def test_conformance_format(self):
-        resp = self.testapp.get(sd.api_conformance_uri, headers=self.json_headers)
+        resp = self.testapp.get(sd.api_conformance_service.path, headers=self.json_headers)
         assert resp.status_code == 200
         try:
             sd.ConformanceSchema().deserialize(resp.json)
         except colander.Invalid as ex:
             self.fail("expected valid response format as defined in schema [{!s}]".format(ex))
 
     def test_swagger_api_format(self):
-        resp = self.testapp.get(sd.api_swagger_ui_uri)
+        resp = self.testapp.get(sd.api_swagger_ui_service.path)
         assert resp.status_code == 200
         assert "<title>{}</title>".format(sd.API_TITLE) in resp.text
 
-        resp = self.testapp.get(sd.api_swagger_json_uri, headers=self.json_headers)
+        resp = self.testapp.get(sd.api_swagger_json_service.path, headers=self.json_headers)
         assert resp.status_code == 200
         assert "tags" in resp.json
         assert "info" in resp.json
@@ -60,10 +60,10 @@ def test_status_unauthorized_and_forbidden(self):
         Shouldn't be the default behaviour to employ 403 on both cases.
         """
         with mock.patch("weaver.wps_restapi.api.get_weaver_url", side_effect=HTTPUnauthorized):
-            resp = self.testapp.get(sd.api_frontpage_uri, headers=self.json_headers, expect_errors=True)
+            resp = self.testapp.get(sd.api_frontpage_service.path, headers=self.json_headers, expect_errors=True)
             assert resp.status_code == 401
         with mock.patch("weaver.wps_restapi.api.get_weaver_url", side_effect=HTTPForbidden):
-            resp = self.testapp.get(sd.api_frontpage_uri, headers=self.json_headers, expect_errors=True)
+            resp = self.testapp.get(sd.api_frontpage_service.path, headers=self.json_headers, expect_errors=True)
             assert resp.status_code == 403
 
     def test_status_not_found_and_method_not_allowed(self):
@@ -75,7 +75,7 @@ def test_status_not_found_and_method_not_allowed(self):
         assert resp.status_code == 404
 
         # test an existing route with wrong method, shouldn't be the default '404' on both cases
-        resp = self.testapp.post(sd.api_frontpage_uri, headers=self.json_headers, expect_errors=True)
+        resp = self.testapp.post(sd.api_frontpage_service.path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 405
 
 
@@ -139,7 +139,7 @@ def test_swagger_api_request_base_path_original(self):
         resp = testapp.get(sd.api_swagger_json_service.path, headers=self.json_headers)
         assert resp.status_code == 200, "API definition should be accessed directly"
         assert resp.json["host"] in [self.app_host, "{}:80".format(self.app_host)]
-        assert resp.json["basePath"] == sd.api_frontpage_uri
+        assert resp.json["basePath"] == sd.api_frontpage_service.path
 
         resp = testapp.get(sd.api_swagger_ui_service.path)
         assert resp.status_code == 200, "API definition should be accessed directly"

tests/wps_restapi/test_jobs.py

@@ -32,7 +32,7 @@
 from weaver.utils import get_path_kvp
 from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC
 from weaver.warning import TimeZoneInfoAlreadySetWarning
-from weaver.wps_restapi.swagger_definitions import jobs_full_uri, jobs_short_uri, process_jobs_uri
+from weaver.wps_restapi import swagger_definitions as sd
 
 if TYPE_CHECKING:
     from typing import Iterable, List, Tuple, Union
@@ -163,12 +163,16 @@ def check_job_format(job):
         assert "status" in job and isinstance(job["status"], str)
         assert "message" in job and isinstance(job["message"], str)
         assert "percentCompleted" in job and isinstance(job["percentCompleted"], int)
-        assert "logs" in job and isinstance(job["logs"], str)
+        assert "links" in job and isinstance(job["links"], list) and len(job["links"])
+        assert all(isinstance(link_info, dict) for link_info in job["links"])
+        assert all(any(link_info["rel"] == rel for link_info in job["links"]) for rel in ["self", "logs"])
+        for link_info in job["links"]:
+            assert "href" in link_info and isinstance(link_info["href"], str)
         assert job["status"] in JOB_STATUS_VALUES
         if job["status"] == STATUS_SUCCEEDED:
-            assert "result" in job and isinstance(job["result"], str)
+            assert len([link for link in job["links"] if link["rel"] == "results"])
         elif job["status"] == STATUS_FAILED:
-            assert "exceptions" in job and isinstance(job["exceptions"], str)
+            assert len([link for link in job["links"] if link["rel"] == "exceptions"])
 
     @staticmethod
     def check_basic_jobs_info(response):
@@ -204,21 +208,21 @@ def check_basic_jobs_grouped_info(response, groups):
         assert total == response.json["total"]
 
     def test_get_jobs_normal_paged(self):
-        resp = self.app.get(jobs_short_uri, headers=self.json_headers)
+        resp = self.app.get(sd.jobs_service.path, headers=self.json_headers)
         self.check_basic_jobs_info(resp)
         for job_id in resp.json["jobs"]:
             assert isinstance(job_id, str)
 
         for detail in ("false", 0, "False", "no", "None", "null", None, ""):
-            path = get_path_kvp(jobs_short_uri, detail=detail)
+            path = get_path_kvp(sd.jobs_service.path, detail=detail)
             resp = self.app.get(path, headers=self.json_headers)
             self.check_basic_jobs_info(resp)
             for job_id in resp.json["jobs"]:
                 assert isinstance(job_id, str)
 
     def test_get_jobs_detail_paged(self):
         for detail in ("true", 1, "True", "yes"):
-            path = get_path_kvp(jobs_short_uri, detail=detail)
+            path = get_path_kvp(sd.jobs_service.path, detail=detail)
             resp = self.app.get(path, headers=self.json_headers)
             self.check_basic_jobs_info(resp)
             for job in resp.json["jobs"]:
@@ -227,7 +231,7 @@ def test_get_jobs_detail_paged(self):
     def test_get_jobs_normal_grouped(self):
         for detail in ("false", 0, "False", "no"):
             groups = ["process", "service"]
-            path = get_path_kvp(jobs_short_uri, detail=detail, groups=groups)
+            path = get_path_kvp(sd.jobs_service.path, detail=detail, groups=groups)
             resp = self.app.get(path, headers=self.json_headers)
             self.check_basic_jobs_grouped_info(resp, groups=groups)
             for grouped_jobs in resp.json["groups"]:
@@ -237,15 +241,15 @@ def test_get_jobs_normal_grouped(self):
     def test_get_jobs_detail_grouped(self):
         for detail in ("true", 1, "True", "yes"):
             groups = ["process", "service"]
-            path = get_path_kvp(jobs_short_uri, detail=detail, groups=groups)
+            path = get_path_kvp(sd.jobs_service.path, detail=detail, groups=groups)
             resp = self.app.get(path, headers=self.json_headers)
             self.check_basic_jobs_grouped_info(resp, groups=groups)
             for grouped_jobs in resp.json["groups"]:
                 for job in grouped_jobs["jobs"]:
                     self.check_job_format(job)
 
     def test_get_jobs_valid_grouping_by_process(self):
-        path = get_path_kvp(jobs_short_uri, detail="false", groups="process")
+        path = get_path_kvp(sd.jobs_service.path, detail="false", groups="process")
         resp = self.app.get(path, headers=self.json_headers)
         self.check_basic_jobs_grouped_info(resp, groups="process")
 
@@ -272,7 +276,7 @@ def test_get_jobs_valid_grouping_by_process(self):
                 pytest.fail("Unknown job grouping 'process' value not expected.")
 
     def test_get_jobs_valid_grouping_by_service(self):
-        path = get_path_kvp(jobs_short_uri, detail="false", groups="service")
+        path = get_path_kvp(sd.jobs_service.path, detail="false", groups="service")
         resp = self.app.get(path, headers=self.json_headers)
         self.check_basic_jobs_grouped_info(resp, groups="service")
 
@@ -322,87 +326,88 @@ def test_get_jobs_by_encrypted_email(self):
         assert job.notification_email != email and job.notification_email is not None
         assert int(job.notification_email, 16) != 0  # email should be encrypted with hex string
 
-        path = get_path_kvp(jobs_short_uri, detail="true", notification_email=email)
+        path = get_path_kvp(sd.jobs_service.path, detail="true", notification_email=email)
         resp = self.app.get(path, headers=self.json_headers)
         assert resp.status_code == 200
         assert resp.content_type == CONTENT_TYPE_APP_JSON
         assert resp.json["total"] == 1, "Should match exactly 1 email with specified literal string as query param."
         assert resp.json["jobs"][0]["jobID"] == job_id
 
     def test_get_jobs_process_in_query_normal(self):
-        path = get_path_kvp(jobs_short_uri, process=self.job_info[0].process)
+        path = get_path_kvp(sd.jobs_service.path, process=self.job_info[0].process)
         resp = self.app.get(path, headers=self.json_headers)
         self.check_basic_jobs_info(resp)
         assert self.job_info[0].id in resp.json["jobs"], self.message_with_jobs_mapping("expected in")
         assert self.job_info[1].id not in resp.json["jobs"], self.message_with_jobs_mapping("expected not in")
 
     def test_get_jobs_process_in_query_detail(self):
-        path = get_path_kvp(jobs_short_uri, process=self.job_info[0].process, detail="true")
+        path = get_path_kvp(sd.jobs_service.path, process=self.job_info[0].process, detail="true")
         resp = self.app.get(path, headers=self.json_headers)
         self.check_basic_jobs_info(resp)
         job_ids = [j["jobID"] for j in resp.json["jobs"]]
         assert self.job_info[0].id in job_ids, self.message_with_jobs_mapping("expected in")
         assert self.job_info[1].id not in job_ids, self.message_with_jobs_mapping("expected not in")
 
     def test_get_jobs_process_in_path_normal(self):
-        path = process_jobs_uri.format(process_id=self.job_info[0].process)
+        path = sd.process_jobs_service.path.format(process_id=self.job_info[0].process)
         resp = self.app.get(path, headers=self.json_headers)
         self.check_basic_jobs_info(resp)
         assert self.job_info[0].id in resp.json["jobs"], self.message_with_jobs_mapping("expected in")
         assert self.job_info[1].id not in resp.json["jobs"], self.message_with_jobs_mapping("expected not in")
 
     def test_get_jobs_process_in_path_detail(self):
-        path = process_jobs_uri.format(process_id=self.job_info[0].process) + "?detail=true"
+        path = sd.process_jobs_service.path.format(process_id=self.job_info[0].process) + "?detail=true"
         resp = self.app.get(path, headers=self.json_headers)
         self.check_basic_jobs_info(resp)
         job_ids = [j["jobID"] for j in resp.json["jobs"]]
         assert self.job_info[0].id in job_ids, self.message_with_jobs_mapping("expected in")
         assert self.job_info[1].id not in job_ids, self.message_with_jobs_mapping("expected not in")
 
     def test_get_jobs_process_unknown_in_path(self):
-        path = process_jobs_uri.format(process_id="unknown-process-id")
+        path = sd.process_jobs_service.path.format(process_id="unknown-process-id")
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 404
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_process_unknown_in_query(self):
-        path = get_path_kvp(jobs_short_uri, process="unknown-process-id")
+        path = get_path_kvp(sd.jobs_service.path, process="unknown-process-id")
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 404
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_private_process_unauthorized_in_path(self):
-        path = process_jobs_uri.format(process_id=self.process_private.identifier)
+        path = sd.process_jobs_service.path.format(process_id=self.process_private.identifier)
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 401
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_private_process_not_returned_in_query(self):
-        path = get_path_kvp(jobs_short_uri, process=self.process_private.identifier)
+        path = get_path_kvp(sd.jobs_service.path, process=self.process_private.identifier)
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 401
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_service_and_process_unknown_in_path(self):
-        path = jobs_full_uri.format(provider_id="unknown-service-id", process_id="unknown-process-id")
+        path = sd.provider_jobs_service.path.format(provider_id="unknown-service-id", process_id="unknown-process-id")
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 404
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_service_and_process_unknown_in_query(self):
-        path = get_path_kvp(jobs_short_uri, service="unknown-service-id", process="unknown-process-id")
+        path = get_path_kvp(sd.jobs_service.path, service="unknown-service-id", process="unknown-process-id")
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 404
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_private_service_public_process_unauthorized_in_path(self):
-        path = jobs_full_uri.format(provider_id=self.service_private.name, process_id=self.process_public.identifier)
+        path = sd.provider_jobs_service.path.format(provider_id=self.service_private.name,
+                                                    process_id=self.process_public.identifier)
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
         assert resp.status_code == 401
         assert resp.content_type == CONTENT_TYPE_APP_JSON
 
     def test_get_jobs_private_service_public_process_unauthorized_in_query(self):
-        path = get_path_kvp(jobs_short_uri,
+        path = get_path_kvp(sd.jobs_service.path,
                             service=self.service_private.name,
                             process=self.process_public.identifier)
         resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
@@ -415,7 +420,7 @@ def test_get_jobs_public_service_private_process_unauthorized_in_query(self):
             it is up to the remote service to hide private processes
             if the process is visible, the a job can be executed and it is automatically considered public
         """
-        path = get_path_kvp(jobs_short_uri,
+        path = get_path_kvp(sd.jobs_service.path,
                             service=self.service_public.name,
                             process=self.process_private.identifier)
         with contextlib.ExitStack() as stack:
@@ -431,7 +436,7 @@ def test_get_jobs_public_service_no_processes(self):
             it is up to the remote service to hide private processes
             if the process is invisible, no job should have been executed nor can be fetched
         """
-        path = get_path_kvp(jobs_short_uri,
+        path = get_path_kvp(sd.jobs_service.path,
                             service=self.service_public.name,
                             process=self.process_private.identifier)
         with contextlib.ExitStack() as stack:
@@ -443,9 +448,9 @@ def test_get_jobs_public_service_no_processes(self):
 
     def test_get_jobs_public_with_access_and_request_user(self):
         """Verifies that corresponding processes are returned when proper access/user-id are respected."""
-        uri_direct_jobs = jobs_short_uri
-        uri_process_jobs = process_jobs_uri.format(process_id=self.process_public.identifier)
-        uri_provider_jobs = jobs_full_uri.format(
+        uri_direct_jobs = sd.jobs_service.path
+        uri_process_jobs = sd.process_jobs_service.path.format(process_id=self.process_public.identifier)
+        uri_provider_jobs = sd.provider_jobs_service.path.format(
             provider_id=self.service_public.name, process_id=self.process_public.identifier)
 
         admin_public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, self.job_info))

tests/wps_restapi/test_processes.py

@@ -553,7 +553,7 @@ def test_execute_process_dont_cast_one_of(self):
             resp = self.app.post_json(path, params=data_execute, headers=self.json_headers)
             assert resp.status_code == 201, "Expected job submission without inputs created without error."
             job = self.job_store.fetch_by_id(resp.json["jobID"])
-            assert job.inputs[0]["data"] == "100"  # not cast to float or integer
+            assert job.inputs[0]["value"] == "100"  # not cast to float or integer
 
     def test_execute_process_no_error_not_required_params(self):
         """