Job output transform

Makefile

@@ -228,10 +228,10 @@ conda-env-export:		## export the conda environment
 install: install-all    ## alias for 'install-all' target
 
 .PHONY: install-run
-install-run: conda-install install-sys install-pkg install-raw 	## install requirements and application to run locally
+install-run: conda-install install-sys install-pkg install-raw install-transform ## install requirements and application to run locally
 
 .PHONY: install-all
-install-all: conda-install install-sys install-pkg install-pip install-dev  ## install application with all dependencies
+install-all: conda-install install-sys install-pkg install-pip install-transform ## install application with all dependencies
 
 .PHONY: install-doc
 install-doc: install-pip	## install documentation dependencies
@@ -298,6 +298,28 @@ install-npm-remarklint: install-npm    ## install remark-lint dependency for 'ch
 		 	remark-preset-lint-markdown-style-guide \
 	)
 
+y.PHONY: install-transform
+install-transform: install-cairo-dependencies       # install-trfm-dependencies install-gdal
+
+
+#.PHONY: install-gdal
+#install-gdal:    ## install GDAL
+#	@echo "Installing conda packages for GDAL"
+#	@bash -c '$(CONDA_CMD) conda install -c conda-forge gdal'
+#	@echo "Install GDAL complete."
+#
+#.PHONY: install-trfm-dependencies
+#install-trfm-dependencies:   ## install required dependencies for Transformer
+#	@echo "Install required dependencies for Transformer"
+#	@bash -c '$(CONDA_CMD) pip install --upgrade -r "$(APP_ROOT)/requirements-trfm.txt"'
+#	@echo "Install with pip complete."
+
+.PHONY: install-cairo-dependencies
+install-cairo-dependencies:   ## install required dependencies for Transformer
+	@[ -f "$(shell which cairo)" ] || ( \
+		echo "Binary package manager cairo not found. Attempting to install it."; \
+		apt-get install libpangocairo-1.0-0 \
+	)
 ## -- Cleanup targets ----------------------------------------------------------------------------------------------- ##
 
 .PHONY: clean
@@ -532,6 +554,7 @@ check-security-deps-only: mkdir-reports  ## run security checks on package depen
 			-r "$(APP_ROOT)/requirements-dev.txt" \
 			-r "$(APP_ROOT)/requirements-doc.txt" \
 			-r "$(APP_ROOT)/requirements-sys.txt" \
+			-r "$(APP_ROOT)/requirements-trfm.txt" \
 			$(SAFETY_IGNORE) \
 		1> >(tee "$(REPORTS_DIR)/check-security-deps.txt")'
 
@@ -834,6 +857,14 @@ docker-clean:  ## remove all built docker images (only matching current/latest v
 	docker rmi -f "$(APP_NAME):latest" || true
 	docker rmi -f "$(APP_NAME):base" || true
 
+.PHONY: transformer
+fix-fstring-only: mkdir-reports
+	@echo "Fixing code string formats substitutions to f-string definitions..."
+	@-rm -f "$(REPORTS_DIR)/fixed-fstring.txt"
+	@bash -c '$(CONDA_CMD) \
+		flynt $(FLYNT_FLAGS) "$(APP_ROOT)" \
+		1> >(tee "$(REPORTS_DIR)/fixed-fstring.txt")'
+
 ## -- Launchers targets --------------------------------------------------------------------------------------------- ##
 
 .PHONY: start
@@ -848,3 +879,5 @@ stop: 		## kill application instance(s) started with gunicorn (pserve)
 .PHONY: stat
 stat: 		## display processes with PID(s) of gunicorn (pserve) instance(s) running the application
 	@lsof -i :4001 || echo "No instance running"
+
+

config/weaver.ini.example

@@ -15,7 +15,7 @@ pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 
 # mongodb
-mongodb.host = mongodb
+mongodb.host = 127.0.0.1
 mongodb.port = 27017
 mongodb.db_name = weaver
 
@@ -48,7 +48,7 @@ cache.quotation.enabled = true
 #   EMS:     current instance dispatches execution to other remote ADES or WPS providers
 #   HYBRID:  current instance does both ADES and EMS roles with flexible combinations depending on situation
 weaver.configuration = HYBRID
-weaver.url = http://localhost:4001
+weaver.url = http://127.0.0.1:4002
 
 # Static endpoint to external schema locations for reference in responses.
 # If not provided, the current instance OpenAPI JSON path is employed instead.
@@ -63,7 +63,7 @@ weaver.log_level =
 # SSL verification should be enabled for secured connections
 # setting is available for convenience, debug purposes and local environments
 # preferably, use specific request options
-weaver.ssl_verify = true
+weaver.ssl_verify = false
 # file with request options to be used with 'weaver.utils.request_extra'
 # see 'requests_options.yml.example'
 weaver.request_options =
@@ -111,7 +111,7 @@ weaver.wps_output_url =
 weaver.wps_output_path = /wpsoutputs
 weaver.wps_output_s3_bucket =
 weaver.wps_output_s3_region =
-weaver.wps_workdir =
+weaver.wps_workdir = /tmp/weaver/wps_workdir
 
 # --- Weaver WPS metadata ---
 # all attributes under "metadata:main" can be specified as 'weaver.wps_metadata_<field>'
@@ -171,7 +171,7 @@ weaver.vault_dir = /tmp/vault
 ###
 [celery]
 #USE_CELERYCONFIG = True
-BROKER_URL = mongodb://mongodb:27017/celery
+BROKER_URL = mongodb://127.0.0.1:27017/celery
 # Result backend is required for SYNC execution.
 #   Using only the backend type matching the broker URL will automatically resolve to use it database location.
 #   For an alternative result location, provide the full backend directly rather than using "mongodb_backend_settings".
@@ -191,7 +191,7 @@ RESULT_BACKEND = mongodb
 
 # --- gunicorn ---
 use = egg:gunicorn#main
-bind = 0.0.0.0:4001
+bind = 127.0.0.1:4002
 workers = 10
 timeout = 10
 

requirements-dev.txt

@@ -33,3 +33,13 @@ typing_extensions
 WebTest
 wsgiproxy
 WSGIProxy2
+
+# transformer
+fpdf
+Pillow
+pandas
+cairosvg
+multipagetiff
+# gdal
+rasterio
+python-magic

requirements.txt

@@ -53,6 +53,7 @@ json2xml<3.19.0; python_version <= "3.6"  # pyup: ignore
 # reduced dependencies contrains to let packages update to latest (https://github.com/vinitkumar/json2xml/issues/157)
 json2xml>=3.20.0; python_version >= "3.7"
 jsonschema>=3.0.1
+
 # FIXME: kombu for pymongo>=4 not yet released as 5.3.0 (only pre-releases available)
 # - https://github.com/crim-ca/weaver/issues/386
 # - https://github.com/celery/kombu/pull/1536
@@ -101,3 +102,13 @@ urlmatch
 xmltodict
 webob
 werkzeug>2
+
+# transformer
+fpdf
+Pillow
+pandas
+cairosvg
+multipagetiff
+# gdal
+rasterio
+python-magic

tests/resources/__init__.py

@@ -58,6 +58,8 @@
     TEST_REMOTE_SERVER_URL, WPS_NO_INPUTS_ID
 )
 
+WPS_TRANFORM_RESSOURCES = os.path.join(RESOURCES_PATH, "transform")
+
 
 def _load_path(file_path, text=False, xml=False):
     # type: (str, bool, bool) -> Union[JSON, xml_util.XML, str]