Merge pull request #68 from common-workflow-language/update-wes-1.0

Update to advertise support for WES 1.0.  Update README.

Author: tetron

README.md

@@ -1,10 +1,9 @@
 # Workflow as a Service
 
-This provides client and server implementations of the [GA4GH Workflow
-Execution Service](https://github.com/ga4gh/workflow-execution-schemas) API for
-the Common Workflow Language.
+This is a client and server implementation of the [GA4GH Workflow
+Execution Service](https://github.com/ga4gh/workflow-execution-schemas) 1.0.0 API.
 
-It provides an [Arvados](https://github.com/curoverse/arvados) backend.  It
+It provides [Arvados](https://arvados.org/) and [Toil](http://toil.ucsc-cgl.org/) backends.  It
 also works with any `cwl-runner` that supports the CWL standard command line
 interface: http://www.commonwl.org/v1.0/CommandLineTool.html#Executing_CWL_documents_as_scripts
 
@@ -16,43 +15,66 @@ pip install wes-service
 
 ## Usage
 
-Run a standalone server with default `cwl-runner` backend:
+### Client configuration
+
+Command line parameter or environment variable.
+
+`--host` or `WES_API_HOST`
+
+The host to contact.
+
+`--proto` or `WES_API_PROTO`
+
+The protocol (http or https) to use.
+
+`--auth` or `WES_API_AUTH`
+
+Credentials.  Format is 'Header: value' or just 'value'.  If header name is not provided, value goes in the 'Authorization'.
+
+### Get service info
 
 ```
-$ wes-server
+$ wes-client --info
 ```
 
 ### Submit a workflow to run:
 
-Note! All inputs files must be accessible from the filesystem.
+Attachments must be accessible from the filesystem.  Workflow runners may also support http URLs or other storage systems.
 
 ```
-$ wes-client --host=localhost:8080 --proto=http --attachments="testdata/dockstore-tool-md5sum.cwl,testdata/md5sum.input" testdata/md5sum.cwl testdata/md5sum.cwl.json
+$ wes-client --attachments="testdata/dockstore-tool-md5sum.cwl,testdata/md5sum.input" testdata/md5sum.cwl testdata/md5sum.cwl.json
 ```
 
 ### List workflows
 
 ```
-$ wes-client --proto http --host=localhost:8080 --list
+$ wes-client --list
 ```
 
 ### Get workflow status
 
 ```
-$ wes-client --proto http --host=localhost:8080 --get <workflow-id>
+$ wes-client --get <run-id>
 ```
 
 ### Get stderr log from workflow:
 
 ```
-$ wes-client --proto http --host=localhost:8080 --log <workflow-id>
+$ wes-client --log <run-id>
 ```
 
 ## Server Configuration
 
+### Run a standalone server with default `cwl-runner` backend:
+
+```
+$ wes-server
+```
+
 ### Run a standalone server with Arvados backend:
 
 ```
+$ pip install arvados-cwl-runner
 $ wes-server --backend=wes_service.arvados_wes
 ```
 
@@ -63,46 +85,21 @@ $ pip install toil[all]
 $ wes-server --backend=wes_service.toil_wes --opt extra=--clean=never
 ```
 
-### Use a different executable with cwl_runner backend
+### Use alternate executable with cwl-runner backend
 
 ```
-$ pip install toil[all]
-$ wes-server --backend=wes_service.cwl_runner --opt runner=cwltoil --opt extra=--logLevel=CRITICAL
+$ pip install cwltool
+$ wes-server --backend=wes_service.cwl_runner --opt runner=cwltool --opt extra=--logLevel=CRITICAL
 ```
 
 ### Pass parameters to cwl-runner
 
-```
-$ wes-server --backend=wes_service.cwl_runner --opt extra=--workDir=/
-```
-
-## Client Configuration
-
-These options will be read in as defaults when running the client from the
-command line. The default protocol is https, to support secure communications,
-but the server starts using http, to ease development.
-
-Set service endpoint:
-
-```
-$ export WES_API_HOST=localhost:8080
-```
-
-Set the value to pass in the `Authorization` header:
+Use "--opt" following by "key=value"
 
 ```
-$ export WES_API_AUTH=my_api_token
+$ wes-server --backend=wes_service.cwl_runner --opt extra=--workDir=/tmp/work
 ```
 
-Set the protocol (one of http, https)
-
-```
-$ export WES_API_PROTO=http
-```
-
-Then, when you call `wes-client` these defaults will be used in place of the
-flags, `--host`, `--auth`, and `proto` respectively.
-
 ## Development
 If you would like to develop against `workflow-service` make sure you pass the provided test and it is flake8 compliant
 
@@ -113,7 +110,7 @@ $ virtualenv venv && source venv/bin/activate && pip install toil[all] && pip in
 ```
 
 #### Running Tests
-From path `workflow-service` run 
+From path `workflow-service` run
 
 ```
 $ pytest && flake8

setup.py

@@ -11,7 +11,7 @@
     long_description = readmeFile.read()
 
 setup(name='wes-service',
-      version='2.9',
+      version='3.0',
       description='GA4GH Workflow Execution Service reference implementation',
       long_description=long_description,
       author='GA4GH Containers and Workflows task team',
@@ -25,9 +25,9 @@
       install_requires=[
           'future',
           'connexion==1.4.2',
-          'ruamel.yaml >= 0.12.4, < 0.15',
+          'ruamel.yaml >= 0.12.4, <= 0.15.77',
           'cwlref-runner==1.0',
-          'schema-salad>=2.6, <3',
+          'schema-salad >= 3.0, < 3.1',
           'subprocess32==3.5.2'
                         ],
       entry_points={
@@ -37,7 +37,7 @@
       extras_require={
           "arvados": ["arvados-cwl-runner"
                       ],
-          "toil": ["toil[all]==3.16.0"
+          "toil": ["toil[all]==3.18.0"
                    ]},
       zip_safe=False
       )

test/test_integration.py

@@ -8,6 +8,7 @@
 import shutil
 import logging
 import sys
+import requests
 
 pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))  # noqa
 sys.path.insert(0, pkg_root)  # noqa
@@ -23,7 +24,7 @@ class IntegrationTest(unittest.TestCase):
     def setUpClass(cls):
         # cwl
         cls.cwl_dockstore_url = 'https://dockstore.org:8443/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/master/plain-CWL/descriptor/%2FDockstore.cwl'
-        cls.cwl_local_path = os.path.abspath('testdata/md5sum.cwl')
+        cls.cwl_local_path = "file://" + os.path.abspath('testdata/md5sum.cwl')
         cls.cwl_json_input = "file://" + os.path.abspath('testdata/md5sum.json')
         cls.cwl_attachments = ['file://' + os.path.abspath('testdata/md5sum.input'),
                                'file://' + os.path.abspath('testdata/dockstore-tool-md5sum.cwl')]
@@ -52,33 +53,37 @@ def tearDown(self):
                     time.sleep(3)
                 except OSError as e:
                     print(e)
-        if os.path.exists('workflows'):
-            shutil.rmtree('workflows')
         unittest.TestCase.tearDown(self)
 
     def test_dockstore_md5sum(self):
         """HTTP md5sum cwl (dockstore), run it on the wes-service server, and check for the correct output."""
-        outfile_path, _ = self.run_md5sum(wf_input=self.cwl_dockstore_url,
+        outfile_path, run_id = self.run_md5sum(wf_input=self.cwl_dockstore_url,
                                           json_input=self.cwl_json_input,
                                           workflow_attachment=self.cwl_attachments)
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+        state = self.wait_for_finish(run_id)
+        self.check_complete(run_id)
+        self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
     def test_local_md5sum(self):
         """LOCAL md5sum cwl to the wes-service server, and check for the correct output."""
         outfile_path, run_id = self.run_md5sum(wf_input=self.cwl_local_path,
                                                json_input=self.cwl_json_input,
                                                workflow_attachment=self.cwl_attachments)
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+        state = self.wait_for_finish(run_id)
+        self.check_complete(run_id)
+        self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
     def test_run_attachments(self):
         """LOCAL md5sum cwl to the wes-service server, check for attachments."""
         outfile_path, run_id = self.run_md5sum(wf_input=self.cwl_local_path,
                                                json_input=self.cwl_json_input,
                                                workflow_attachment=self.cwl_attachments)
         get_response = self.client.get_run_log(run_id)["request"]
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + get_response["workflow_attachment"])
+        state = self.wait_for_finish(run_id)
+        self.check_complete(run_id)
+        self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + get_response["workflow_attachment"])
         attachment_tool_path = get_response["workflow_attachment"][7:] + "/dockstore-tool-md5sum.cwl"
-        self.assertTrue(check_for_file(attachment_tool_path), 'Attachment file was not found: ' + get_response["workflow_attachment"])
+        self.assertTrue(self.check_for_file(attachment_tool_path), 'Attachment file was not found: ' + get_response["workflow_attachment"])
 
     def test_get_service_info(self):
         """
@@ -90,7 +95,7 @@ def test_get_service_info(self):
         assert 'workflow_type_versions' in r
         assert 'supported_wes_versions' in r
         assert 'supported_filesystem_protocols' in r
-        assert 'engine_versions' in r
+        assert 'workflow_engine_versions' in r
 
     def test_list_runs(self):
         """
@@ -121,6 +126,37 @@ def run_md5sum(self, wf_input, json_input, workflow_attachment=None):
         output_dir = os.path.abspath(os.path.join('workflows', response['run_id'], 'outdir'))
         return os.path.join(output_dir, 'md5sum.txt'), response['run_id']
 
+    def wait_for_finish(self, run_id, seconds=120):
+        """Return True if a file exists within a certain amount of time."""
+        wait_counter = 0
+        r = self.client.get_run_status(run_id)
+        while r["state"] in ("QUEUED", "INITIALIZING", "RUNNING"):
+            time.sleep(1)
+            wait_counter += 1
+            if wait_counter > seconds:
+                return None
+            r = self.client.get_run_status(run_id)
+        return r["state"]
+
+    def check_complete(self, run_id):
+        s = self.client.get_run_log(run_id)
+        if s["state"] != "COMPLETE":
+            logging.info(str(s["run_log"]["stderr"]))
+            if str(s["run_log"]["stderr"]).startswith("http"):
+                logs = requests.get(s["run_log"]["stderr"], headers=self.client.auth).text
+                logging.info("Run log:\n" + logs)
+        assert s["state"] == "COMPLETE"
+
+    def check_for_file(self, filepath, seconds=120):
+        """Return True if a file exists within a certain amount of time."""
+        wait_counter = 0
+        while not os.path.exists(filepath):
+            time.sleep(1)
+            wait_counter += 1
+            if wait_counter > seconds:
+                return False
+        return True
+
 
 def get_server_pids():
     try:
@@ -130,16 +166,6 @@ def get_server_pids():
     return pids
 
 
-def check_for_file(filepath, seconds=120):
-    """Return True if a file exists within a certain amount of time."""
-    wait_counter = 0
-    while not os.path.exists(filepath):
-        time.sleep(1)
-        wait_counter += 1
-        if wait_counter > seconds:
-            return False
-    return True
-
 
 class CwltoolTest(IntegrationTest):
     """Test using cwltool."""
@@ -149,9 +175,13 @@ def setUp(self):
         Start a (local) wes-service server to make requests against.
         Use cwltool as the wes-service server 'backend'.
         """
+        if os.path.exists('workflows'):
+            shutil.rmtree('workflows')
         self.wes_server_process = subprocess.Popen(
-            'python {}'.format(os.path.abspath('wes_service/wes_service_main.py')),
-            shell=True)
+            ['python', os.path.abspath('wes_service/wes_service_main.py'),
+             '--backend=wes_service.cwl_runner',
+             '--port=8080',
+             '--debug'])
         time.sleep(5)
 
 
@@ -176,8 +206,31 @@ def test_local_wdl(self):
             outfile_path, run_id = self.run_md5sum(wf_input=self.wdl_local_path,
                                                    json_input=self.wdl_json_input,
                                                    workflow_attachment=self.wdl_attachments)
-            self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+            self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+
+
+class ArvadosTest(IntegrationTest):
+    """Test using arvados-cwl-runner."""
+
+    def setUp(self):
+        """
+        Start a (local) wes-service server to make requests against.
+        Use arvados-cwl-runner as the wes-service server 'backend'.
+        Requires ARVADOS_API_HOST and ARVADOS_API_TOKEN to be set in the environment.
+        """
+        if os.path.exists('workflows'):
+            shutil.rmtree('workflows')
+        self.wes_server_process = subprocess.Popen(
+            ['python', os.path.abspath('wes_service/wes_service_main.py'),
+             '--backend=wes_service.arvados_wes',
+             '--port=8080',
+             '--debug'])
+        self.client.auth = {"Authorization": "Bearer " + os.environ["ARVADOS_API_TOKEN"]}
+        time.sleep(5)
 
+    def check_for_file(self, filepath, seconds=120):
+        # Doesn't make sense for arvados
+        return True
 
 # Prevent pytest/unittest's discovery from attempting to discover the base test class.
 del IntegrationTest

testdata/md5sum.json

@@ -1,2 +1 @@
-{"output_file": {"path": "/tmp/md5sum.txt", "class": "File"},
- "input_file": {"path": "md5sum.input", "class": "File"}}
+{"input_file": {"path": "md5sum.input", "class": "File"}}