From 4433f45b428fda481ef4b793592e06666c147803 Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 16:28:00 -0700 Subject: [PATCH 1/8] add async loading option for reports --- pori_python/ipr/connection.py | 45 ++++++++++++++++++++++++++++++++--- pori_python/ipr/main.py | 6 ++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index c86a373..2691ef0 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -4,8 +4,9 @@ import os import zlib from typing import Dict, List - +import time from .constants import DEFAULT_URL +from .util import logger IMAGE_MAX = 20 # cannot upload more than 20 images at a time @@ -62,8 +63,46 @@ def post(self, uri: str, data: Dict = {}, **kwargs) -> Dict: **kwargs, ) - def upload_report(self, content: Dict) -> Dict: - return self.post('reports', content) + def get(self, uri: str, data: Dict = {}, **kwargs) -> Dict: + """Convenience method for making get requests""" + return self.request( + uri, + method='GET', + data=zlib.compress(json.dumps(data, allow_nan=False).encode('utf-8')), + **kwargs, + ) + + def upload_report(self, content: Dict, mins_to_wait: int = 5, async_upload: bool = False) -> Dict: + if async_upload: + initial_result = self.post('reports-async', content) + report_id = initial_result["ident"] + for i in range(5): + logger.info('checking report loading status in 5 seconds') + time.sleep(5) + current_status = self.get(f'reports-async/{report_id}') + if current_status['state'] not in ['active', 'ready']: + raise Exception(f'async report upload in unexpected state: {current_status}') + if current_status['state'] == 'ready': + return current_status + for i in range(5): + logger.info('checking report loading status in 30 seconds') + time.sleep(30) + current_status = self.get(f'reports-async/{report_id}') + if current_status['state'] not in ['active', 'ready']: + raise Exception(f'async report upload in unexpected state: {current_status}') + if current_status['state'] == 'ready': + return current_status + for i in range(mins_to_wait): + logger.info('checking report loading status in 1 minute') + time.sleep(60) + current_status = self.get(f'reports-async/{report_id}') + if current_status['state'] not in ['active', 'ready']: + raise Exception(f'async report upload in unexpected state: {current_status}') + if current_status['state'] == 'ready': + return current_status + raise Exception(f'async report upload taking longer than expected: {current_status}') + else: + return self.post('reports', content) def set_analyst_comments(self, report_id: str, data: Dict) -> Dict: """ diff --git a/pori_python/ipr/main.py b/pori_python/ipr/main.py index ea53473..2c6eefd 100644 --- a/pori_python/ipr/main.py +++ b/pori_python/ipr/main.py @@ -216,6 +216,8 @@ def ipr_report( generate_comments: bool = True, match_germline: bool = False, custom_kb_match_filter=None, + async_upload: bool = False, + mins_to_wait: int = 5, ) -> Dict: """Run the matching and create the report JSON for upload to IPR. @@ -234,6 +236,8 @@ def ipr_report( generate_comments: create the analyst comments section for upload with the report match_germline: match only germline statements to germline events and non-germline statements to non-germline events. custom_kb_match_filter: function(List[kbMatch]) -> List[kbMatch] + async_upload: use report_async endpoint to upload reports + mins_to_wait: if using report_async, number of minutes to wait for success before exception raised Returns: ipr_conn.upload_report return dictionary @@ -444,7 +448,7 @@ def ipr_report( if ipr_upload: try: logger.info(f'Uploading to IPR {ipr_conn.url}') - ipr_result = ipr_conn.upload_report(output) + ipr_result = ipr_conn.upload_report(output, async_upload, mins_to_wait) logger.info(ipr_result) output.update(ipr_result) except Exception as err: From 11edcb604aa58740cf68633037038ba0c8b2676f Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 16:38:18 -0700 Subject: [PATCH 2/8] cleanup --- pori_python/ipr/connection.py | 43 +++++++++++++++-------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index 2691ef0..29c9a46 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -76,31 +76,24 @@ def upload_report(self, content: Dict, mins_to_wait: int = 5, async_upload: bool if async_upload: initial_result = self.post('reports-async', content) report_id = initial_result["ident"] - for i in range(5): - logger.info('checking report loading status in 5 seconds') - time.sleep(5) - current_status = self.get(f'reports-async/{report_id}') - if current_status['state'] not in ['active', 'ready']: - raise Exception(f'async report upload in unexpected state: {current_status}') - if current_status['state'] == 'ready': - return current_status - for i in range(5): - logger.info('checking report loading status in 30 seconds') - time.sleep(30) - current_status = self.get(f'reports-async/{report_id}') - if current_status['state'] not in ['active', 'ready']: - raise Exception(f'async report upload in unexpected state: {current_status}') - if current_status['state'] == 'ready': - return current_status - for i in range(mins_to_wait): - logger.info('checking report loading status in 1 minute') - time.sleep(60) - current_status = self.get(f'reports-async/{report_id}') - if current_status['state'] not in ['active', 'ready']: - raise Exception(f'async report upload in unexpected state: {current_status}') - if current_status['state'] == 'ready': - return current_status - raise Exception(f'async report upload taking longer than expected: {current_status}') + + def check_status(interval: int = 5, num_attempts: int = 5): + for i in range (num_attempts): + logger.info(f'checking report loading status in {interval} seconds') + time.sleep(interval) + current_status = self.get(f'reports-async/{report_id}') + if current_status['state'] not in ['active', 'ready']: + raise Exception(f'async report upload in unexpected state: {current_status}') + if current_status['state'] == 'ready': + return current_status + current_status = check_status() + if current_status['state'] == 'active': + current_status = check_status(interval=30) + if current_status['state'] == 'active': + current_status = check_status(interval=60, num_attempts=mins_to_wait) + if current_status['state'] == 'active': + raise Exception(f'async report upload taking longer than expected: {current_status}') + return current_status else: return self.post('reports', content) From 3a3e594953fd07941c4a8a15abf1f2ca59ef37fe Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 16:47:03 -0700 Subject: [PATCH 3/8] lint with black --- pori_python/ipr/connection.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index 29c9a46..be83488 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -72,27 +72,34 @@ def get(self, uri: str, data: Dict = {}, **kwargs) -> Dict: **kwargs, ) - def upload_report(self, content: Dict, mins_to_wait: int = 5, async_upload: bool = False) -> Dict: + def upload_report( + self, content: Dict, mins_to_wait: int = 5, async_upload: bool = False + ) -> Dict: if async_upload: initial_result = self.post('reports-async', content) report_id = initial_result["ident"] def check_status(interval: int = 5, num_attempts: int = 5): - for i in range (num_attempts): + for i in range(num_attempts): logger.info(f'checking report loading status in {interval} seconds') time.sleep(interval) current_status = self.get(f'reports-async/{report_id}') if current_status['state'] not in ['active', 'ready']: - raise Exception(f'async report upload in unexpected state: {current_status}') + raise Exception( + f'async report upload in unexpected state: {current_status}' + ) if current_status['state'] == 'ready': return current_status + current_status = check_status() if current_status['state'] == 'active': current_status = check_status(interval=30) if current_status['state'] == 'active': current_status = check_status(interval=60, num_attempts=mins_to_wait) if current_status['state'] == 'active': - raise Exception(f'async report upload taking longer than expected: {current_status}') + raise Exception( + f'async report upload taking longer than expected: {current_status}' + ) return current_status else: return self.post('reports', content) From ef0ca7be5fddf02ab1002f5ddb9d932b7847a63c Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 17:01:24 -0700 Subject: [PATCH 4/8] handle other possible states, add return --- pori_python/ipr/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index be83488..4c9c597 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -84,12 +84,13 @@ def check_status(interval: int = 5, num_attempts: int = 5): logger.info(f'checking report loading status in {interval} seconds') time.sleep(interval) current_status = self.get(f'reports-async/{report_id}') - if current_status['state'] not in ['active', 'ready']: + if current_status['state'] not in ['active', 'ready', 'waiting', 'completed']: raise Exception( f'async report upload in unexpected state: {current_status}' ) - if current_status['state'] == 'ready': + if current_status['state'] in ['ready', 'completed']: return current_status + return current_status current_status = check_status() if current_status['state'] == 'active': From fefd2535fd66df67c6120ed61b7f4b67297e9241 Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 17:02:33 -0700 Subject: [PATCH 5/8] handle waiting --- pori_python/ipr/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index 4c9c597..3d83554 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -93,11 +93,11 @@ def check_status(interval: int = 5, num_attempts: int = 5): return current_status current_status = check_status() - if current_status['state'] == 'active': + if current_status['state'] in ['active', 'waiting']: current_status = check_status(interval=30) - if current_status['state'] == 'active': + if current_status['state'] == ['active', 'waiting']: current_status = check_status(interval=60, num_attempts=mins_to_wait) - if current_status['state'] == 'active': + if current_status['state'] == ['active', 'waiting']: raise Exception( f'async report upload taking longer than expected: {current_status}' ) From 75d23b8d82a40424b78b121ba2b41fedca9ecf3a Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 17:42:22 -0700 Subject: [PATCH 6/8] reformat, fix bug --- pori_python/ipr/connection.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index 3d83554..a7a372f 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -93,14 +93,17 @@ def check_status(interval: int = 5, num_attempts: int = 5): return current_status current_status = check_status() + if current_status['state'] in ['active', 'waiting']: current_status = check_status(interval=30) - if current_status['state'] == ['active', 'waiting']: - current_status = check_status(interval=60, num_attempts=mins_to_wait) - if current_status['state'] == ['active', 'waiting']: - raise Exception( - f'async report upload taking longer than expected: {current_status}' - ) + + if current_status['state'] in ['active', 'waiting']: + current_status = check_status(interval=60, num_attempts=mins_to_wait) + + if current_status['state'] in ['active', 'waiting']: + raise Exception( + f'async report upload taking longer than expected: {current_status}' + ) return current_status else: return self.post('reports', content) From 2ab0cb42a10394bd9947684260eddf49d04b861f Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Mon, 8 Jul 2024 17:46:30 -0700 Subject: [PATCH 7/8] log failedReason --- pori_python/ipr/connection.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index a7a372f..9e8c468 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -84,10 +84,14 @@ def check_status(interval: int = 5, num_attempts: int = 5): logger.info(f'checking report loading status in {interval} seconds') time.sleep(interval) current_status = self.get(f'reports-async/{report_id}') - if current_status['state'] not in ['active', 'ready', 'waiting', 'completed']: + if current_status['state'] not in ['active', 'ready', 'waiting', 'completed', 'failed']: raise Exception( f'async report upload in unexpected state: {current_status}' ) + if current_status['state'] == 'failed': + raise Exception( + f'report upload failed with reason: {current_status["failedReason"]}' + ) if current_status['state'] in ['ready', 'completed']: return current_status return current_status @@ -104,6 +108,7 @@ def check_status(interval: int = 5, num_attempts: int = 5): raise Exception( f'async report upload taking longer than expected: {current_status}' ) + return current_status else: return self.post('reports', content) From 0e0aa5cc2f00be5d3011e3391999d4f2dc460ec9 Mon Sep 17 00:00:00 2001 From: Eleanor Lewis Date: Tue, 9 Jul 2024 10:18:41 -0700 Subject: [PATCH 8/8] lint with black --- pori_python/ipr/connection.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pori_python/ipr/connection.py b/pori_python/ipr/connection.py index 9e8c468..b267f06 100644 --- a/pori_python/ipr/connection.py +++ b/pori_python/ipr/connection.py @@ -84,7 +84,13 @@ def check_status(interval: int = 5, num_attempts: int = 5): logger.info(f'checking report loading status in {interval} seconds') time.sleep(interval) current_status = self.get(f'reports-async/{report_id}') - if current_status['state'] not in ['active', 'ready', 'waiting', 'completed', 'failed']: + if current_status['state'] not in [ + 'active', + 'ready', + 'waiting', + 'completed', + 'failed', + ]: raise Exception( f'async report upload in unexpected state: {current_status}' )