🚂Append Ensembl&Eutils API to retrieve sequences

Author: NatureGeorge

README.md

@@ -12,6 +12,8 @@
 
 Profiling Protein Structures from Protein Data Bank and integrate various resources.
 
+[![HitCount](http://hits.dwyl.com/naturegeorge/pdb-profiling.svg)](http://hits.dwyl.com/naturegeorge/pdb-profiling)
+
 ## Goal
 
 * Gather helpful/insightful indexes to evaluate a PDB structure's usefulness in:
@@ -45,6 +47,12 @@ Profiling Protein Structures from Protein Data Bank and integrate various resour
   * Interactome3D API
     * <https://interactome3d.irbbarcelona.org/>
   * ModBase API (?)
+  * Ensembl REST API
+    * <https://rest.ensembl.org/documentation>
+    * NOTE: currently only support <https://rest.ensembl.org/documentation/info/sequence_id> to retrieve ensembl sequences
+  * Eutils API
+    * <https://eutils.ncbi.nlm.nih.gov/entrez/eutils/>
+    * NOTE: currently only support minimum use
 * Download data from PDB Archive against unexpected needs
   * wwwPDB&RCSB: <https://ftp.wwpdb.org/pub/pdb/data/structures/>
   * EBI: <http://ftp.ebi.ac.uk/pub/databases/pdb/data/structures/>

pdb_profiling/__init__.py

@@ -36,21 +36,13 @@ def default_config():
     from pdb_profiling.processors.uniprot.api import UniProtFASTA
     # Use Existing Handled PDBe API Results (e.g. tsv format results)
     ProcessPDBe.use_existing = True
-    # Init Abclog Logger
-    Abclog.init_logger(logName='PDB-Profiling')
-    # Init ProcessPDBe's Logger (pass it with Abclog Logger)
-    ProcessPDBe.init_logger(logger=Abclog.logger)
     # Use Existing API Results (e.g. json format results downloaded from web)
     UnsyncFetch.use_existing = True
+    # Init Abclog Logger
+    Abclog.init_logger(logName='PDB-Profiling')
     # Init WebFetcher's Logger (pass it with Abclog Logger)
-    UnsyncFetch.init_setting(ProcessPDBe.logger)
+    UnsyncFetch.init_setting(Abclog.logger)
     # Set WebFetcher's Semaphore
     Base.set_web_semaphore(30)
     # Set Folder that store downloaded and handled files
     Base.set_folder('./')
-    # Init ModelServer API's Logger (pass it with Abclog Logger)
-    PDBeModelServer.init_logger(logger=ProcessPDBe.logger)
-    # Init PDBArchive API's Logger (pass it with Abclog Logger)
-    PDBArchive.init_logger(logger=ProcessPDBe.logger)
-    # Init UniProtFASTA API's Logger (pass it with Abclog Logger)
-    UniProtFASTA.init_logger(logger=ProcessPDBe.logger)

pdb_profiling/processors/ensembl/__init__.py

@@ -0,0 +1,65 @@
+# @Created Date: 2020-09-26 01:54:56 pm
+# @Filename: api.py
+# @Email:  1730416009@stu.suda.edu.cn
+# @Author: ZeFeng Zhu
+# @Last Modified: 2020-09-26 01:55:04 pm
+# @Copyright (c) 2020 MinghuiGroup, Soochow University
+from typing import Dict, Tuple, Iterable, Generator, Union
+from pathlib import Path
+from pdb_profiling.log import Abclog
+from pdb_profiling.fetcher.webfetch import UnsyncFetch
+
+
+BASE_URL = 'https://rest.ensembl.org/'
+
+
+class EnsemblAPI(Abclog):
+    '''
+    Implement The Ensembl REST API
+
+    * <https://rest.ensembl.org/documentation/>
+    * DEMO: <https://rest.ensembl.org/sequence/id/ENST00000288602?content-type=text/x-fasta;type=protein>
+    '''
+    headers = {"Content-Type": "text/x-fasta"}
+    api_set = frozenset(('sequence/id/',))
+
+    @classmethod
+    def get_file_suffix(cls) -> str:
+        res = cls.headers["Content-Type"].split('/')[1]
+        assert res in ('plain', 'x-seqxml+xml', 'x-fasta'), f"Unexcepted Case: {cls.headers}"
+        return res.replace('x-', '').replace('seqxml+', '')
+
+    @classmethod
+    def task_unit(cls, suffix: str, identifier: str, params: Dict, folder: Path) -> Tuple:
+        args = dict(
+            url=f'{BASE_URL}{suffix}{identifier}',
+            headers=cls.headers,
+            params=params)
+        return 'get', args, folder/f'{identifier}.{cls.get_file_suffix()}'
+
+    @classmethod
+    def yieldTasks(cls, suffix: str, identifiers: Iterable[str], params_collection: Iterable[Dict], folder: Path) -> Generator:
+        for identifier, params in zip(identifiers, params_collection):
+            yield cls.task_unit(suffix, identifier, params, folder)
+    
+    @classmethod
+    def retrieve(cls, suffix: str, identifiers: Iterable[str], params_collection: Iterable[Dict], folder: Union[Path, str], concur_req: int = 20, rate: float = 1.5, ret_res: bool = True, **kwargs):
+        assert suffix in cls.api_set, f"Invalid suffix! Valid set is \n{cls.api_set}"
+        folder = Path(folder)
+        res = UnsyncFetch.multi_tasks(
+            cls.yieldTasks(suffix, identifiers, params_collection, folder),
+            concur_req=concur_req,
+            rate=rate,
+            logger=cls.logger,
+            ret_res=ret_res,
+            semaphore=kwargs.get('semaphore', None))
+        return res
+
+    @classmethod
+    def single_retrieve(cls, suffix: str, identifier: str, params: Dict, folder: Union[Path, str], semaphore, rate: float = 1.5):
+        assert suffix in cls.api_set, f"Invalid suffix! Valid set is \n{cls.api_set}"
+        folder = Path(folder)
+        return UnsyncFetch.single_task(
+            task=cls.task_unit(suffix, identifier, params, folder),
+            semaphore=semaphore,
+            rate=rate)

pdb_profiling/processors/ensembl/api.py

@@ -0,0 +1,64 @@
+# @Created Date: 2020-09-26 02:51:49 pm
+# @Filename: api.py
+# @Email:  1730416009@stu.suda.edu.cn
+# @Author: ZeFeng Zhu
+# @Last Modified: 2020-09-26 02:51:59 pm
+# @Copyright (c) 2020 MinghuiGroup, Soochow University
+from typing import Dict, Tuple, Iterable, Generator, Union
+from pathlib import Path
+from pdb_profiling.log import Abclog
+from pdb_profiling.fetcher.webfetch import UnsyncFetch
+
+BASE_URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
+
+
+class EutilsAPI(Abclog):
+    '''
+    Implement The Entrez Programming Utilities (E-utilities) API
+
+    * Entrez Programming Utilities Help: <https://www.ncbi.nlm.nih.gov/books/NBK25501/>
+    * DEMO URL 1: <https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=protein&id=NP_001291289&rettype=fasta>
+    * DEMO URL 2: <https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=sequences&id=NM_001304360&rettype=fasta>
+    '''
+    headers = {"Content-Type": "text/plain"}
+    api_set = frozenset(('efetch.fcgi', 'einfo.fcgi', 'esearch.fcgi',
+                         'epost.fcgi', 'esummary.fcgi'))
+
+    @classmethod
+    def dumpsParams(cls, params: Dict) -> str:
+        return '&'.join(f'{key}={value}' for key, value in params.items())
+
+    @classmethod
+    def task_unit(cls, suffix: str, params: Dict, folder: Path) -> Tuple:
+        args = dict(
+            url=f'{BASE_URL}{suffix}',
+            headers=cls.headers,
+            params=params)
+        return 'get', args, folder/f'{cls.dumpsParams(params)}.{params.get("retmode", params.get("rettype", "txt"))}'
+
+    @classmethod
+    def yieldTasks(cls, suffix: str, params_collection: Iterable[Dict], folder: Path) -> Generator:
+        for params in params_collection:
+            yield cls.task_unit(suffix, params, folder)
+
+    @classmethod
+    def retrieve(cls, suffix: str, params_collection: Iterable[Dict], folder: Union[Path, str], concur_req: int = 20, rate: float = 1.5, ret_res: bool = True, **kwargs):
+        assert suffix in cls.api_set, f"Invalid suffix! Valid set is \n{cls.api_set}"
+        folder = Path(folder)
+        res = UnsyncFetch.multi_tasks(
+            cls.yieldTasks(suffix, params_collection, folder),
+            concur_req=concur_req,
+            rate=rate,
+            logger=cls.logger,
+            ret_res=ret_res,
+            semaphore=kwargs.get('semaphore', None))
+        return res
+
+    @classmethod
+    def single_retrieve(cls, suffix: str, params: Dict, folder: Union[Path, str], semaphore, rate: float = 1.5):
+        assert suffix in cls.api_set, f"Invalid suffix! Valid set is \n{cls.api_set}"
+        folder = Path(folder)
+        return UnsyncFetch.single_task(
+            task=cls.task_unit(suffix, params, folder),
+            semaphore=semaphore,
+            rate=rate)

pdb_profiling/processors/eutils/__init__.py

@@ -36,7 +36,7 @@
 
 PDB_ARCHIVE_URL_EBI: str = 'http://ftp.ebi.ac.uk/pub/databases/pdb/data/structures/'
 PDB_ARCHIVE_URL_WWPDB: str = 'https://ftp.wwpdb.org/pub/pdb/data/structures/'
-PDB_ARCHIVE_VERSIONED_URL: str = 'http://ftp-versioned.wwpdb.org/pdb_versioned/data/
+PDB_ARCHIVE_VERSIONED_URL: str = 'http://ftp-versioned.wwpdb.org/pdb_versioned/data/'
 
 # https://ftp.wwpdb.org/pub/pdb/data/structures/obsolete/mmCIF/a0/2a01.cif.gz
 # http://ftp.ebi.ac.uk/pub/databases/pdb/data/structures/obsolete/mmCIF/a0/2a01.cif.gz
@@ -707,7 +707,7 @@ class PDBeModelServer(Abclog):
 
     root = 'model-server/v1/'
     headers =  {'accept': 'text/plain', 'Content-Type': 'application/json'}
-    api_sets = frozenset(('atoms', 'residueInteraction', 'assembly', 'full', 'ligand'
+    api_set = frozenset(('atoms', 'residueInteraction', 'assembly', 'full', 'ligand'
                 'residueSurroundings', 'symmetryMates', 'query-many'))
 
     @classmethod
@@ -765,7 +765,7 @@ class PDBArchive(Abclog):
     * EBI: PDB_ARCHIVE_URL_EBI: str = 'http://ftp.ebi.ac.uk/pub/databases/pdb/data/structures/'
     '''
     root = PDB_ARCHIVE_URL_EBI
-    api_sets = frozenset(f'{i}/{j}/' for i in ('obsolete', 'divided')
+    api_set = frozenset(f'{i}/{j}/' for i in ('obsolete', 'divided')
                 for j in ('mmCIF', 'pdb', 'XML'))
 
     @classmethod
@@ -802,9 +802,14 @@ class PDBVersioned(PDBArchive):
     Download files from PDB Versioned
 
     * wwwPDB Versioned: PDB_ARCHIVE_VERSIONED_URL: str = 'http://ftp-versioned.wwpdb.org/pdb_versioned/data/entries/'
+
+    >>> PDBVersioned.single_retrieve(
+        ('2wmg', '_v1-2'), 'entries/', 
+        init_folder_from_suffix(Base.get_folder(), 'pdb-versioned/entries'), 
+        Base.get_web_semaphore()).result()
     '''
     root = PDB_ARCHIVE_VERSIONED_URL
-    api_sets = frozenset(('entries/', 'removed/'))
+    api_set = frozenset(('entries/', 'removed/'))
 
     @classmethod
     def task_unit(cls, pdb_with_version: Tuple, suffix: str, file_suffix: str, folder: Path):

pdb_profiling/processors/eutils/api.py

@@ -200,7 +200,7 @@ def get_id(self):
         return self.pdb_id
 
     def fetch_from_modelServer_api(self, api_suffix: str, method: str = 'post', data_collection=None, params=None, then_func: Optional[Callable[[Unfuture], Unfuture]] = None) -> Unfuture:
-        assert api_suffix in PDBeModelServer.api_sets, f"Invlaid API SUFFIX! Valid set:\n{PDBeModelServer.api_sets}"
+        assert api_suffix in PDBeModelServer.api_set, f"Invlaid API SUFFIX! Valid set:\n{PDBeModelServer.api_set}"
         task = self.tasks.get((PDBeModelServer.root, api_suffix, method, data_collection, params, then_func), None)
         if task is not None:
             return task
@@ -218,7 +218,7 @@ def fetch_from_modelServer_api(self, api_suffix: str, method: str = 'post', data
         return task
 
     def fetch_from_PDBArchive(self, api_suffix: str, then_func: Optional[Callable[[Unfuture], Unfuture]] = None, **kwargs) -> Unfuture:
-        assert api_suffix in PDBArchive.api_sets, f"Invlaid API SUFFIX! Valid set:\n{PDBArchive.api_sets}"
+        assert api_suffix in PDBArchive.api_set, f"Invlaid API SUFFIX! Valid set:\n{PDBArchive.api_set}"
         task = self.tasks.get((PDBArchive.root, api_suffix, then_func), None)
         if task is not None:
             return task
@@ -880,6 +880,7 @@ def __init__(self, identifier:str):
         self.set_id(identifier)
         self.tasks = dict()
 
+
 '''
 TODO: Deal with carbohydrate polymer in PISA
 

pdb_profiling/processors/pdbe/api.py

@@ -21,11 +21,12 @@ class ProteinsAPI(Abclog):
     '''
 
     headers = {'Accept': 'application/json'}
-    api_sets = frozenset((
-                'coordinates', 'coordinates/', 'coordinates/location/',
-                'uniparc', 'uniparc/accession/', 'uniparc/best/guess',
-                'uniparc/dbreference/', 'uniparc/proteome/', 'uniparc/sequence',  # NOTE: uniparc/sequence use POST method!
-                'uniparc/upi/'))
+    api_set = frozenset((
+        'proteins', 'proteins/covid-19/entries', 'proteins/interaction/', 'proteins/',
+        'coordinates', 'coordinates/', 'coordinates/location/',
+        'uniparc', 'uniparc/accession/', 'uniparc/best/guess',
+        'uniparc/dbreference/', 'uniparc/proteome/', 'uniparc/sequence',  # NOTE: uniparc/sequence use POST method!
+        'uniparc/upi/'))
 
     @classmethod
     def get_file_suffix(cls) -> str:
@@ -43,7 +44,7 @@ def task_unit(cls, suffix: str, params: Dict, folder: Path, identifier:Optional[
             url=f'{BASE_URL}{suffix}' if identifier is None else f'{BASE_URL}{suffix}{identifier}',
             headers=cls.headers,
             params=params)
-        return 'get', args, folder/f'{identifier if identifier is not None else cls.dumpsParams(params)}.{cls.get_file_suffix()}'
+        return 'get', args, folder/f'{identifier.replace(":", "_")+"_"+cls.dumpsParams(params) if identifier is not None else cls.dumpsParams(params)}.{cls.get_file_suffix()}'
 
     @classmethod
     def yieldTasks(cls, suffix: str, params_collection: Iterable[Dict], folder: Path, identifiers: Optional[Iterable[str]]) -> Generator:
@@ -57,7 +58,7 @@ def yieldTasks(cls, suffix: str, params_collection: Iterable[Dict], folder: Path
 
     @classmethod
     def retrieve(cls, suffix: str, params_collection: Iterable[Dict], folder: Union[Path, str], identifiers: Optional[Iterable[str]] = None, concur_req: int = 20, rate: float = 1.5, ret_res: bool = True, **kwargs):
-        assert suffix in cls.api_sets, f"Invalid suffix! Valid set is \n{cls.api_sets}"
+        assert suffix in cls.api_set, f"Invalid suffix! Valid set is \n{cls.api_set}"
         folder = Path(folder)
         res = UnsyncFetch.multi_tasks(
             cls.yieldTasks(suffix, params_collection, folder, identifiers),
@@ -70,7 +71,7 @@ def retrieve(cls, suffix: str, params_collection: Iterable[Dict], folder: Union[
 
     @classmethod
     def single_retrieve(cls, suffix: str, params:Dict, folder: Union[Path, str], semaphore, identifier:Optional[str]=None, rate: float = 1.5):
-        assert suffix in cls.api_sets, f"Invalid suffix! Valid set is \n{cls.api_sets}"
+        assert suffix in cls.api_set, f"Invalid suffix! Valid set is \n{cls.api_set}"
         folder = Path(folder)
         return UnsyncFetch.single_task(
             task=cls.task_unit(suffix, params, folder, identifier),

pdb_profiling/processors/pdbe/record.py

@@ -405,12 +405,16 @@ async def process(cls, path: Union[str, Path, Unfuture]):
             await fob
         return path
 
+    @classmethod
+    def task_unit(cls, unp:str, folder: Union[str, Path]):
+        cur_fileName = f'{unp}.fasta'
+        cur_filePath = str(Path(folder, cur_fileName))
+        return ('get', {'url': f'{BASE_URL}/uniprot/{cur_fileName}', 'params': cls.params}, cur_filePath)
+
     @classmethod
     def yieldTasks(cls, lyst: Iterable, folder: Union[str, Path]) -> Generator:
         for unp in lyst:
-            cur_fileName = f'{unp}.fasta'
-            cur_filePath = str(Path(folder, cur_fileName))
-            yield ('get', {'url': f'{BASE_URL}/uniprot/{cur_fileName}', 'params': cls.params}, cur_filePath)
+            return cls.task_unit(unp, folder)
 
     @classmethod
     def retrieve(cls, lyst: Iterable, folder: Union[str, Path], concur_req: int = 20, rate: float = 1.5, ret_res: bool = True, semaphore=None):
@@ -421,3 +425,10 @@ def retrieve(cls, lyst: Iterable, folder: Union[str, Path], concur_req: int = 20
             logger=cls.logger,
             ret_res=ret_res,
             semaphore=semaphore)
+    
+    @classmethod
+    def single_retrieve(cls, identifier: str, folder: Union[str, Path], semaphore, rate: float = 1.5):
+        return UnsyncFetch.single_task(
+            task=cls.task_unit(identifier, folder),
+            semaphore=semaphore,
+            rate=rate)