diff --git a/bluepyemodel/access_point/__init__.py b/bluepyemodel/access_point/__init__.py index 4354dbc..0c4513b 100644 --- a/bluepyemodel/access_point/__init__.py +++ b/bluepyemodel/access_point/__init__.py @@ -69,29 +69,6 @@ def get_access_point(access_point, emodel, **kwargs): brain_region = kwargs.get("brain_region", None) brain_region = brain_region.replace("__", " ") if brain_region else None - if access_point == "nexus": - from bluepyemodel.access_point.nexus import NexusAccessPoint - - if not kwargs.get("project"): - raise ValueError("Nexus project name is required for Nexus access point.") - - return NexusAccessPoint( - emodel=emodel, - etype=etype, - ttype=ttype, - mtype=mtype, - species=kwargs.get("species", None), - brain_region=brain_region, - iteration_tag=kwargs.get("iteration_tag", None), - synapse_class=kwargs.get("synapse_class", None), - project=kwargs.get("project", None), - organisation=kwargs.get("organisation", "bbp"), - endpoint=kwargs.get("endpoint", "https://staging.openbluebrain.com/api/nexus/v1"), - forge_path=kwargs.get("forge_path", None), - forge_ontology_path=kwargs.get("forge_ontology_path", None), - access_token=kwargs.get("access_token", None), - ) - if access_point == "local": from bluepyemodel.access_point.local import LocalAccessPoint @@ -111,4 +88,4 @@ def get_access_point(access_point, emodel, **kwargs): with_seeds=kwargs.get("with_seeds", False), ) - raise ValueError(f"Unknown access point: {access_point}. Should be 'nexus' or 'local'.") + raise ValueError(f"Unknown access point: {access_point}. Should be 'local'.") diff --git a/bluepyemodel/access_point/forge_access_point.py b/bluepyemodel/access_point/forge_access_point.py deleted file mode 100644 index 2fc3e05..0000000 --- a/bluepyemodel/access_point/forge_access_point.py +++ /dev/null @@ -1,1244 +0,0 @@ -"""NexusForgeAccessPoint class.""" - -""" -Copyright 2024 Blue Brain Project / EPFL - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import getpass -import json -import logging -import pathlib -from datetime import datetime -from datetime import timedelta -from datetime import timezone - -import jwt -from entity_management.state import refresh_token -from kgforge.core import KnowledgeGraphForge -from kgforge.core import Resource -from kgforge.core.commons.strategies import ResolvingStrategy -from kgforge.specializations.resources import Dataset - -from bluepyemodel.efeatures_extraction.targets_configuration import TargetsConfiguration -from bluepyemodel.emodel_pipeline.emodel import EModel -from bluepyemodel.emodel_pipeline.emodel_script import EModelScript -from bluepyemodel.emodel_pipeline.emodel_settings import EModelPipelineSettings -from bluepyemodel.emodel_pipeline.emodel_workflow import EModelWorkflow -from bluepyemodel.emodel_pipeline.memodel import MEModel -from bluepyemodel.evaluation.fitness_calculator_configuration import FitnessCalculatorConfiguration -from bluepyemodel.model.distribution_configuration import DistributionConfiguration -from bluepyemodel.model.neuron_model_configuration import NeuronModelConfiguration -from bluepyemodel.tools.utils import yesno - -logger = logging.getLogger("__main__") - - -# pylint: disable=bare-except,consider-iterating-dictionary - -DEFAULT_NEXUS_ENDPOINT = "https://openbluebrain.com/api/nexus/v1" - -CLASS_TO_NEXUS_TYPE = { - "TargetsConfiguration": "ExtractionTargetsConfiguration", - "EModelPipelineSettings": "EModelPipelineSettings", - "FitnessCalculatorConfiguration": "FitnessCalculatorConfiguration", - "NeuronModelConfiguration": "EModelConfiguration", - "EModel": "EModel", - "DistributionConfiguration": "EModelChannelDistribution", - "EModelWorkflow": "EModelWorkflow", - "EModelScript": "EModelScript", - "MEModel": "MEModel", -} - -CLASS_TO_RESOURCE_NAME = { - "TargetsConfiguration": "ETC", - "EModelPipelineSettings": "EMPS", - "FitnessCalculatorConfiguration": "FCC", - "NeuronModelConfiguration": "EMC", - "EModel": "EM", - "DistributionConfiguration": "EMCD", - "EModelWorkflow": "EMW", - "EModelScript": "EMS", - "MEModel": "MEM", -} - -NEXUS_TYPE_TO_CLASS = { - "ExtractionTargetsConfiguration": TargetsConfiguration, - "EModelPipelineSettings": EModelPipelineSettings, - "FitnessCalculatorConfiguration": FitnessCalculatorConfiguration, - "EModelConfiguration": NeuronModelConfiguration, - "EModel": EModel, - "EModelChannelDistribution": DistributionConfiguration, - "EModelWorkflow": EModelWorkflow, - "EModelScript": EModelScript, - "MEModel": MEModel, -} - -NEXUS_ENTRIES = [ - "objectOfStudy", - "contribution", - "type", - "id", - "distribution", - "@type", - "annotation", - "name", -] - -NEXUS_PROJECTS_TRACES = [ - {"project": "lnmce", "organisation": "bbp"}, - {"project": "thalamus", "organisation": "public"}, - {"project": "mmb-point-neuron-framework-model", "organisation": "bbp"}, -] - - -class AccessPointException(Exception): - """For Exceptions related to the NexusForgeAccessPoint""" - - -class NexusForgeAccessPoint: - """Access point to Nexus Knowledge Graph using Nexus Forge""" - - forges = {} - - def __init__( - self, - project="emodel_pipeline", - organisation="demo", - endpoint=DEFAULT_NEXUS_ENDPOINT, - forge_path=None, - limit=5000, - debug=False, - cross_bucket=True, - access_token=None, - search_endpoint="sparql", - ): - self.limit = limit - self.debug = debug - self.cross_bucket = cross_bucket - self.search_endpoint = search_endpoint - - self.endpoint = endpoint - self.bucket = organisation + "/" + project - self.forge_path = forge_path - - # reuse token to avoid redundant user prompts - self.access_token = access_token - - if not self.access_token: - self.access_token = self.get_access_token() - decoded_token = jwt.decode(self.access_token, options={"verify_signature": False}) - self.agent = self.forge.reshape( - self.forge.from_json(decoded_token), - keep=["name", "email", "sub", "preferred_username"], - ) - username = decoded_token["preferred_username"] - self.agent.id = f"https://bbp.epfl.ch/nexus/v1/realms/bbp/users/{username}" - self.agent.type = ["Person", "Agent"] - - self._available_etypes = None - self._available_mtypes = None - self._available_ttypes = None - self._atlas_release = None - - def refresh_token(self, offset=30): - """refresh token if token is expired or will be soon. Returns new expiring time. - - Args: - offset (int): offset to apply to the expiring time in s. - """ - # Check if the access token has expired - decoded_token = jwt.decode(self.access_token, options={"verify_signature": False}) - token_exp_timestamp = decoded_token["exp"] - # Get the current UTC time as a timezone-aware datetime object - utc_now = datetime.now(timezone.utc) - current_timestamp = int(utc_now.timestamp()) - if current_timestamp > token_exp_timestamp - offset: - logger.info("Nexus access token has expired, refreshing token...") - self.access_token = self.get_access_token() - decoded_token = jwt.decode(self.access_token, options={"verify_signature": False}) - token_exp_timestamp = decoded_token["exp"] - - return token_exp_timestamp - - @property - def forge(self): - key = f"{self.endpoint}|{self.bucket}|{self.forge_path}" - if key in self.__class__.forges: - expiry, forge = self.__class__.forges[key] - if expiry > datetime.now(timezone.utc): - return forge - - token_exp_timestamp = self.refresh_token() - forge = KnowledgeGraphForge( - self.forge_path, - bucket=self.bucket, - endpoint=self.endpoint, - token=self.access_token, - ) - - self.__class__.forges[key] = ( - datetime.fromtimestamp(token_exp_timestamp, timezone.utc) - timedelta(minutes=15), - forge, - ) - return forge - - @property - def available_etypes(self): - """List of ids of available etypes in this forge graph""" - if self._available_etypes is None: - self._available_etypes = self.get_available_etypes() - return self._available_etypes - - @property - def available_mtypes(self): - """List of ids of available mtypes in this forge graph""" - if self._available_mtypes is None: - self._available_mtypes = self.get_available_mtypes() - return self._available_mtypes - - @property - def available_ttypes(self): - """List of ids of available ttypes in this forge graph""" - if self._available_ttypes is None: - self._available_ttypes = self.get_available_ttypes() - return self._available_ttypes - - @property - def atlas_release(self): - """Hard-coded atlas release fields for metadata""" - # pylint: disable=protected-access - atlas_def = { - "id": "https://bbp.epfl.ch/neurosciencegraph/data/4906ab85-694f-469d-962f-c0174e901885", - "type": ["BrainAtlasRelease", "AtlasRelease"], - } - - if self._atlas_release is None: - self.refresh_token() - atlas_access_point = atlas_forge_access_point( - access_token=self.access_token, forge_path=self.forge_path, endpoint=self.endpoint - ) - atlas_resource = atlas_access_point.retrieve(atlas_def["id"]) - atlas_def["_rev"] = atlas_resource._store_metadata["_rev"] - self._atlas_release = atlas_def - return self._atlas_release - - def get_available_etypes(self): - """Returns a list of nexus ids of all the etype resources using sparql""" - query = """ - SELECT ?e_type_id - - WHERE {{ - ?e_type_id label ?e_type ; - subClassOf* EType ; - }} - """ - # should we use self.limit here? - resources = self.forge.sparql(query, limit=self.limit) - if resources is None: - return [] - return [r.e_type_id for r in resources] - - def get_available_mtypes(self): - """Returns a list of nexus ids of all the mtype resources using sparql""" - query = """ - SELECT ?m_type_id - - WHERE {{ - ?m_type_id label ?m_type ; - subClassOf* MType ; - }} - """ - # should we use self.limit here? - resources = self.forge.sparql(query, limit=self.limit) - if resources is None: - return [] - return [r.m_type_id for r in resources] - - def get_available_ttypes(self): - """Returns a list of nexus ids of all the ttype resources using sparql""" - query = """ - SELECT ?t_type_id - - WHERE {{ - ?t_type_id label ?t_type ; - subClassOf* BrainCellTranscriptomeType ; - }} - """ - # should we use self.limit here? - resources = self.forge.sparql(query, limit=self.limit) - if resources is None: - return [] - return [r.t_type_id for r in resources] - - @staticmethod - def get_access_token(): - """Define access token either from bbp-workflow or provided by the user""" - - try: - access_token = refresh_token() - except: # noqa: E722 - logger.info("Please get your Nexus access token from https://bbp.epfl.ch/nexus/web/.") - access_token = getpass.getpass() - if access_token is None: - logger.info("Please get your Nexus access token from https://bbp.epfl.ch/nexus/web/.") - access_token = getpass.getpass() - - return access_token - - @staticmethod - def connect_forge(bucket, endpoint, access_token, forge_path=None): - """Creation of a forge session""" - - if not forge_path: - forge_path = ( - "https://raw.githubusercontent.com/BlueBrain/nexus-forge/" - + "master/examples/notebooks/use-cases/prod-forge-nexus.yml" - ) - - forge = KnowledgeGraphForge( - forge_path, bucket=bucket, endpoint=endpoint, token=access_token - ) - - return forge - - def add_contribution(self, resource): - """Add the contributing agent to the resource""" - - if self.agent: - if isinstance(resource, Dataset): - resource.add_contribution(self.agent, versioned=False) - elif isinstance(resource, Resource): - resource.contribution = Resource(type="Contribution", agent=self.agent) - - return resource - - def resolve(self, text, scope="ontology", strategy="all", limit=1): - """Resolves a string to find the matching ontology""" - - if strategy == "all": - resolving_strategy = ResolvingStrategy.ALL_MATCHES - elif strategy == "best": - resolving_strategy = ResolvingStrategy.BEST_MATCH - elif strategy == "exact": - resolving_strategy = ResolvingStrategy.EXACT_MATCH - else: - raise ValueError( - f"Resolving strategy {strategy} does not exist. " - "Strategy should be 'all', 'best' or 'exact'" - ) - - return self.forge.resolve(text, scope=scope, strategy=resolving_strategy, limit=limit) - - def add_images_to_resource(self, images, resource, filters_existence=None): - """Attach images to a resource. - - Args: - images (list of str): list of local paths to images - resource (kgforge.core.Resource): resource to attach the images to - filters_existence (dict): contains resource type, name and metadata, - can be used to search for existence of resource on nexus. - Used to get image type if cannot be extracted from image path. - """ - if filters_existence is None: - filters_existence = {} - for path in images: - try: - resource_type = path.split("__")[-1].split(".")[0] - except IndexError: - resource_type = filters_existence.get("type", None) - # Do NOT do this BEFORE turning resource into a Dataset. - # That would break the storing LazyAction into a string - resource.add_image( - path=path, - content_type=f"application/{path.split('.')[-1]}", - about=resource_type, - ) - return resource - - def register( - self, - resource_description, - filters_existence=None, - legacy_filters_existence=None, - replace=False, - distributions=None, - images=None, - type_=None, - ): - """Register a resource from its dictionary description. - - Args: - resource_description (dict): contains resource type, name and metadata - filters_existence (dict): contains resource type, name and metadata, - can be used to search for existence of resource on nexus - legacy_filters_existence (dict): same as filters_existence, - but with legacy nexus metadata - replace (bool): whether to replace resource if found with filters_existence - distributions (list): paths to resource object as json and other distributions - images (list): paths to images to be attached to the resource - type_ (str): type of the resource. Will be used to get the schemas. - """ - # pylint: disable=protected-access - - if "type" not in resource_description: - raise AccessPointException("The resource description should contain 'type'.") - - previous_resources = None - if filters_existence: - previous_resources = self.fetch_legacy_compatible( - filters_existence, legacy_filters_existence - ) - - if previous_resources: - if replace: - for resource in previous_resources: - rr = self.retrieve(resource.id) - self.forge.deprecate(rr) - - else: - logger.warning( - "The resource you are trying to register already exist and will be ignored." - ) - return - - resource_description["objectOfStudy"] = { - "@id": "http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells", - "label": "Single Cell", - } - if resource_description.get("brainLocation", None) is not None: - resource_description["atlasRelease"] = self.atlas_release - - logger.debug("Registering resources: %s", resource_description) - - resource = self.forge.from_json(resource_description, na="None") - resource = self.add_contribution(resource) - - if distributions or images: - # do this before adding lazy actions. - # if we do it after, it turns the lazy actions into a str - resource = Dataset.from_resource(self.forge, resource) - - if distributions: - for path in distributions: - resource.add_distribution(path, content_type=f"application/{path.split('.')[-1]}") - - if images: - resource = self.add_images_to_resource(images, resource, filters_existence) - - # validate with Entity schema at creation. - # validation with EModelWorkflow schema is done at a later step, - # when EModelWorkflow resource is complete - if type_ == "EModelWorkflow": - type_ = "Entity" - schema_id = self.forge._model.schema_id(type_) - - self.forge.register(resource, schema_id=schema_id) - - def retrieve(self, id_): - """Retrieve a resource based on its id""" - - resource = self.forge.retrieve(id=id_, cross_bucket=self.cross_bucket) - - if resource: - return resource - - logger.debug("Could not retrieve resource of id: %s", id_) - - return None - - def fetch(self, filters, cross_bucket=None): - """Fetch resources based on filters. - - Args: - filters (dict): keys and values used for the "WHERE". Should include "type" or "id". - cross_bucket (bool): whether to also fetch from other projects or not. - - Returns: - resources (list): list of resources - """ - if "type" not in filters and "id" not in filters: - raise AccessPointException("Search filters should contain either 'type' or 'id'.") - - if cross_bucket is None: - cross_bucket = self.cross_bucket - - logger.debug("Searching: %s", filters) - - resources = self.forge.search( - filters, - cross_bucket=cross_bucket, - limit=self.limit, - debug=self.debug, - search_endpoint=self.search_endpoint, - ) - - if resources: - return resources - - logger.debug("No resources for filters: %s", filters) - - return None - - def fetch_legacy_compatible(self, filters, legacy_filters=None): - """Fetch resources based on filters. Use legacy filters if no resources are found. - - Args: - filters (dict): keys and values used for the "WHERE". Should include "type" or "id". - legacy_filters (dict): same as filters, with legacy nexus metadata - - Returns: - resources (list): list of resources - """ - resources = self.fetch(filters, cross_bucket=self.cross_bucket) - if not resources and legacy_filters is not None: - resources = self.fetch(legacy_filters, cross_bucket=self.cross_bucket) - - if resources: - return resources - return None - - def fetch_one(self, filters, legacy_filters=None, strict=True): - """Fetch one and only one resource based on filters.""" - - resources = self.fetch_legacy_compatible(filters, legacy_filters) - - if resources is None: - if strict: - raise AccessPointException(f"Could not get resource for filters {filters}") - return None - - if len(resources) > 1: - if strict: - raise AccessPointException(f"More than one resource for filters {filters}") - return resources[0] - - return resources[0] - - def download(self, resource_id, download_directory, content_type=None): - """Download datafile from nexus.""" - resource = self.forge.retrieve(resource_id, cross_bucket=True) - - if resource is None: - raise AccessPointException(f"Could not find resource for id: {resource_id}") - - if hasattr(resource, "distribution"): - file_paths = [] - if isinstance(resource.distribution, list): - for dist in resource.distribution: - if hasattr(dist, "name"): - file_paths.append(pathlib.Path(download_directory) / dist.name) - else: - raise AttributeError( - f"A distribution of the resource {resource.name} does " - "not have a file name." - ) - else: - file_paths = [pathlib.Path(download_directory) / resource.distribution.name] - - self.forge.download( - resource, - "distribution.contentUrl", - download_directory, - cross_bucket=True, - content_type=content_type, - overwrite=True, - ) - - # Verify that each datafile for the resource was successfully downloaded - for fp in file_paths: - if not fp.exists(): - raise AccessPointException( - f"Download failed: file {fp} does not exist for resource {resource_id}" - ) - - return [str(fp) for fp in file_paths] - - return [] - - def deprecate(self, filters, legacy_filters=None): - """Deprecate resources based on filters.""" - - tmp_cross_bucket = self.cross_bucket - self.cross_bucket = False - - resources = self.fetch_legacy_compatible(filters, legacy_filters) - - if resources: - for resource in resources: - rr = self.retrieve(resource.id) - if rr is not None: - self.forge.deprecate(rr) - - self.cross_bucket = tmp_cross_bucket - - def deprecate_all(self, metadata, metadata_legacy=None): - """Deprecate all resources used or produced by BluePyModel. Use with extreme caution.""" - - if not yesno("Confirm deprecation of all BluePyEmodel resources in Nexus project"): - return - - for type_ in NEXUS_TYPE_TO_CLASS.keys(): - filters = {"type": type_} - filters.update(metadata) - - if metadata_legacy is None: - legacy_filters = None - else: - legacy_filters = {"type": type_} - legacy_filters.update(metadata_legacy) - - self.deprecate(filters, legacy_filters) - - def resource_location(self, resource, download_directory): - """Get the path of the files attached to a resource. If the resource is - not located on gpfs, download it instead""" - - paths = [] - - if not hasattr(resource, "distribution"): - raise AccessPointException(f"Resource {resource} does not have distribution") - - if isinstance(resource.distribution, list): - distribution_iter = resource.distribution - else: - distribution_iter = [resource.distribution] - - for distrib in distribution_iter: - filepath = None - - if hasattr(distrib, "atLocation"): - loc = self.forge.as_json(distrib.atLocation) - if "location" in loc: - filepath = loc["location"].replace("file:/", "") - - if filepath is None: - filepath = self.download(resource.id, download_directory)[0] - - paths.append(filepath) - - return paths - - @staticmethod - def resource_name(class_name, metadata, seed=None): - """Create a resource name from the class name and the metadata.""" - name_parts = [CLASS_TO_RESOURCE_NAME[class_name]] - if "iteration" in metadata: - name_parts.append(metadata["iteration"]) - if "eModel" in metadata: - name_parts.append(metadata["eModel"]) - if "tType" in metadata: - name_parts.append(metadata["tType"]) - # legacy nexus emodel metadata - if "emodel" in metadata: - name_parts.append(metadata["emodel"]) - # legacy nexus ttype metadata - if "ttype" in metadata: - name_parts.append(metadata["ttype"]) - if seed is not None: - name_parts.append(str(seed)) - - return "__".join(name_parts) - - @staticmethod - def dump_json_and_get_distributions(object_, class_name, metadata_str, seed=None): - """Write object as json dict, and get distribution paths (obj as json and others)""" - json_payload = object_.as_dict() - - path_json = f"{CLASS_TO_RESOURCE_NAME[class_name]}__{metadata_str}" - if seed is not None: - path_json += f"__{seed}" - iteration = metadata_str.split("__iteration=")[1].split("__")[0] - path_json = str((pathlib.Path("./nexus_temp") / iteration / f"{path_json}.json").resolve()) - - distributions = [path_json] - json_payload.pop("nexus_images", None) # remove nexus_images from payload - if "nexus_distributions" in json_payload: - distributions += json_payload.pop("nexus_distributions") - - with open(path_json, "w") as fp: - json.dump(json_payload, fp, indent=2) - - return distributions - - @staticmethod - def get_seed_from_object(object_, class_name): - """Get the seed from the object if it has one else None.""" - seed = None - if class_name in ("EModel", "EModelScript"): - seed = object_.seed - return seed - - def object_to_nexus( - self, - object_, - metadata_dict, - metadata_str, - metadata_dict_legacy, - replace=True, - currents=None, - ): - """Transform a BPEM object into a dict which gets registered into Nexus as - the distribution of a Dataset of the matching type. The metadata - are also attached to the object to be able to retrieve the Resource.""" - - class_name = object_.__class__.__name__ - type_ = CLASS_TO_NEXUS_TYPE[class_name] - - seed = self.get_seed_from_object(object_, class_name) - score = None - if class_name == "EModel": - score = object_.fitness - - base_payload = { - "type": ["Entity", type_], - "name": self.resource_name(class_name, metadata_dict, seed=seed), - } - payload_existence = { - "type": type_, - "name": self.resource_name(class_name, metadata_dict, seed=seed), - } - payload_existence_legacy = { - "type": type_, - "name": self.resource_name(class_name, metadata_dict_legacy, seed=seed), - } - - base_payload.update(metadata_dict) - if score is not None: - base_payload["score"] = score - if currents is not None: - base_payload["holding_current"] = currents["holding"] - base_payload["threshold_current"] = currents["threshold"] - if hasattr(object_, "get_related_nexus_ids"): - related_nexus_ids = object_.get_related_nexus_ids() - if related_nexus_ids: - base_payload.update(related_nexus_ids) - - payload_existence.update(metadata_dict) - payload_existence.pop("annotation", None) - - payload_existence_legacy.update(metadata_dict_legacy) - payload_existence_legacy.pop("annotation", None) - - nexus_images = object_.as_dict().get("nexus_images", None) - distributions = self.dump_json_and_get_distributions( - object_=object_, class_name=class_name, metadata_str=metadata_str, seed=seed - ) - - self.register( - base_payload, - filters_existence=payload_existence, - legacy_filters_existence=payload_existence_legacy, - replace=replace, - distributions=distributions, - images=nexus_images, - type_=type_, - ) - - def update_distribution(self, resource, metadata_str, object_): - """Update a resource distribution using python object. - - Cannot update resource that has more than one resource.""" - class_name = object_.__class__.__name__ - seed = self.get_seed_from_object(object_, class_name) - - distributions = self.dump_json_and_get_distributions( - object_=object_, - class_name=class_name, - metadata_str=metadata_str, - seed=seed, - ) - - path_json = distributions[0] - - resource = Dataset.from_resource(self.forge, resource, store_metadata=True) - # Nexus behavior: - # - if only one element, gives either a dict or a list - # - if multiple elements, returns a list of elements - # Here, we want to be sure that we only have one element - if isinstance(resource.distribution, list): - if len(resource.distribution) != 1: - raise TypeError( - f"'update_distribution' method cannot be used on {class_name} {metadata_str} " - "with more than 1 distribution." - ) - elif not isinstance(resource.distribution, dict): - raise TypeError( - "'update_distribution' method requires a dict or a single-element list for " - f"{class_name} {metadata_str}, got {type(resource.distribution)} instead." - ) - - # add distribution from object and remove old one from resource - resource.add_distribution(path_json, content_type=f"application/{path_json.split('.')[-1]}") - resource.distribution = [resource.distribution[1]] - return resource - - def resource_to_object(self, type_, resource, metadata, download_directory): - """Transform a Resource into a BPEM object of the matching type""" - - file_paths = self.download(resource.id, download_directory) - json_path = next((fp for fp in file_paths if pathlib.Path(fp).suffix == ".json"), None) - - if json_path is None: - # legacy case where the payload is in the Resource - # can no longer use this for recent resources - payload = self.forge.as_json(resource) - - for k in metadata: - payload.pop(k, None) - - for k in NEXUS_ENTRIES: - payload.pop(k, None) - - else: - # Case in which the payload is in a .json distribution - with open(json_path, "r") as f: - payload = json.load(f) - - return NEXUS_TYPE_TO_CLASS[type_](**payload) - - def nexus_to_object(self, type_, metadata, download_directory, legacy_metadata=None): - """Search for a single Resource matching the ``type_`` and metadata and return it - as a BPEM object of the matching type""" - - filters = {"type": type_} - filters.update(metadata) - - legacy_filters = None - if legacy_metadata: - legacy_filters = {"type": type_} - legacy_filters.update(legacy_metadata) - - resource = self.fetch_one(filters, legacy_filters) - - return self.resource_to_object(type_, resource, metadata, download_directory) - - def nexus_to_objects(self, type_, metadata, download_directory, legacy_metadata=None): - """Search for Resources matching the ``type_`` and metadata and return them - as BPEM objects of the matching type""" - - filters = {"type": type_} - filters.update(metadata) - - legacy_filters = None - if legacy_metadata: - legacy_filters = {"type": type_} - legacy_filters.update(legacy_metadata) - - resources = self.fetch_legacy_compatible(filters, legacy_filters) - - objects_ = [] - ids = [] - - if resources: - for resource in resources: - objects_.append( - self.resource_to_object(type_, resource, metadata, download_directory) - ) - ids.append(resource.id) - - return objects_, ids - - def get_nexus_id(self, type_, metadata, legacy_metadata=None): - """Search for a single Resource matching the ``type_`` and metadata and return its id""" - filters = {"type": type_} - filters.update(metadata) - - legacy_filters = None - if legacy_metadata: - legacy_filters = {"type": type_} - legacy_filters.update(legacy_metadata) - - resource = self.fetch_one(filters, legacy_filters) - - return resource.id - - @staticmethod - def brain_region_filter(resources): - """Filter resources to keep only brain regions - - Arguments: - resources (list of Resource): resources to be filtered - - Returns: - list of Resource: the filtered resources - """ - return [ - r for r in resources if hasattr(r, "subClassOf") and r.subClassOf == "nsg:BrainRegion" - ] - - def type_filter(self, resources, filter): - """Filter resources to keep only etypes/mtypes/ttypes - - Arguments: - resources (list of Resource): resources to be filtered - filter (str): can be "etype", "mytype" or "ttype" - - Returns: - list of Resource: the filtered resources - """ - if filter == "etype": - available_names = self.available_etypes - elif filter == "mtype": - available_names = self.available_mtypes - elif filter == "ttype": - available_names = self.available_ttypes - else: - raise AccessPointException( - f'filter is {filter} but should be in ["etype", "mtype", "ttype"]' - ) - return [r for r in resources if r.id in available_names] - - def filter_resources(self, resources, filter): - """Filter resources - - Arguments: - resources (list of Resource): resources to be filtered - filter (str): which filter to use - can be "brain_region", "etype", "mtype", "ttype" - - Returns: - list of Resource: the filtered resources - - Raises: - AccessPointException if filter not in ["brain_region", "etype", "mtype", "ttype"] - """ - if filter == "brain_region": - return self.brain_region_filter(resources) - if filter in ["etype", "mtype", "ttype"]: - return self.type_filter(resources, filter) - - filters = ["brain_region", "etype", "mtype", "ttype"] - raise AccessPointException( - f"Filter not expected in filter_resources: {filter}" - f"Please choose among the following filters: {filters}" - ) - - -def ontology_forge_access_point( - access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT -): - """Returns an access point targeting the project containing the ontology for the - species and brain regions""" - - access_point = NexusForgeAccessPoint( - project="datamodels", - organisation="neurosciencegraph", - endpoint=endpoint, - forge_path=forge_path, - access_token=access_token, - ) - - return access_point - - -def atlas_forge_access_point(access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT): - """Returns an access point targeting the project containing the atlas""" - - access_point = NexusForgeAccessPoint( - project="atlas", - organisation="bbp", - endpoint=endpoint, - forge_path=forge_path, - access_token=access_token, - ) - - return access_point - - -def raise_not_found_exception(base_text, label, access_point, filter, limit=30): - """Raise an exception mentioning the possible appropriate resource names available on nexus - - Arguments: - base_text (str): text to display in the Exception - label (str): name of the resource to search for - access_point (NexusForgeAccessPoint) - filter (str): which filter to use - can be "brain_region", "etype", "mtype", or "ttype" - limit (int): maximum number of resources to fetch when looking up - for resource name suggestions - """ - if not base_text.endswith("."): - base_text = f"{base_text}." - - resources = access_point.resolve(label, strategy="all", limit=limit) - if resources is None: - raise AccessPointException(base_text) - - # make sure that resources is iterable - if not isinstance(resources, list): - resources = [resources] - filtered_names = "\n".join( - set(r.label for r in access_point.filter_resources(resources, filter)) - ) - if filtered_names: - raise AccessPointException(f"{base_text} Maybe you meant one of those:\n{filtered_names}") - - raise AccessPointException(base_text) - - -def check_resource( - label, - category, - access_point=None, - access_token=None, - forge_path=None, - endpoint=DEFAULT_NEXUS_ENDPOINT, -): - """Checks that resource is present on nexus and is part of the provided category - - Arguments: - label (str): name of the resource to search for - category (str): can be "etype", "mtype" or "ttype" - access_point (str): ontology_forge_access_point(access_token) - forge_path (str): path to a .yml used as configuration by nexus-forge. - endpoint (str): nexus endpoint - """ - allowed_categories = ["etype", "mtype", "ttype"] - if category not in allowed_categories: - raise AccessPointException(f"Category is {category}, but should be in {allowed_categories}") - - if access_point is None: - access_point = ontology_forge_access_point(access_token, forge_path, endpoint) - - resource = access_point.resolve(label, strategy="exact") - # raise Exception if resource was not found - if resource is None: - base_text = f"Could not find {category} with name {label}" - raise_not_found_exception(base_text, label, access_point, category) - - # if resource found but not of the appropriate category, also raise Exception - available_names = [] - if category == "etype": - available_names = access_point.available_etypes - elif category == "mtype": - available_names = access_point.available_mtypes - elif category == "ttype": - available_names = access_point.available_ttypes - if resource.id not in available_names: - base_text = f"Resource {label} is not a {category}" - raise_not_found_exception(base_text, label, access_point, category) - - -def get_available_traces(species=None, brain_region=None, access_token=None, forge_path=None): - """Returns a list of Resources of type Traces from the bbp/lnmce Nexus project""" - - filters = {"type": "ExperimentalTrace", "distribution": {"encodingFormat": "application/nwb"}} - - if species: - filters["subject"] = species - if brain_region: - filters["brainLocation"] = brain_region - - resources = [] - for proj_traces in NEXUS_PROJECTS_TRACES: - access_point = NexusForgeAccessPoint( - project=proj_traces["project"], - organisation=proj_traces["organisation"], - endpoint=DEFAULT_NEXUS_ENDPOINT, - forge_path=forge_path, - access_token=access_token, - cross_bucket=True, - ) - tmp_resources = access_point.fetch(filters=filters) - if tmp_resources: - resources += tmp_resources - - return resources - - -def get_brain_region( - brain_region, access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT -): - """Returns the resource corresponding to the brain region - - If the brain region name is not present in nexus, - raise an exception mentioning the possible brain region names available on nexus - - Arguments: - brain_region (str): name of the brain region to search for - access_token (str): nexus connection token - forge_path (str): path to a .yml used as configuration by nexus-forge. - endpoint (str): nexus endpoint - """ - - filter = "brain_region" - access_point = ontology_forge_access_point(access_token, forge_path, endpoint) - - if brain_region in ["SSCX", "sscx"]: - brain_region = "somatosensory areas" - if brain_region == "all": - # http://api.brain-map.org/api/v2/data/Structure/8 - brain_region = "Basic cell groups and regions" - - resource = access_point.resolve(brain_region, strategy="exact") - # try with capital 1st letter, or every letter lowercase - if resource is None: - # do not use capitalize, because it also make every other letter lowercase - if len(brain_region) > 1: - brain_region = f"{brain_region[0].upper()}{brain_region[1:]}" - elif len(brain_region) == 1: - brain_region = brain_region.upper() - resource = access_point.resolve(brain_region, strategy="exact") - - if resource is None: - resource = access_point.resolve(brain_region.lower(), strategy="exact") - - if isinstance(resource, list): - resource = resource[0] - - # raise Exception if resource was not found - if resource is None: - base_text = f"Could not find any brain region with name {brain_region}" - raise_not_found_exception(base_text, brain_region, access_point, filter) - - return resource - - -def get_brain_region_dict( - brain_region, access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT -): - """Returns a dict with id and label of the resource corresponding to the brain region - - Arguments: - brain_region (str): name of the brain region to search for - access_token (str): nexus connection token - forge_path (str): path to a .yml used as configuration by nexus-forge. - endpoint (str): nexus endpoint - - Returns: - dict: the id and label of the nexus resource of the brain region - """ - br_resource = get_brain_region(brain_region, access_token, forge_path, endpoint) - - access_point = ontology_forge_access_point(access_token, forge_path, endpoint) - - # if no exception was raised, filter to get id and label and return them - brain_region_dict = access_point.forge.as_json(br_resource) - return { - "id": brain_region_dict["id"], - "label": brain_region_dict["label"], - } - - -def get_brain_region_notation( - brain_region, access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT -): - """Get the ontology of the brain location.""" - if brain_region is None: - return None - - brain_region_resource = get_brain_region( - brain_region, access_token=access_token, forge_path=forge_path, endpoint=endpoint - ) - - return brain_region_resource.notation - - -def get_nexus_brain_region( - brain_region, access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT -): - """Get the ontology of the brain location.""" - if brain_region is None: - return None - - brain_region_from_nexus = get_brain_region_dict( - brain_region, access_token=access_token, forge_path=forge_path, endpoint=endpoint - ) - - return { - "type": "BrainLocation", - "brainRegion": brain_region_from_nexus, - } - - -def get_all_species(access_token=None, forge_path=None, endpoint=DEFAULT_NEXUS_ENDPOINT): - access_point = ontology_forge_access_point(access_token, forge_path, endpoint) - - resources = access_point.forge.search({"subClassOf": "nsg:Species"}, limit=100) - - return sorted(set(r.label for r in resources)) - - -def get_curated_morphology(resources): - """Get curated morphology from multiple resources with same morphology name""" - for r in resources: - if hasattr(r, "annotation"): - annotations = r.annotation if isinstance(r.annotation, list) else [r.annotation] - for annotation in annotations: - if "QualityAnnotation" in annotation.type: - if annotation.hasBody.label == "Curated": - return r - if hasattr(r, "derivation"): - return r - return None - - -def filter_mechanisms_with_brain_region(forge, resources, brain_region_label, br_visited): - """Filter mechanisms by brain region""" - br_visited.add(brain_region_label) - filtered_resources = [ - r - for r in resources - if hasattr(r, "brainLocation") and r.brainLocation.brainRegion.label == brain_region_label - ] - if len(filtered_resources) > 0: - return filtered_resources, br_visited - - query = ( - """ - SELECT DISTINCT ?br ?label - WHERE{ - ?id label \"""" - + f"{brain_region_label}" - + """\" ; - isPartOf ?br . - ?br label ?label . - } - """ - ) - brs = forge.sparql(query, limit=1000) - # when fails can be None or empty list - if brs: - new_brain_region_label = brs[0].label - return filter_mechanisms_with_brain_region( - forge, resources, new_brain_region_label, br_visited - ) - - # if no isPartOf present, try with isLayerPartOf - query = ( - """ - SELECT DISTINCT ?br ?label - WHERE{ - ?id label \"""" - + f"{brain_region_label}" - + """\" ; - isLayerPartOf ?br . - ?br label ?label . - } - """ - ) - brs = forge.sparql(query, limit=1000) - # when fails can be None or empty list - if brs: - # can have multiple brain regions - for br in brs: - new_brain_region_label = br.label - resources, br_visited = filter_mechanisms_with_brain_region( - forge, resources, new_brain_region_label, br_visited - ) - if resources is not None: - return resources, br_visited - - return None, br_visited diff --git a/bluepyemodel/access_point/nexus.py b/bluepyemodel/access_point/nexus.py deleted file mode 100755 index 65c00d9..0000000 --- a/bluepyemodel/access_point/nexus.py +++ /dev/null @@ -1,1502 +0,0 @@ -"""Access point using Nexus Forge""" - -""" -Copyright 2024 Blue Brain Project / EPFL - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import copy -import logging -import os -import pathlib -import subprocess -import time -from itertools import chain - -import pandas -from kgforge.core import Resource - -from bluepyemodel.access_point.access_point import DataAccessPoint -from bluepyemodel.access_point.forge_access_point import DEFAULT_NEXUS_ENDPOINT -from bluepyemodel.access_point.forge_access_point import NEXUS_PROJECTS_TRACES -from bluepyemodel.access_point.forge_access_point import AccessPointException -from bluepyemodel.access_point.forge_access_point import NexusForgeAccessPoint -from bluepyemodel.access_point.forge_access_point import check_resource -from bluepyemodel.access_point.forge_access_point import filter_mechanisms_with_brain_region -from bluepyemodel.access_point.forge_access_point import get_available_traces -from bluepyemodel.access_point.forge_access_point import get_brain_region_notation -from bluepyemodel.access_point.forge_access_point import get_curated_morphology -from bluepyemodel.access_point.forge_access_point import get_nexus_brain_region -from bluepyemodel.access_point.forge_access_point import ontology_forge_access_point -from bluepyemodel.efeatures_extraction.trace_file import TraceFile -from bluepyemodel.emodel_pipeline.emodel_script import EModelScript -from bluepyemodel.emodel_pipeline.emodel_settings import EModelPipelineSettings -from bluepyemodel.emodel_pipeline.emodel_workflow import EModelWorkflow -from bluepyemodel.export_emodel.utils import copy_hocs_to_new_output_path -from bluepyemodel.export_emodel.utils import get_hoc_file_path -from bluepyemodel.export_emodel.utils import get_output_path -from bluepyemodel.export_emodel.utils import select_emodels -from bluepyemodel.model.mechanism_configuration import MechanismConfiguration -from bluepyemodel.tools.mechanisms import NEURON_BUILTIN_MECHANISMS -from bluepyemodel.tools.mechanisms import discriminate_by_temp - -# pylint: disable=too-many-arguments,unused-argument - -logger = logging.getLogger("__main__") - - -class NexusAccessPoint(DataAccessPoint): - """API to retrieve, push and format data from and to the Knowledge Graph""" - - def __init__( - self, - emodel=None, - etype=None, - ttype=None, - mtype=None, - species=None, - brain_region=None, - iteration_tag=None, - synapse_class=None, - project="emodel_pipeline", - organisation="demo", - endpoint=DEFAULT_NEXUS_ENDPOINT, - forge_path=None, - forge_ontology_path=None, - access_token=None, - sleep_time=10, - ): - """Init - - Args: - emodel (str): name of the emodel - etype (str): name of the electric type. - ttype (str): name of the transcriptomic type. - Required if using the gene expression or IC selector. - mtype (str): name of the morphology type. - species (str): name of the species. - brain_region (str): name of the brain location. - iteration_tag (str): tag associated to the current run. - synapse_class (str): synapse class (neurotransmitter). - project (str): name of the Nexus project. - organisation (str): name of the Nexus organization to which the project belong. - endpoint (str): Nexus endpoint. - forge_path (str): path to a .yml used as configuration by nexus-forge. - forge_ontology_path (str): path to a .yml used for the ontology in nexus-forge. - access_token (str): Nexus connection token. - sleep_time (int): time to wait between two Nexus requests (in case of slow indexing). - """ - - super().__init__( - emodel, - etype, - ttype, - mtype, - species, - brain_region, - iteration_tag, - synapse_class, - ) - - self.access_point = NexusForgeAccessPoint( - project=project, - organisation=organisation, - endpoint=endpoint, - forge_path=forge_path, - access_token=access_token, - ) - - if forge_ontology_path is None: - self.forge_ontology_path = forge_path - else: - self.forge_ontology_path = forge_ontology_path - - # This trick is used to have nexus type descriptions on one side and basic - # strings on the other - self.emodel_metadata_ontology = copy.deepcopy(self.emodel_metadata) - self.build_ontology_based_metadata() - self.emodel_metadata.allen_notation = get_brain_region_notation( - self.emodel_metadata.brain_region, - self.access_point.access_token, - self.forge_ontology_path, - self.access_point.endpoint, - ) - - self.pipeline_settings = self.get_pipeline_settings(strict=False) - - self.download_directory.mkdir(parents=True, exist_ok=True) - - self.sleep_time = sleep_time - - @property - def download_directory(self): - return pathlib.Path("./nexus_temp") / str(self.emodel_metadata.iteration) - - def check_mettypes(self): - """Check that etype, mtype and ttype are present on nexus""" - ontology_access_point = ontology_forge_access_point( - self.access_point.access_token, self.forge_ontology_path, self.access_point.endpoint - ) - - logger.info("Checking if etype %s is present on nexus...", self.emodel_metadata.etype) - check_resource( - self.emodel_metadata.etype, - "etype", - access_point=ontology_access_point, - access_token=self.access_point.access_token, - forge_path=self.forge_ontology_path, - endpoint=self.access_point.endpoint, - ) - logger.info("Etype checked") - - if self.emodel_metadata.mtype is not None: - logger.info( - "Checking if mtype %s is present on nexus...", - self.emodel_metadata.mtype, - ) - check_resource( - self.emodel_metadata.mtype, - "mtype", - access_point=ontology_access_point, - access_token=self.access_point.access_token, - forge_path=self.forge_ontology_path, - endpoint=self.access_point.endpoint, - ) - logger.info("Mtype checked") - else: - logger.info("Mtype is None, its presence on Nexus is not being checked.") - - if self.emodel_metadata.ttype is not None: - logger.info( - "Checking if ttype %s is present on nexus...", - self.emodel_metadata.ttype, - ) - check_resource( - self.emodel_metadata.ttype, - "ttype", - access_point=ontology_access_point, - access_token=self.access_point.access_token, - forge_path=self.forge_ontology_path, - endpoint=self.access_point.endpoint, - ) - logger.info("Ttype checked") - else: - logger.info("Ttype is None, its presence on Nexus is not being checked.") - - def get_pipeline_settings(self, strict=True): - if strict: - return self.access_point.nexus_to_object( - type_="EModelPipelineSettings", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - try: - return self.access_point.nexus_to_object( - type_="EModelPipelineSettings", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - except AccessPointException: - return EModelPipelineSettings() - - def store_pipeline_settings(self, pipeline_settings): - """Save an EModelPipelineSettings on Nexus""" - - self.access_point.object_to_nexus( - pipeline_settings, - self.emodel_metadata_ontology.for_resource(), - self.emodel_metadata.as_string(), - self.emodel_metadata_ontology.filters_for_resource_legacy(), - replace=True, - ) - - def build_ontology_based_metadata(self): - """Get the ontology related to the metadata""" - - self.emodel_metadata_ontology.species = self.get_nexus_subject(self.emodel_metadata.species) - self.emodel_metadata_ontology.brain_region = get_nexus_brain_region( - self.emodel_metadata.brain_region, - self.access_point.access_token, - self.forge_ontology_path, - self.access_point.endpoint, - ) - - def get_nexus_subject(self, species): - """ - Get the ontology of a species based on the species name. - - Args: - species (str): The common name or scientific name of the species. - Can be None, in which case the function will return None. - - Returns: - dict: The ontology data for the specified species. - - Raises: - ValueError: If the species is not recognized. - """ - - if species is None: - return None - - species = species.lower() - if species in ("human", "homo sapiens"): - subject = { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_9606", - "label": "Homo sapiens", - }, - } - - elif species in ("rat", "rattus norvegicus"): - subject = { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_10116", - "label": "Rattus norvegicus", - }, - } - - elif species in ("mouse", "mus musculus"): - subject = { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_10090", - "label": "Mus musculus", - }, - } - else: - raise ValueError(f"Unknown species {species}.") - - return subject - - def store_object( - self, - object_, - seed=None, - description=None, - currents=None, - is_analysis_suitable=False, - is_curated=False, - ): - """Store a BPEM object on Nexus""" - - metadata_dict = self.emodel_metadata_ontology.for_resource( - is_analysis_suitable=is_analysis_suitable, - is_curated=is_curated, - ) - if seed is not None: - metadata_dict["seed"] = seed - if description is not None: - metadata_dict["description"] = description - - self.access_point.object_to_nexus( - object_, - metadata_dict, - self.emodel_metadata.as_string(), - self.emodel_metadata_ontology.filters_for_resource_legacy(), - replace=True, - currents=currents, - ) - - def get_targets_configuration(self): - """Get the configuration of the targets (targets and ephys files used)""" - - configuration = self.access_point.nexus_to_object( - type_="ExtractionTargetsConfiguration", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - configuration.available_traces = self.get_available_traces() - configuration.available_efeatures = self.get_available_efeatures() - - if not configuration.files: - logger.debug( - "Empty list of files in the TargetsConfiguration, filling " - "it using what is available on Nexus for the present etype." - ) - filtered_traces = [ - trace - for trace in configuration.available_traces - if trace.etype == self.emodel_metadata.etype - ] - if not filtered_traces: - raise AccessPointException( - "Could not find any trace with etype {self.emodel_metadata.etype}. " - "Please specify files in your ExtractionTargetsConfiguration." - ) - configuration.files = filtered_traces - - for file in configuration.files: - file.filepath = self.download_trace( - id_=file.id, id_legacy=file.resource_id, name=file.filename - ) - - return configuration - - def store_targets_configuration(self, configuration): - """Store the configuration of the targets (targets and ephys files used)""" - - # Search for all Traces on Nexus and add their Nexus ids to the configuration - traces = get_available_traces( - access_token=self.access_point.access_token, - forge_path=self.access_point.forge_path, - ) - - available_traces_names = [trace.name for trace in traces] - - for file in configuration.files: - if file.cell_name in available_traces_names: - file.id = traces[available_traces_names.index(file.cell_name)].id - else: - logger.warning("Trace %s not found.", file.cell_name) - - self.store_object(configuration) - - def get_fitness_calculator_configuration(self, record_ions_and_currents=False): - """Get the configuration of the fitness calculator (efeatures and protocols)""" - - configuration = self.access_point.nexus_to_object( - type_="FitnessCalculatorConfiguration", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - # contains ion currents and ionic concentrations to be recorded - ion_variables = None - if record_ions_and_currents: - ion_currents, ionic_concentrations = self.get_ion_currents_concentrations() - if ion_currents is not None and ionic_concentrations is not None: - ion_variables = list(chain.from_iterable((ion_currents, ionic_concentrations))) - - for prot in configuration.protocols: - prot.recordings, prot.recordings_from_config = prot.init_recordings( - prot.recordings_from_config, ion_variables - ) - - if configuration.name_rmp_protocol is None: - configuration.name_rmp_protocol = self.pipeline_settings.name_rmp_protocol - if configuration.name_rin_protocol is None: - configuration.name_rin_protocol = self.pipeline_settings.name_Rin_protocol - if configuration.validation_protocols is None or configuration.validation_protocols == []: - configuration.validation_protocols = self.pipeline_settings.validation_protocols - if configuration.stochasticity is None: - configuration.stochasticity = self.pipeline_settings.stochasticity - - return configuration - - def store_fitness_calculator_configuration(self, configuration): - """Store a fitness calculator configuration as a resource of type - FitnessCalculatorConfiguration""" - workflow, nexus_id = self.get_emodel_workflow() - - if workflow is None: - raise AccessPointException( - "No EModelWorkflow available to which the EModels can be linked" - ) - - configuration.workflow_id = nexus_id - self.store_object(configuration) - # wait for the object to be uploaded and fetchable - time.sleep(self.sleep_time) - - # fetch just uploaded FCC resource to get its id and give it to emodel workflow - type_ = "FitnessCalculatorConfiguration" - filters = {"type": type_} - filters.update(self.emodel_metadata_ontology.filters_for_resource()) - filters_legacy = {"type": type_} - filters_legacy.update(self.emodel_metadata_ontology.filters_for_resource_legacy()) - resource = self.access_point.fetch_one(filters, filters_legacy) - fitness_id = resource.id - - workflow.fitness_configuration_id = fitness_id - self.store_or_update_emodel_workflow(workflow) - - def get_model_configuration(self, skip_get_available_morph=True): - """Get the configuration of the model, including parameters, mechanisms and distributions - - Args: - skip_get_available_morphs (bool): set to True to skip getting the available - morphologies and setting them to configuration. - available_morphologies are only used in - bluepyemodel.model.model_configuration.configure_model, so we assume - they have already been checked for configuration present on nexus. - """ - - configuration = self.access_point.nexus_to_object( - type_="EModelConfiguration", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - morph_path = self.download_morphology( - configuration.morphology.name, - configuration.morphology.format, - configuration.morphology.id, - ) - any_downloaded = False - if self.pipeline_settings.use_ProbAMPANMDA_EMS: - any_downloaded = self.download_ProbAMPANMDA_EMS() - self.download_mechanisms(configuration.mechanisms, any_downloaded) - - configuration.morphology.path = morph_path - logger.debug("Using morphology: %s", configuration.morphology.path) - configuration.available_mechanisms = self.get_available_mechanisms() - if not skip_get_available_morph: - configuration.available_morphologies = self.get_available_morphologies() - - return configuration - - def fetch_and_filter_mechanism(self, mechanism, ontology_access_point): - """Find a mech resource based on its brain region, temperature, species and ljp correction - - Args: - mechanism (MechanismConfiguration): the mechanism to find on nexus - ontology_access_point (NexusForgeAccessPoint): access point - where to find the brain regions - - Returns the mechanism as a nexus resource - """ - default_temperatures = [34, 35, 37] - default_ljp = True - - resources = self.access_point.fetch( - {"type": "SubCellularModelScript", "name": mechanism.name} - ) - if resources is None: - raise AccessPointException(f"SubCellularModelScript {mechanism.name} not found") - - # brain region filtering - br_visited = set() - filtered_resources, br_visited = filter_mechanisms_with_brain_region( - ontology_access_point.forge, - resources, - self.emodel_metadata_ontology.brain_region["brainRegion"]["label"], - br_visited, - ) - br_visited_to_str = ", ".join(br_visited) - error_msg = f"brain region in ({br_visited_to_str})" - if filtered_resources is not None: - resources = filtered_resources - - # temperature filtering - if mechanism.temperature is not None: - error_msg += f"temperature = {mechanism.temperature} " - filtered_resources = [ - r - for r in resources - if hasattr(r, "temperature") - and getattr(r.temperature, "value", r.temperature) == mechanism.temperature - ] - if len(filtered_resources) > 0: - resources = filtered_resources - - # species filtering - error_msg += f"species = {self.emodel_metadata_ontology.species['species']['label']} " - filtered_resources = [ - r - for r in resources - if hasattr(r, "subject") - and r.subject.species.label == self.emodel_metadata_ontology.species["species"]["label"] - ] - if len(filtered_resources) > 0: - resources = filtered_resources - - # ljp correction filtering - if mechanism.ljp_corrected is not None: - error_msg += f"ljp correction = {mechanism.ljp_corrected} " - filtered_resources = [ - r - for r in resources - if hasattr(r, "isLjpCorrected") and r.isLjpCorrected == mechanism.ljp_corrected - ] - if len(filtered_resources) > 0: - resources = filtered_resources - - if len(resources) == 0: - raise AccessPointException( - f"SubCellularModelScript {mechanism.name} not found with {error_msg}" - ) - - # use default values - if len(resources) > 1: - logger.warning( - "More than one resource fetched for mechanism %s", - mechanism.name, - ) - if len(resources) > 1 and mechanism.temperature is None: - resources = discriminate_by_temp(resources, default_temperatures) - - if len(resources) > 1 and mechanism.ljp_corrected is None: - tmp_resources = [ - r - for r in resources - if hasattr(r, "isLjpCorrected") and r.isLjpCorrected is default_ljp - ] - if len(tmp_resources) > 0 and len(tmp_resources) < len(resources): - logger.warning( - "Discriminating resources based on ljp correction. " - "Keeping only resource with ljp correction." - ) - resources = tmp_resources - - if len(resources) > 1: - logger.warning( - "Could not reduce the number of resources fetched down to one. " - "Keeping the 1st resource of the list." - ) - - return resources[0] - - def store_model_configuration(self, configuration, path=None): - """Store a model configuration as a resource of type EModelConfiguration""" - - # Search for all Morphologies on Nexus and add their Nexus ids to the configuration - morphologies = self.access_point.fetch({"type": "NeuronMorphology"}) - if not morphologies: - raise AccessPointException( - "Cannot find morphologies on Nexus. Please make sure that " - "morphologies can be reached from the current Nexus session." - ) - - available_morphologies_names = [morphology.name for morphology in morphologies] - - if configuration.morphology.name in available_morphologies_names: - configuration.morphology.id = morphologies[ - available_morphologies_names.index(configuration.morphology.name) - ].id - else: - logger.warning("Morphology %s not found.", configuration.morphology.name) - - # set id to mechanisms by filtering with brain region, temperature, species, ljp correction - ontology_access_point = ontology_forge_access_point( - self.access_point.access_token, self.forge_ontology_path, self.access_point.endpoint - ) - for mechanism in configuration.mechanisms: - if mechanism.name in NEURON_BUILTIN_MECHANISMS: - continue - mech_resource = self.fetch_and_filter_mechanism(mechanism, ontology_access_point) - mechanism.id = mech_resource.id - - if self.pipeline_settings.use_ProbAMPANMDA_EMS: - ProbAMPANMDA_EMS_id = self.get_ProbAMPANMDA_EMS_resource().id - configuration.extra_mech_ids = [(ProbAMPANMDA_EMS_id, "SynapsePhysiologyModel")] - - self.store_object(configuration) - - def get_distributions(self): - """Get the list of available distributions""" - - return self.access_point.nexus_to_objects( - type_="EModelChannelDistribution", - metadata={}, - download_directory=self.download_directory, - )[0] - - def store_distribution(self, distribution): - """Store a channel distribution as a resource of type EModelChannelDistribution""" - - self.store_object(distribution) - - def create_emodel_workflow(self, state="not launched"): - """Create an EModelWorkflow instance filled with the appropriate configuration""" - - try: - targets_configuration_id = self.access_point.get_nexus_id( - type_="ExtractionTargetsConfiguration", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - except AccessPointException: - targets_configuration_id = None - - try: - pipeline_settings_id = self.access_point.get_nexus_id( - type_="EModelPipelineSettings", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - except AccessPointException: - pipeline_settings_id = None - - emodel_configuration_id = self.access_point.get_nexus_id( - type_="EModelConfiguration", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - try: - fitness_configuration_id = self.access_point.get_nexus_id( - type_="FitnessCalculatorConfiguration", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - except AccessPointException: - fitness_configuration_id = None - - return EModelWorkflow( - targets_configuration_id, - pipeline_settings_id, - emodel_configuration_id, - fitness_configuration_id=fitness_configuration_id, - state=state, - ) - - def get_emodel_workflow(self): - """Get the emodel workflow, containing configuration data and workflow status - - Returns None if the emodel workflow is not present on nexus.""" - - emodel_workflow, ids = self.access_point.nexus_to_objects( - type_="EModelWorkflow", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - if emodel_workflow: - return emodel_workflow[0], ids[0] - - return None, None - - def check_emodel_workflow_configurations(self, emodel_workflow): - """Return True if the emodel workflow's configurations are on nexus, and False otherwise""" - - for id_ in emodel_workflow.get_configuration_ids(): - if id_ is not None and self.access_point.retrieve(id_) is None: - return False - - return True - - def store_or_update_emodel_workflow(self, emodel_workflow): - """If emodel workflow is not on nexus, store it. If it is, fetch it and update its state""" - # pylint: disable=protected-access - type_ = "EModelWorkflow" - - filters = {"type": type_} - filters.update(self.emodel_metadata_ontology.filters_for_resource()) - filters_legacy = {"type": type_} - filters_legacy.update(self.emodel_metadata_ontology.filters_for_resource_legacy()) - resources = self.access_point.fetch_legacy_compatible(filters, filters_legacy) - - # not present on nexus yet -> store it - if resources is None: - self.access_point.object_to_nexus( - emodel_workflow, - self.emodel_metadata_ontology.for_resource(), - self.emodel_metadata.as_string(), - self.emodel_metadata_ontology.filters_for_resource_legacy(), - replace=False, - ) - # if present on nexus -> update its state - else: - schema_type = "Entity" - resource = resources[0] - resource.state = emodel_workflow.state - ids_dict = emodel_workflow.get_related_nexus_ids() - if "generates" in ids_dict: - resource.generates = ids_dict["generates"] - schema_type = "EModelWorkflow" - if "hasPart" in ids_dict: - resource.hasPart = ids_dict["hasPart"] - - # in case some data has been updated, e.g. fitness_configuration_id - updated_resource = self.access_point.update_distribution( - resource, self.emodel_metadata.as_string(), emodel_workflow - ) - - schema_id = self.access_point.forge._model.schema_id(schema_type) - self.access_point.forge.update(updated_resource, schema_id=schema_id) - - def update_emodel_images(self, seed, keep_old_images=False): - """Update an EModel resource with local emodel plots.""" - # pylint: disable=protected-access - type_ = "EModel" - - filters = {"type": type_} - filters.update(self.emodel_metadata_ontology.filters_for_resource()) - filters_legacy = {"type": type_} - filters_legacy.update(self.emodel_metadata_ontology.filters_for_resource_legacy()) - filters["seed"] = int(seed) - filters_legacy["seed"] = int(seed) - resources = self.access_point.fetch_legacy_compatible(filters, filters_legacy) - if resources is None: - return - em_r = resources[0] - if not keep_old_images: - em_r.image = [] # remove any previous images - - em = self.get_emodel(seed=seed) - - em_r = self.access_point.add_images_to_resource( - em.as_dict()["nexus_images"], em_r, filters_existence=None - ) - schema_id = self.access_point.forge._model.schema_id("EModel") - - self.access_point.forge.update(em_r, schema_id=schema_id) - - def get_emodel(self, seed=None): - """Fetch an emodel""" - - metadata = self.emodel_metadata_ontology.filters_for_resource() - legacy_metadata = self.emodel_metadata_ontology.filters_for_resource_legacy() - - if seed is not None: - metadata["seed"] = int(seed) - legacy_metadata["seed"] = int(seed) - - emodel = self.access_point.nexus_to_object( - type_="EModel", - metadata=metadata, - download_directory=self.download_directory, - legacy_metadata=legacy_metadata, - ) - emodel.emodel_metadata = copy.deepcopy(self.emodel_metadata) - - return emodel - - def store_or_update_emodel(self, emodel): - """Update emodel if already present on nexus. If not, store it.""" - # pylint: disable=protected-access - type_ = "EModel" - - filters = {"type": type_} - filters.update(self.emodel_metadata_ontology.filters_for_resource()) - filters_legacy = {"type": type_} - filters_legacy.update(self.emodel_metadata_ontology.filters_for_resource_legacy()) - filters["seed"] = int(emodel.seed) - filters_legacy["seed"] = int(emodel.seed) - resources = self.access_point.fetch_legacy_compatible(filters, filters_legacy) - - if resources is None: - self.store_emodel(emodel) - return - - em_r = resources[0] - emodel_dict = emodel.as_dict() - - em_r = self.access_point.add_images_to_resource( - emodel_dict["nexus_images"], em_r, filters_existence=None - ) - self.access_point.update_distribution(em_r, self.emodel_metadata.as_string(), emodel) - em_r.score = emodel.fitness - - schema_id = self.access_point.forge._model.schema_id("EModel") - self.access_point.forge.update(em_r, schema_id=schema_id) - - def store_emodel(self, emodel, description=None): - """Store an EModel on Nexus""" - - workflow, nexus_id = self.get_emodel_workflow() - - if workflow is None: - raise AccessPointException( - "No EModelWorkflow available to which the EModels can be linked" - ) - - emodel.workflow_id = nexus_id - # make it analysis suitable AND curated if this is True - is_analysis_suitable = ( - self.has_fitness_calculator_configuration - and self.has_model_configuration - and self.has_pipeline_settings - and self.has_targets_configuration - ) - self.store_object( - emodel, - seed=emodel.seed, - description=description, - is_analysis_suitable=is_analysis_suitable, - is_curated=is_analysis_suitable, - ) - # wait for the object to be uploaded and fetchable - time.sleep(self.sleep_time) - - # fetch just uploaded emodel resource to get its id and give it to emodel workflow - type_ = "EModel" - filters = {"type": type_, "seed": emodel.seed} - filters.update(self.emodel_metadata_ontology.filters_for_resource()) - filters_legacy = {"type": type_, "seed": emodel.seed} - filters_legacy.update(self.emodel_metadata_ontology.filters_for_resource_legacy()) - resource = self.access_point.fetch_one(filters, filters_legacy) - model_id = resource.id - - workflow.add_emodel_id(model_id) - self.store_or_update_emodel_workflow(workflow) - - def get_emodels(self, emodels=None): - """Get all the emodels""" - - emodels, _ = self.access_point.nexus_to_objects( - type_="EModel", - metadata=self.emodel_metadata_ontology.filters_for_resource(), - download_directory=self.download_directory, - legacy_metadata=self.emodel_metadata_ontology.filters_for_resource_legacy(), - ) - - for em in emodels: - em.emodel_metadata = copy.deepcopy(self.emodel_metadata) - - return emodels - - def has_best_model(self, seed): - """Check if the best model has been stored.""" - - try: - self.get_emodel(seed=seed) - return True - except AccessPointException: - return False - - def is_checked_by_validation(self, seed): - """Check if the emodel with a given seed has been checked by Validation task. - - Reminder: the logic of validation is as follows: - if None: did not go through validation - if False: failed validation - if True: passed validation - """ - - try: - emodel = self.get_emodel(seed=seed) - except AccessPointException: - return False - - if emodel.passed_validation is True or emodel.passed_validation is False: - return True - - return False - - def is_validated(self): - """Check if enough models have been validated. - - Reminder: the logic of validation is as follows: - if None: did not go through validation - if False: failed validation - if True: passed validation - """ - - try: - emodels = self.get_emodels() - except TypeError: - return False - - n_validated = len([em for em in emodels if em.passed_validation]) - - return n_validated >= self.pipeline_settings.n_model - - def has_pipeline_settings(self): - """Returns True if pipeline settings are present on Nexus""" - - try: - _ = self.get_pipeline_settings(strict=True) - return True - except AccessPointException: - return False - - def has_fitness_calculator_configuration(self): - """Check if the fitness calculator configuration exists""" - - try: - _ = self.get_fitness_calculator_configuration() - return True - except AccessPointException: - return False - - def has_targets_configuration(self): - """Check if the target configuration exists""" - - try: - _ = self.get_targets_configuration() - return True - except AccessPointException: - return False - - def has_model_configuration(self): - """Check if the model configuration exists""" - - try: - _ = self.get_model_configuration() - return True - except AccessPointException: - return False - - def get_ProbAMPANMDA_EMS_resource(self): - """Get the ProbAMPANMDA_EMS resource from nexus.""" - resources = self.access_point.fetch( - {"type": "SynapsePhysiologyModel", "name": "ProbAMPANMDA_EMS"} - ) - if resources is None: - raise AccessPointException("SynapsePhysiologyModel ProbAMPANMDA_EMS not found") - - if len(resources) > 1: - logger.warning( - "Could not reduce the number of resources fetched down to one. " - "Keeping the 1st resource of the list." - ) - return resources[0] - - def download_ProbAMPANMDA_EMS(self): - """Download the ProbAMPANMDA_EMS mod file. - - Returns True if the mod file has been downloaded, returns False otherwise. - """ - mechanisms_directory = self.get_mechanisms_directory() - resource = self.get_ProbAMPANMDA_EMS_resource() - - mod_file_name = "ProbAMPANMDA_EMS.mod" - if os.path.isfile(str(mechanisms_directory / mod_file_name)): - return False - - filepath = self.access_point.download( - resource.id, str(mechanisms_directory), content_type="application/neuron-mod" - )[0] - - # Rename the file in case it's different from the name of the resource - filepath = pathlib.Path(filepath) - if filepath.stem != "ProbAMPANMDA_EMS": - filepath.rename(pathlib.Path(filepath.parent / mod_file_name)) - - return True - - def download_mechanisms(self, mechanisms, any_downloaded=False): - """Download the mod files if not already downloaded""" - # pylint: disable=protected-access - - mechanisms_directory = self.get_mechanisms_directory() - ontology_access_point = ontology_forge_access_point( - self.access_point.access_token, self.forge_ontology_path, self.access_point.endpoint - ) - - for mechanism in mechanisms: - if mechanism.name in NEURON_BUILTIN_MECHANISMS: - continue - - resource = None - if mechanism.id is not None: - resource = self.access_point.forge.retrieve(mechanism.id) - if resource is not None and resource._store_metadata["_deprecated"]: - logger.info( - "Nexus resource for mechanism %s is deprecated. " - "Looking for a new resource...", - mechanism.name, - ) - resource = None - - # if could not find by id, try with filtering - if resource is None: - resource = self.fetch_and_filter_mechanism(mechanism, ontology_access_point) - - mod_file_name = f"{mechanism.name}.mod" - if os.path.isfile(str(mechanisms_directory / mod_file_name)): - continue - - filepath = self.access_point.download(resource.id, str(mechanisms_directory))[0] - any_downloaded = True - # Rename the file in case it's different from the name of the resource - filepath = pathlib.Path(filepath) - if filepath.stem != mechanism: - filepath.rename(pathlib.Path(filepath.parent / mod_file_name)) - - if any_downloaded: - previous_dir = os.getcwd() - os.chdir(pathlib.Path(mechanisms_directory.parent)) - subprocess.run("nrnivmodl mechanisms", shell=True, check=True) - os.chdir(previous_dir) - - def download_morphology(self, name=None, format_=None, id_=None): - """Download a morphology by its id if provided. If no id is given, - the function attempts to download the morphology by its name, - provided it has not already been downloaded - - Args: - name (str): name of the morphology resource - format_ (str): Optional. Can be 'asc', 'swc', or 'h5'. - Must be available in the resource. - id_ (str): id of the nexus resource - - Raises: - TypeError if id_ and name are not given - AccessPointException if resource could not be retrieved - FileNotFoundError if downloaded morphology could not be find locally - """ - - if id_ is None and name is None: - raise TypeError("In download_morphology, at least name or id_ must be given.") - - if id_ is None: - species_label = self.emodel_metadata_ontology.species["species"]["label"] - resources = self.access_point.fetch( - { - "type": "NeuronMorphology", - "name": name, - "subject": {"species": {"label": species_label}}, - } - ) - if resources is None: - raise AccessPointException(f"Could not get resource for morphology {name}") - if len(resources) == 1: - resource = resources[0] - elif len(resources) >= 2: - resource = get_curated_morphology(resources) - if resource is None: - raise AccessPointException(f"Could not get resource for morphology {name}") - - res_id = resource.id - else: - res_id = id_ - - filepath = pathlib.Path(self.access_point.download(res_id, self.download_directory)[0]) - - # Some morphologies have .h5 attached and we don't want that: - if format_: - suffix = "." + format_ - filepath = filepath.with_suffix(suffix) - # special case example: format_ is 'asc', but morph has '.ASC' format - if not filepath.is_file() and filepath.with_suffix(suffix.upper()).is_file(): - filepath = filepath.with_suffix(suffix.upper()) - elif filepath.suffix == ".h5": - for suffix in [".swc", ".asc", ".ASC"]: - if filepath.with_suffix(suffix).is_file(): - filepath = filepath.with_suffix(suffix) - break - else: - raise FileNotFoundError( - f"Could not find morphology {filepath.stem}" - f"at path {filepath.parent} with allowed suffix '.asc', '.swc' or '.ASC'." - ) - - return str(filepath) - - def download_trace(self, id_=None, id_legacy=None, name=None): - """Does not actually download the Trace since traces are already stored on Nexus""" - - for proj_traces in NEXUS_PROJECTS_TRACES: - access_point = NexusForgeAccessPoint( - project=proj_traces["project"], - organisation=proj_traces["organisation"], - endpoint=DEFAULT_NEXUS_ENDPOINT, - forge_path=self.access_point.forge_path, - access_token=self.access_point.access_token, - cross_bucket=True, - ) - - if id_: - resource = access_point.retrieve(id_) - elif id_legacy: - resource = access_point.retrieve(id_legacy) - elif name: - resource = access_point.fetch_one( - { - "type": "Trace", - "name": name, - "distribution": {"encodingFormat": "application/nwb"}, - }, - strict=False, - ) - else: - raise TypeError("At least id_ or name should be informed.") - - if resource: - return access_point.resource_location(resource, self.download_directory)[0] - - raise ValueError(f"No matching resource for {id_} {name}") - - def get_mechanisms_directory(self): - """Return the path to the directory containing the mechanisms for the current emodel""" - - mechanisms_directory = self.download_directory / "mechanisms" - - return mechanisms_directory.resolve() - - def load_channel_gene_expression(self, name): - """Retrieve a channel gene expression resource and read its content""" - - dataset = self.access_point.fetch_one(filters={"type": "RNASequencing", "name": name}) - - filepath = self.access_point.resource_location(dataset, self.download_directory)[0] - - df = pandas.read_csv(filepath, index_col=["me-type", "t-type", "modality"]) - - return df, filepath - - def load_ic_map(self): - """Get the ion channel/genes map from Nexus""" - - resource_ic_map = self.access_point.fetch_one( - {"type": "IonChannelMapping", "name": "icmapping"} - ) - - return self.access_point.download(resource_ic_map.id, self.download_directory)[0] - - def get_t_types(self, table_name): - """Get the list of t-types available for the present emodel""" - - df, _ = self.load_channel_gene_expression(table_name) - # replace non-alphanumeric characters with underscores in t-types from RNASeq data - df["me-type"] = df["me-type"].str.replace(r"\W", "_") - return ( - df.loc[self.emodel_metadata.emodel].index.get_level_values("t-type").unique().tolist() - ) - - def get_available_morphologies(self): - """Get the list of names of the available morphologies""" - - resources = self.access_point.fetch({"type": "NeuronMorphology"}) - - if resources: - return {r.name for r in resources} - - logger.warning("Did not find any available morphologies.") - return set() - - def get_available_mechanisms(self, filters=None): - """Get all the available mechanisms. - Optional filters can be applied to refine the search.""" - - filter = {"type": "SubCellularModelScript"} - if filters: - filter.update(filters) - # do not look in other projects for these resources, - # because resources have 'full' metadata only in a few projects. - resources = self.access_point.fetch(filter, cross_bucket=False) - if resources is None: - logger.warning("No SubCellularModelScript mechanisms available") - return None - - available_mechanisms = [] - for r in resources: - logger.debug("fetching %s mechanism from nexus.", r.name) - version = r.modelId if hasattr(r, "modelId") else None - temperature = ( - getattr(r.temperature, "value", r.temperature) - if hasattr(r, "temperature") - else None - ) - ljp_corrected = r.isLjpCorrected if hasattr(r, "isLjpCorrected") else None - stochastic = r.stochastic if hasattr(r, "stochastic") else None - - parameters = {} - ion_currents = [] - non_specific_currents = [] - ionic_concentrations = [] - if hasattr(r, "exposesParameter"): - exposes_parameters = r.exposesParameter - if not isinstance(exposes_parameters, list): - exposes_parameters = [exposes_parameters] - for ep in exposes_parameters: - if ep.type == "ConductanceDensity": - lower_limit = ep.lowerLimit if hasattr(ep, "lowerLimit") else None - upper_limit = ep.upperLimit if hasattr(ep, "upperLimit") else None - # resource name is the mech SUFFIX - parameters[f"{ep.name}_{r.name}"] = [lower_limit, upper_limit] - elif ep.type == "CurrentDensity": - if not hasattr(ep, "ionName"): - logger.warning( - "Will not add %s current, " - "because 'ionName' field was not found in %s.", - ep.name, - r.name, - ) - elif ep.ionName == "non-specific": - non_specific_currents.append(ep.name) - else: - ion_currents.append(ep.name) - ionic_concentrations.append(f"{ep.ionName}i") - elif ep.type == "IonConcentration": - ionic_concentrations.append(ep.name) - - # remove duplicates - ionic_concentrations = list(set(ionic_concentrations)) - - mech = MechanismConfiguration( - r.name, - location=None, - stochastic=stochastic, - version=version, - temperature=temperature, - ljp_corrected=ljp_corrected, - parameters=parameters, - ion_currents=ion_currents, - ionic_concentrations=ionic_concentrations, - id=r.id, - ) - - available_mechanisms.append(mech) - - return available_mechanisms - - def get_available_traces(self, filter_species=True, filter_brain_region=False): - """Get the list of available Traces for the current species from Nexus""" - - species = None - if filter_species: - species = self.emodel_metadata_ontology.species - brain_region = None - if filter_brain_region: - brain_region = self.emodel_metadata_ontology.brain_region - - resource_traces = get_available_traces( - species=species, - brain_region=brain_region, - access_token=self.access_point.access_token, - forge_path=self.access_point.forge_path, - ) - - traces = [] - if resource_traces is None: - return traces - - for r in resource_traces: - ecodes = None - if hasattr(r, "stimulus"): - # is stimulus a list - stimuli = r.stimulus if isinstance(r.stimulus, list) else [r.stimulus] - ecodes = { - stim.stimulusType.label: {} - for stim in stimuli - if hasattr(stim.stimulusType, "label") - } - species = None - if hasattr(r, "subject") and hasattr(r.subject, "species"): - species = r.subject.species - - brain_region = None - if hasattr(r, "brainLocation"): - brain_region = r.brainLocation - - etype = None - if hasattr(r, "annotation"): - if isinstance(r.annotation, Resource): - if "e-type" in r.annotation.name.lower(): - etype = r.annotation.hasBody.label - else: - for annotation in r.annotation: - if "e-type" in annotation.name.lower(): - etype = annotation.hasBody.label - - traces.append( - TraceFile( - r.name, - filename=None, - filepath=None, - resource_id=r.id, - ecodes=ecodes, - other_metadata=None, - species=species, - brain_region=brain_region, - etype=etype, - id=r.id, - ) - ) - return traces - - def store_morphology(self, morphology_name, morphology_path, mtype=None, reconstructed=True): - payload = { - "type": [ - "NeuronMorphology", - "Dataset", - ( - "ReconstructedNeuronMorphology" - if reconstructed - else "PlaceholderNeuronMorphology" - ), - ], - "name": pathlib.Path(morphology_path).stem, - "objectOfStudy": { - "@id": "http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells", - "label": "Single Cell", - }, - } - - if mtype: - payload["annotation"] = ( - { - "@type": ["Annotation", "nsg:MTypeAnnotation"], - "hasBody": { - "@id": "nsg:InhibitoryNeuron", - "@type": ["Mtype", "AnnotationBody"], - "label": mtype, - "prefLabel": mtype, - }, - "name": "M-type Annotation", - }, - ) - - self.access_point.register( - resource_description=payload, - distributions=[morphology_path], - type_="NeuronMorphology", - ) - - def store_hocs( - self, - only_validated=False, - only_best=True, - seeds=None, - map_function=map, - new_emodel_name=None, - description=None, - output_base_dir="export_emodels_hoc", - ): - """Store hoc files on nexus. - - Args: - export_function (function): can be export_emodels_hoc or export_emodels_sonata - """ - workflow, workflow_id = self.get_emodel_workflow() - if workflow is None: - raise AccessPointException( - "No EModelWorkflow available to which the EModelScripts can be linked" - ) - - emodels = self.get_emodels() - emodels = select_emodels( - self.emodel_metadata.emodel, - emodels, - only_validated=only_validated, - only_best=only_best, - seeds=seeds, - iteration=self.emodel_metadata.iteration, - ) - - if not emodels: - logger.warning( - "No emodels selected in store_hocs. No hoc file will be registered on nexus." - ) - - hold_key = "SearchHoldingCurrent.soma.v.bpo_holding_current" - thres_key = "SearchThresholdCurrent.soma.v.bpo_threshold_current" - - # maybe use map here? - for emodel in emodels: - if new_emodel_name is not None: - emodel.emodel_metadata.emodel = new_emodel_name - - # in case hocs have been created with the local output path - copy_hocs_to_new_output_path(emodel, output_base_dir) - - output_path = get_output_path( - emodel, output_base_dir=output_base_dir, use_allen_notation=True - ) - - hoc_file_path = pathlib.Path(get_hoc_file_path(output_path)).resolve() - if not hoc_file_path.is_file(): - logger.warning( - "Could not find the hoc file for %s. " - "Will not register EModelScript on nexus.", - emodel.emodel_metadata.emodel, - ) - continue - - currents = None - if emodel.features is not None: - if hold_key in emodel.features and thres_key in emodel.features: - currents = { - "holding": emodel.features[hold_key], - "threshold": emodel.features[thres_key], - } - - emodelscript = EModelScript(str(hoc_file_path), emodel.seed, workflow_id) - self.store_object( - emodelscript, - seed=emodel.seed, - description=description, - currents=currents, - ) - # wait for the object to be uploaded and fetchable - time.sleep(self.sleep_time) - - # fetch just uploaded emodelscript resource to get its id - type_ = "EModelScript" - filters = {"type": type_, "seed": emodel.seed} - filters.update(self.emodel_metadata_ontology.filters_for_resource()) - filters_legacy = {"type": type_, "seed": emodel.seed} - filters_legacy.update(self.emodel_metadata_ontology.filters_for_resource_legacy()) - resource = self.access_point.fetch_one(filters, filters_legacy, strict=True) - modelscript_id = resource.id - workflow.add_emodel_script_id(modelscript_id) - - time.sleep(self.sleep_time) - self.store_or_update_emodel_workflow(workflow) - - def store_emodels_hoc( - self, - only_validated=False, - only_best=True, - seeds=None, - map_function=map, - new_emodel_name=None, - description=None, - ): - self.store_hocs( - only_validated, - only_best, - seeds, - map_function, - new_emodel_name, - description, - "export_emodels_hoc", - ) - - def store_emodels_sonata( - self, - only_validated=False, - only_best=True, - seeds=None, - map_function=map, - new_emodel_name=None, - description=None, - ): - self.store_hocs( - only_validated, - only_best, - seeds, - map_function, - new_emodel_name, - description, - "export_emodels_sonata", - ) - - def get_hoc(self, seed=None): - """Get the EModelScript resource""" - metadata = self.emodel_metadata_ontology.filters_for_resource() - legacy_metadata = self.emodel_metadata_ontology.filters_for_resource_legacy() - if seed is not None: - metadata["seed"] = int(seed) - legacy_metadata["seed"] = int(seed) - - hoc = self.access_point.nexus_to_object( - type_="EModelScript", - metadata=metadata, - download_directory=self.download_directory, - legacy_metadata=legacy_metadata, - ) - return hoc - - def sonata_exists(self, seed): - """Returns True if the sonata hoc file has been exported""" - try: - _ = self.get_hoc(seed) - return True - except AccessPointException: - return False diff --git a/bluepyemodel/emodel_pipeline/emodel_pipeline.py b/bluepyemodel/emodel_pipeline/emodel_pipeline.py index 4538065..e9ed741 100644 --- a/bluepyemodel/emodel_pipeline/emodel_pipeline.py +++ b/bluepyemodel/emodel_pipeline/emodel_pipeline.py @@ -22,7 +22,6 @@ import warnings from bluepyemodel.access_point import get_access_point -from bluepyemodel.access_point.forge_access_point import DEFAULT_NEXUS_ENDPOINT from bluepyemodel.efeatures_extraction.efeatures_extraction import extract_save_features_protocols from bluepyemodel.emodel_pipeline import plotting from bluepyemodel.export_emodel.export_emodel import export_emodels_sonata @@ -140,13 +139,6 @@ def __init__( else: self.mapper = map - if nexus_endpoint == "prod": - endpoint = DEFAULT_NEXUS_ENDPOINT - elif nexus_endpoint == "staging": - endpoint = "https://staging.openbluebrain.com/api/nexus/v1" - else: - endpoint = nexus_endpoint - self.access_point = get_access_point( emodel=emodel, etype=etype, @@ -162,7 +154,7 @@ def __init__( final_path="final.json", organisation=nexus_organisation, project=nexus_project, - endpoint=endpoint, + endpoint=nexus_endpoint, access_point=data_access_point, forge_path=forge_path, forge_ontology_path=forge_ontology_path, @@ -313,99 +305,6 @@ def summarize(self): print(self.access_point) -class EModel_pipeline_nexus(EModel_pipeline): - """The EModel_pipeline_nexus class is there to allow the execution of the steps - of the e-model building pipeline for Nexus using python (as opposed to the Luigi workflow). - This class is deprecated and maintained for legacy purposes. - """ - - def __init__( - self, - emodel, - etype=None, - ttype=None, - mtype=None, - species=None, - brain_region=None, - iteration_tag=None, - morph_class=None, - synapse_class=None, - layer=None, - forge_path=None, - forge_ontology_path=None, - nexus_organisation=None, - nexus_project=None, - nexus_endpoint="staging", - use_ipyparallel=None, - use_multiprocessing=None, - ): - """Initializes the Nexus EModel_pipeline. - - Args: - emodel (str): name of the emodel. - etype (str): name of the e-type of the e-model. Used as an identifier for the e-model. - ttype (str): name of the t-type of the e-model. Used as an identifier for the e-model. - This argument is required when using the gene expression or IC selector. - mtype (str): name of the m-type of the e-model. Used as an identifier for the e-model. - species (str): name of the species of the e-model. Used as an identifier for the - e-model. - brain_region (str): name of the brain region of the e-model. Used as an identifier for - the e-model. - iteration_tag (str): tag associated to the current run. Used as an identifier for the - e-model. - morph_class (str): name of the morphology class, has to be "PYR", "INT". To be - depracted. - synapse_class (str): name of the synapse class of the e-model, has to be "EXC", "INH". - Not used at the moment. - layer (str): layer of the e-model. To be depracted. - forge_path (str): path to the .yml used to connect to Nexus Forge. This is only needed - if you wish to customize the connection to Nexus. If not provided, - a default .yml file will be used. - forge_ontology_path (str): path to the .yml used for the ontology in Nexus Forge - if not provided, forge_path will be used. - nexus_organisation (str): name of the Nexus organisation in which the project is - located. - nexus_project (str): name of the Nexus project to which the forge will connect to - retrieve the data. - nexus_endpoint (str): Nexus endpoint address, e.g., ``https://bbp.epfl.ch/nexus/v1``. - use_ipyparallel (bool): should the parallelization map used for the different steps of - the e-model building pipeline be based on ipyparallel. - use_multiprocessing (bool): should the parallelization map used for the different steps - of the e-model building pipeline be based on multiprocessing. - """ - - # pylint: disable=too-many-arguments - - warnings.warn( - "EModel_pipeline_nexus is deprecated." - "Please use EModel_pipeline with data_access_point='nexus' instead.", - DeprecationWarning, - stacklevel=2, - ) - - super().__init__( - emodel=emodel, - etype=etype, - ttype=ttype, - mtype=mtype, - species=species, - brain_region=brain_region, - iteration_tag=iteration_tag, - morph_class=morph_class, - synapse_class=synapse_class, - layer=layer, - recipes_path=None, - use_ipyparallel=use_ipyparallel, - use_multiprocessing=use_multiprocessing, - data_access_point="nexus", - nexus_endpoint=nexus_endpoint, - forge_path=forge_path, - forge_ontology_path=forge_ontology_path, - nexus_organisation=nexus_organisation, - nexus_project=nexus_project, - ) - - def sanitize_gitignore(): """In order to avoid git issue when archiving the current working directory, adds the following lines to .gitignore: 'run/', 'checkpoints/', 'figures/', diff --git a/bluepyemodel/evaluation/evaluation.py b/bluepyemodel/evaluation/evaluation.py index aad0419..50f8a02 100644 --- a/bluepyemodel/evaluation/evaluation.py +++ b/bluepyemodel/evaluation/evaluation.py @@ -314,7 +314,7 @@ def get_evaluator_from_access_point( start_from_emodel = access_point.pipeline_settings.start_from_emodel if start_from_emodel is not None: - access_point_type = "local" if isinstance(access_point, LocalAccessPoint) else "nexus" + access_point_type = "local" if isinstance(access_point, LocalAccessPoint) else None seed = start_from_emodel.pop("seed", None) @@ -325,7 +325,7 @@ def get_evaluator_from_access_point( "recipes_path": access_point.recipes_path, } else: - raise NotImplementedError("start_from_emodel not implemented for Nexus access point") + raise NotImplementedError("start_from_emodel only implemented for nexus access point") starting_access_point = get_access_point( access_point=access_point_type, **start_from_emodel, **kwargs diff --git a/bluepyemodel/export_emodel/export_emodel.py b/bluepyemodel/export_emodel/export_emodel.py index 9d8084d..10f6252 100644 --- a/bluepyemodel/export_emodel/export_emodel.py +++ b/bluepyemodel/export_emodel/export_emodel.py @@ -20,11 +20,9 @@ import os import pathlib import shutil -import time import h5py -from bluepyemodel.access_point.forge_access_point import DEFAULT_NEXUS_ENDPOINT from bluepyemodel.evaluation.evaluation import compute_responses from bluepyemodel.evaluation.evaluation import get_evaluator_from_access_point from bluepyemodel.export_emodel.utils import get_hoc_file_path @@ -246,141 +244,3 @@ def export_emodels_hoc( if not cell_model.morphology.morph_modifiers: # Turn [] into None cell_model.morphology.morph_modifiers = None _export_emodel_hoc(cell_model, mo, output_dir=None, new_emodel_name=new_emodel_name) - - -def export_emodels_nexus( - local_access_point, - nexus_organisation, - nexus_project, - nexus_endpoint=DEFAULT_NEXUS_ENDPOINT, - forge_path=None, - forge_ontology_path=None, - access_token=None, - only_validated=False, - only_best=True, - seeds=None, - description=None, - sleep_time=10, - sonata=True, -): - """Transfer e-models from the LocalAccessPoint to a Nexus project - - Args: - local_access_point (LocalAccessPoint): The local access point containing the e-models. - nexus_organisation (str): The Nexus organisation to which the e-models will be transferred. - nexus_project (str): The Nexus project to which the e-models will be transferred. - nexus_endpoint (str, optional): The Nexus endpoint. - Defaults to DEFAULT_NEXUS_ENDPOINT. - forge_path (str, optional): The path to the forge. - forge_ontology_path (str, optional): The path to the forge ontology. - access_token (str, optional): The access token for Nexus. - only_validated (bool, optional): If True, only validated e-models will be transferred. - only_best (bool, optional): If True, only the best e-models will be transferred. - seeds (list, optional): The chosen seeds to export. - description (str, optional): Optional description to add to the resources in Nexus. - sleep_time (int, optional): time to wait between two Nexus requests - (in case of slow indexing). - sonata (bool, optional): Determines the format for registering e-models. - If True (default), uses Sonata hoc format. Otherwise, uses NEURON hoc format. - - Returns: - None - """ - - from bluepyemodel.access_point.nexus import NexusAccessPoint - - emodels = local_access_point.get_emodels() - emodels = select_emodels( - local_access_point.emodel_metadata.emodel, - emodels, - only_validated=only_validated, - only_best=only_best, - seeds=seeds, - ) - if not emodels: - return - - metadata = vars(local_access_point.emodel_metadata) - iteration = metadata.pop("iteration") - metadata.pop("allen_notation") - nexus_access_point = NexusAccessPoint( - **metadata, - iteration_tag=iteration, - project=nexus_project, - organisation=nexus_organisation, - endpoint=nexus_endpoint, - access_token=access_token, - forge_path=forge_path, - forge_ontology_path=forge_ontology_path, - sleep_time=sleep_time, - ) - - pipeline_settings = local_access_point.pipeline_settings - fitness_configuration = local_access_point.get_fitness_calculator_configuration() - model_configuration = local_access_point.get_model_configuration() - targets_configuration = local_access_point.get_targets_configuration() - - # Register the resources - logger.info("Exporting the emodel %s to Nexus...", local_access_point.emodel_metadata.emodel) - logger.info("Registering EModelPipelineSettings...") - nexus_access_point.store_pipeline_settings(pipeline_settings) - - logger.info("Registering ExtractionTargetsConfiguration...") - # Set local filepath to None to avoid discrepancies between local and Nexus paths - for file in targets_configuration.files: - file.filepath = None - nexus_access_point.store_targets_configuration(targets_configuration) - - logger.info("Registering EModelConfiguration...") - # Remove unused local data from the model configuration before uploading to Nexus - model_configuration.morphology.path = None - nexus_access_point.store_model_configuration(model_configuration) - - logger.info("Registering EModelWorkflow...") - filters = {"type": "EModelWorkflow", "eModel": metadata["emodel"], "iteration": iteration} - filters_legacy = { - "type": "EModelWorkflow", - "emodel": metadata["emodel"], - "iteration": iteration, - } - nexus_access_point.access_point.deprecate(filters, filters_legacy) - time.sleep(sleep_time) - emw = nexus_access_point.create_emodel_workflow(state="done") - nexus_access_point.store_or_update_emodel_workflow(emw) - - logger.info("Registering FitnessCalculatorConfiguration...") - time.sleep(sleep_time) - nexus_access_point.store_fitness_calculator_configuration(fitness_configuration) - - for mo in emodels: - time.sleep(sleep_time) - mo.emodel_metadata.allen_notation = nexus_access_point.emodel_metadata.allen_notation - mo.copy_pdf_dependencies_to_new_path(seed=mo.seed) - logger.info("Registering EModel %s...", mo.emodel_metadata.emodel) - nexus_access_point.store_emodel(mo, description=description) - - time.sleep(sleep_time) - if sonata: - logger.info( - "Registering EModelScript (in sonata hoc format with threshold_current and " - "holding_current in node.h5 file) for circuit building using neurodamus..." - ) - nexus_access_point.store_emodels_sonata( - only_best=only_best, - only_validated=only_validated, - seeds=seeds, - description=description, - ) - else: - logger.info("Registering EModelScript (in hoc format to run e-model using NEURON)...") - nexus_access_point.store_emodels_hoc( - only_best=only_best, - only_validated=only_validated, - seeds=seeds, - description=description, - ) - - logger.info( - "Exporting the emodel %s to Nexus done.", - local_access_point.emodel_metadata.emodel, - ) diff --git a/bluepyemodel/export_emodel/utils.py b/bluepyemodel/export_emodel/utils.py index 4703a41..95f68db 100644 --- a/bluepyemodel/export_emodel/utils.py +++ b/bluepyemodel/export_emodel/utils.py @@ -104,7 +104,7 @@ def select_emodels( iteration=None, ): if not emodels: - logger.warning("In export_emodels_nexus, no emodel for %s", emodel_name) + logger.warning("In select_emodels, no emodel for %s", emodel_name) return [] if iteration: @@ -117,7 +117,7 @@ def select_emodels( emodels = [e for e in emodels if e.seed in seeds] if not emodels: logger.warning( - "In export_emodels_nexus, no emodel for %s and seeds %s", + "In select_emodels, no emodel for %s and seeds %s", emodel_name, seeds, ) @@ -127,7 +127,7 @@ def select_emodels( emodels = [e for e in emodels if e.passed_validation] if not emodels: logger.warning( - "In export_emodels_nexus, no emodel for %s that passed validation", + "In select_emodels, no emodel for %s that passed validation", emodel_name, ) return [] diff --git a/bluepyemodel/tasks/config.py b/bluepyemodel/tasks/config.py index a1a5768..7df2b0e 100644 --- a/bluepyemodel/tasks/config.py +++ b/bluepyemodel/tasks/config.py @@ -18,8 +18,6 @@ import luigi -from bluepyemodel.access_point.forge_access_point import DEFAULT_NEXUS_ENDPOINT - class EmodelAPIConfig(luigi.Config): """Configuration of emodel api database.""" @@ -32,13 +30,6 @@ class EmodelAPIConfig(luigi.Config): final_path = luigi.OptionalParameter(default="./final.json") legacy_dir_structure = luigi.BoolParameter(default=False) - # nexus parameters - forge_path = luigi.OptionalParameter(default=None) - forge_ontology_path = luigi.OptionalParameter(default=None) - nexus_project = luigi.OptionalParameter(default="emodel_pipeline") - nexus_organisation = luigi.OptionalParameter(default="demo") - nexus_endpoint = luigi.OptionalParameter(default=DEFAULT_NEXUS_ENDPOINT) - def __init__(self, *args, **kwargs): """Init.""" super().__init__(*args, **kwargs) @@ -53,17 +44,3 @@ def __init__(self, *args, **kwargs): "final_path": self.final_path, "legacy_dir_structure": self.legacy_dir_structure, } - - if self.api == "nexus": - if self.forge_path is None: - raise ValueError("forge_path cannot be None when api is set to 'nexus'") - if self.forge_ontology_path is None: - raise ValueError("forge_ontology_path cannot be None when api is set to 'nexus'") - - self.api_args = { - "forge_path": self.forge_path, - "forge_ontology_path": self.forge_ontology_path, - "project": self.nexus_project, - "organisation": self.nexus_organisation, - "endpoint": self.nexus_endpoint, - } diff --git a/bluepyemodel/tasks/emodel_creation/optimisation.py b/bluepyemodel/tasks/emodel_creation/optimisation.py index d71a4cc..686b05c 100644 --- a/bluepyemodel/tasks/emodel_creation/optimisation.py +++ b/bluepyemodel/tasks/emodel_creation/optimisation.py @@ -24,7 +24,6 @@ import luigi from bluepyemodel.access_point.access_point import OptimisationState -from bluepyemodel.access_point.nexus import NexusAccessPoint from bluepyemodel.efeatures_extraction.efeatures_extraction import extract_save_features_protocols from bluepyemodel.efeatures_extraction.targets_configurator import TargetsConfigurator from bluepyemodel.emodel_pipeline.plotting import optimisation @@ -840,13 +839,6 @@ def remote_script(self): seeds=list(range(args.seed, args.seed + args.batch_size)), map_function=mapper, ) - if args.api_from_config == "nexus": - access_pt.store_emodels_sonata( - only_validated=False, - only_best=False, - seeds=list(range(args.seed, args.seed + args.batch_size)), - map_function=mapper, - ) def output(self): """ """ @@ -931,8 +923,6 @@ def check_mettypes(func): def inner(self): """Inner decorator function""" - if EmodelAPIConfig().api == "nexus": - self.access_point.check_mettypes() # do this instead of just func(self) because of the yield in EModelCreation yield from func(self) @@ -1285,10 +1275,6 @@ def run(self): sinespec_settings=sinespec_settings, ) - if isinstance(self.access_point, NexusAccessPoint): - for seed in range(self.seed, self.seed + batch_size): - self.access_point.update_emodel_images(seed=seed, keep_old_images=False) - def output(self): """ """ @@ -1383,11 +1369,6 @@ def run(self): only_validated=True, ) - if isinstance(self.access_point, NexusAccessPoint): - seeds = [emodel.seed for emodel in self.access_point.get_emodels()] - for seed in seeds: - self.access_point.update_emodel_images(seed=seed, keep_old_images=False) - def output(self): """ """ diff --git a/bluepyemodel/tasks/luigi_tools.py b/bluepyemodel/tasks/luigi_tools.py index ffbe677..2a6af71 100644 --- a/bluepyemodel/tasks/luigi_tools.py +++ b/bluepyemodel/tasks/luigi_tools.py @@ -76,8 +76,6 @@ def check_mettypes(func): def inner(self): """Inner decorator function""" - if EmodelAPIConfig().api == "nexus": - self.access_point.check_mettypes() func(self) return inner diff --git a/examples/nexus/README.md b/examples/nexus/README.md deleted file mode 100644 index d1d080b..0000000 --- a/examples/nexus/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# E-Model Building Pipeline with Luigi, Nexus and bbp-workflow - -This example demonstrates how to build an e-model using the Nexus access, [Luigi](https://luigi.readthedocs.io/en/stable/) and [bbp-workflow](https://github.com/BlueBrain/bbp-workflow), which automates batch job pipelines. The [NexusAccessPoint](./../../bluepyemodel/access_point/nexus.py) class serves as the API that enables users to store, manage, and use e-model resources on Nexus. This class provides a set of functions to create e-model-related resources on Nexus and link them to the electrophysiological data, mechanisms, and morphology registered on Nexus. - -## 1. Setup the virtual environment -To setup the virtual environment and install all the needed packages, run: - -``` -kinit -./create_venv.sh -``` - -Note: Ensure that your Python version is 3.10 or higher, as bbp-workflow is only compatible with Python >= 3.10. - -Then activate the virtual environment: - -``` -source myvenv/bin/activate -``` - -## 2. Configure the Luigi pipeline: -The ``luigi.cfg`` file contains the configuration for the Luigi pipeline. Ensure you set ``virtual-env`` to the path of your virtual environment (created in the previous step) and ``chdir`` the absolute path of the current working directory. Other parameters can be adjusted based on your requirements. Below is a description of several variables you may wish to customise: - -- [DEFAULT] : settings section used by bbp-workflow. - - ``account``: project name (e.g. proj72). - - ``virtual-env``: path to your virtual environment. - - ``chdir``: absolute path of the current working directory. - - ``workers``: number of workers to be used by the pipeline. Currently, only 1 worker is supported. - - ``time``: maximum time allowed for the pipeline to run. -- [Optimise] : settings section used for the optimisation. - - ``node``: number of nodes to be used on the HPC. - - ``time``: maximum time allowed to run the optimisation. -- [parallel] - - ``backend``: select the backend for processing, which can be either 'ipyparallel' or 'multiprocessing'. If left unspecified, no parallelisation will be used. -- [EmodelAPIConfig] - - ``nexus_project``: a valid Nexus project name to which the emodel should be uploaded. - -For a detailed description of the configuration file, please refer to the [Luigi documentation](https://luigi.readthedocs.io/en/stable/configuration.html). - -## 3. Register the resources that will be used by the pipeline on the Nexus project - -Prior to executing the Luigi pipeline, it is essential to register the following resources within the Nexus project, as they will be needed by the pipeline: - -- ``EModelPipelineSettings`` (EMPS): the pipeline settings of the e-model. -- ``ExtractionTargetsConfiguration`` (ETC): The extraction target configuration of the e-model and the links to the ephys data. This resource is created by parsing the ``targets.py`` using the ``configure_targets`` function in ``pipeline.py``. -- ``EModelConfiguration`` (EMC): the configuration of the e-model, which links to the morphology and mechanisms and stores a reformatted version of the parameters file of the e-model. - - -To create and register the ``EModelPipelineSettings`` (EMPS) and ``ExtractionTargetsConfiguration`` (ETC), run: - -``` -python pipeline.py --step=configure_nexus --emodel=EMODEL_NAME --iteration_tag=ITERATION_TAG --etype=ETYPE --mtype=MTYPE --ttype=TTYPE -``` - -The ``EMODEL_NAME`` can be any string without special characters, except for underscores, e.g., "L5_TPC_B_cAC". - -The ``ETYPE``, ``MTYPE`` and ``TTYPE`` have to be valid names present in the gene map (e.g., ETYPE=="cAC", MTYPE="L3_TPC:C", TTYPE="0103 L2/3 IT CTX Glut_1"). - -The iteration_tag can be any string without spaces. This variable enables the user to run different tests or iterations in parallel on the same e-model. All Nexus resources related to BluePyEModel will be tagged with the iteration_tag, and the pipeline will only use resources with a matching tag. It is important to ensure that the ``iteration_tag`` specified in luigi.cfg matches the ``iteration_tag`` used when running pipeline.py. If a different ``iteration_tag`` is used in pipeline.py, the pipeline will crash because BluePyEModel will not find the expected resources. - -Additionally, please note that the ephys trace files and the targets (e-features and protocols) used in this example are hardcoded in the targets.py file. - -The user will be prompted to confirm their intent to deprecate the resources currently present on Nexus that have the same iteration_tag and emodel name. Following this, they will be required to enter their Nexus token, which can be obtained from https://bbp.epfl.ch/nexus/web/. -![](./img_nexus_token.png) - -To set up the EModelConfiguration (EMC), which includes the model's channels, parameters and parameter distributions. You can either create the configuration based on gene data, or through a legacy json file. You can specify the legacy JSON file by adjusting the path of ``legacy_conf_json_file`` in ``pipeline.py`` and then run: - -``` -python pipeline.py --step=configure_model_from_json --emodel=EMODEL_NAME --iteration_tag=ITERATION_TAG --etype=ETYPE --mtype=MTYPE --ttype=TTYPE -``` - -If you choose to create the configuration based on gene data, you will need to provide a ttype, thus, ensure that you have also specified the ``TTYPE`` when running the ``configure_nexus`` step: - -``` -python pipeline.py --step=configure_model_from_gene --emodel=EMODEL_NAME --iteration_tag=ITERATION_TAG --etype=ETYPE --mtype=MTYPE --ttype=TTYPE -``` - -If you wish to modify the gene-based configuration before proceeding with model optimisation, you can get the configuration from Nexus and modify it before proceeding further. The jupyter notebook [edit_neuron_model_configuration.ipynb](../ncmv3//edit_neuron_model_configuration.ipynb) explains how to do so. - -An example of how to run the `configure_nexus` and `configure_model_from_gene` steps is provided in the [run_pipeline.ipynb](./run_pipeline.ipynb) notebook. - -## 4. Run the Luigi pipeline: -Set the variables ``emodel``, ``etype``, ``iteration``, ``mtype``, and ``ttype`` in ``launch_luigi.sh`` to match the values you chose for ``EMODEL_NAME``, ``ETYPE``, ``ITERATION_TAG``, ``MTYPE``, and ``TTYPE``. Additionally, ensure that the ``species`` and ``brain_region`` variables are consistent with the values in ``pipeline.py``. - - -Then, execute: - -``` -source myvenv/bin/activate -kinit -./launch_luigi.sh -``` - -The pipeline should start, and after a few minutes, you will see the message: "No more log messages received, connection timed out." To monitor the progress of the pipeline, you can use a web app by first executing the following command: - -``` -bbp-workflow webui -o -``` - -Then, open the URL returned by this command in your browser. - -If an error happens during the execution of the workflow, the command ``./launch_luigi.sh`` can be run again and the workflow will restart from the latest step. If the error persists, please refer to the [Troubleshooting](#troubleshooting) section or contact the package's authors for further assistance. - -## 5. Results: - -Once the Luigi pipeline has run successfully, the following resources will be saved in the Nexus project specified by the ``nexus_project`` variable in pipeline.py, along with the hoc file of the e-model: - -- ``FitnessCalculatorConfiguration`` (FCC): the fitness calculator configuration of the e-model, which stores the features and protocols file of the e-model. -- ``EmodelScript`` (ES): the hoc file of the e-model. -- ``EModelWorkflow`` (EMW): the resource to which all the above resources are linked to, including the workflow state. -- ``EModel`` (EM): all the information related to an optimised e-model. It contains the final parameters of the e-model from final.json, and pdfs of the e-model distribution plots, features scores and e-model response traces. It also links to EModelWorflow. - -In conclusion, here is the graph structure that will be generated on Nexus upon completing the entire pipeline: - -``` - EModelWorkflow - | - ├──> EModelPipelineSettings - | - ├──> ExtractionTargetsConfiguration - | | - | ├──> Trace1 - | ├──> ... - | └──> TraceN - | - ├──> EModelConfiguration - | | - | ├──> Mechanism1 - | ├──> ... - | └──> MechanismN - | └──> Morphology - | - ├──> FitnessCalculatorConfiguration - | - ├──> EModel - | - └──> EModelScript -``` - -You can also check the graph structure of the resources created on [Nexus](https://bbp.epfl.ch/nexus/web/). - -To check that the models were optimised successfully, you can refer to the figures created in ``./figures/EMODEL_NAME/``. -The `optimisation` subfolder contains plots of the fitness versus number of generations run by the optimiser, while the `traces` and `scores` subfolders contain the voltage traces and efeatures scores for the e-models. - -In addition, we provide a set of notebooks to help users understand how to edit some of the resources created by the pipeline. The first, [edit_fitness_calculator_configuration.ipynb](./edit_fitness_calculator_configuration.ipynb) explains how to edit an existing neuron model configuration on Nexus in order to add a new e-feature or protocol. The second notebook, [edit_neuron_model_configuration.ipynb](./edit_neuron_model_configuration.ipynb), explains how to load an already existing neuron model configuration from Nexus, edit it, and store it back on Nexus. A third notebook [exploit_model.ipynb](./exploit_model.ipynb) explains how to run other protocols on a model or investigate the final model yourself. - - -## Troubleshooting -Refer to the [Troubleshooting of BluePyEModel](https://github.com/BlueBrain/BluePyEModel/tree/main/examples/L5PC#troubleshooting) for common issues. - - -### When running launch_luigi.sh, if you see the error: - -``` - RAN: /usr/bin/ssh -o StrictHostKeyChecking=no -o ExitOnForwardFailure=yes -o StreamLocalBindUnlink=yes -NT -R/tmp/2609079-sch.sock:localhost:8082 -R/tmp/2609079-agt.sock:/tmp/ssh-IlIhsVxNsOVr/agent.17 r4i3n27 - - STDOUT: - - STDERR: -``` - -Solution: run the command "kinit" and provide your password. - -### There is a failed task in the Luigi webapp: - -Click on the red little "bug" button next to the task, it will display the log. - -If the last line of the log reads something like: "failed to find resource for filter {...}", there are two possible causes: -1. You launched the pipeline with an iteration_tag different from the one you specified when configuring the pipeline. If that's the case, edit your luigi.cfg and inform the correct iteration_tag. -2. It happens from time to time that nexus forge fails to get a result even when a matching resource exists. If that's the case, launch the pipeline again, it will restart from where it stopped. \ No newline at end of file diff --git a/examples/nexus/create_venv.sh b/examples/nexus/create_venv.sh deleted file mode 100755 index 2154fd3..0000000 --- a/examples/nexus/create_venv.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -##################################################################### -# Copyright 2024 Blue Brain Project / EPFL - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -##################################################################### - -deactivate -module purge all - -module load unstable python -python -m venv myvenv -module purge all -source myvenv/bin/activate - -pip install bbp-workflow -pip install bluepyemodel[all] - -deactivate diff --git a/examples/nexus/edit_fitness_calculator_configuration.ipynb b/examples/nexus/edit_fitness_calculator_configuration.ipynb deleted file mode 100644 index 9042ef3..0000000 --- a/examples/nexus/edit_fitness_calculator_configuration.ipynb +++ /dev/null @@ -1,402 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Editing the FitnessCalculatorConfiguration: A Step-by-Step Guide" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To run this notebook, you need to have an existing FitnessCalculatorConfiguration on Nexus." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Please fill in the values to match the metadata of the FitnessCalculatorConfiguration you want to edit." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "emodel = \"L5_TPC\"\n", - "etype = \"cAC\"\n", - "mtype = None\n", - "ttype = None\n", - "iteration_tag = \"XXXX-XX-XX\"\n", - "species = \"mouse\"\n", - "brain_region = \"SSCX\"\n", - "\n", - "nexus_project = \"\" # specify the nexus project to use\n", - "nexus_organisation = \"\" # specify the nexus organisation to use\n", - "nexus_endpoint = \"\" # specify the nexus endpoint to use\n", - "\n", - "# Advanced settings (only change if you know what you are doing)\n", - "forge_path = \"./forge.yml\"\n", - "forge_ontology_path = \"./nsg.yml\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This next cell will ask for your Nexus token:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.access_point.nexus import NexusAccessPoint\n", - "\n", - "data_access_point = \"nexus\"\n", - "access_point = NexusAccessPoint(\n", - " emodel=emodel,\n", - " species=species,\n", - " brain_region=brain_region,\n", - " project=nexus_project,\n", - " organisation=nexus_organisation,\n", - " endpoint=nexus_endpoint,\n", - " forge_path=forge_path,\n", - " forge_ontology_path=forge_ontology_path,\n", - " etype=etype,\n", - " mtype=mtype,\n", - " ttype=ttype,\n", - " iteration_tag=iteration_tag\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fitness Calculator Configuration:\n", - "\n", - "Protocols:\n", - " {'name': 'IDrest_130', 'stimuli': [{'delay': 700.5, 'amp': 0.4508745591995102, 'thresh_perc': 136.24160437248662, 'duration': 2000.25, 'totduration': 3000.0, 'holding_current': -0.09770833142101765}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IDrest_130.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IDrest_200', 'stimuli': [{'delay': 700.5, 'amp': 0.6552015231929614, 'thresh_perc': 197.9834632177735, 'duration': 2000.25, 'totduration': 3000.0, 'holding_current': -0.09765624813735485}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IDrest_200.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IV_0', 'stimuli': [{'delay': 20.0, 'amp': 0.00016210680283757512, 'thresh_perc': 0.048984114201291125, 'duration': 1000.0, 'totduration': 1320.0, 'holding_current': -0.09769097136126624}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IV_0.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IV_-100', 'stimuli': [{'delay': 20.0, 'amp': -0.3150840740753216, 'thresh_perc': -95.2095408542356, 'duration': 1000.0, 'totduration': 1320.0, 'holding_current': -0.097534721924199}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IV_-100.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'APWaveform_280', 'stimuli': [{'delay': 5.03, 'amp': 0.9239451116109707, 'thresh_perc': 279.1902133078517, 'duration': 51.24666666666667, 'totduration': 59.99, 'holding_current': -0.07718750089406967}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'APWaveform_280.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IDhyperpol_150', 'stimuli': [{'delay': 100.0, 'tmid': 700.0000000000001, 'tmid2': 2700.0, 'toff': 2900.0, 'amp': 0.47654440222873184, 'long_amp': -0.22305531470241, 'thresh_perc': 143.99830859749372, 'duration': 2000.0, 'totduration': 3000.0, 'holding_current': -0.1074999999254942}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IDhyperpol_150.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': True, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - "EFeatures:\n", - " {'efel_feature_name': 'Spikecount', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 8.666666666666666, 'original_std': 3.496029493900505, 'sample_size': 6, 'efeature_name': 'Spikecount', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_CV', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.3702316995087826, 'original_std': 0.32566048327925706, 'sample_size': 6, 'efeature_name': 'ISI_CV', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'doublet_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 151.83333333319524, 'original_std': 34.61624939562405, 'sample_size': 6, 'efeature_name': 'doublet_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'mean_frequency', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.70922378383736, 'original_std': 1.6760068225673201, 'sample_size': 6, 'efeature_name': 'mean_frequency', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_first_spike', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 33.458333332939674, 'original_std': 7.231897899039166, 'sample_size': 6, 'efeature_name': 'time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_log_slope', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -0.16581691171366217, 'original_std': 0.8515890058904552, 'sample_size': 6, 'efeature_name': 'ISI_log_slope', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -82.78261651725562, 'original_std': 0.657020244055356, 'sample_size': 6, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_last_spike', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1806.9583333380724, 'original_std': 116.31793863080907, 'sample_size': 6, 'efeature_name': 'time_to_last_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_time_to_first_spike', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 31.44753263082006, 'original_std': 7.29210290109057, 'sample_size': 6, 'efeature_name': 'inv_time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_first_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 6.925399554807271, 'original_std': 1.5071612674614459, 'sample_size': 6, 'efeature_name': 'inv_first_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_second_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.985717646424224, 'original_std': 2.104516309965638, 'sample_size': 6, 'efeature_name': 'inv_second_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_third_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.840173932427828, 'original_std': 1.2897829185036143, 'sample_size': 6, 'efeature_name': 'inv_third_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_last_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.740281701448489, 'original_std': 1.4623481605488782, 'sample_size': 6, 'efeature_name': 'inv_last_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_depth', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 18.961034660480593, 'original_std': 1.1849839363452002, 'sample_size': 6, 'efeature_name': 'AHP_depth', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_time_from_peak', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 25.620500090252204, 'original_std': 2.638148646675542, 'sample_size': 6, 'efeature_name': 'AHP_time_from_peak', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'min_AHP_values', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -63.821581856775026, 'original_std': 1.5391678614966964, 'sample_size': 6, 'efeature_name': 'min_AHP_values', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'depol_block_bool', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'depol_block_bool', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_after_stim', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -87.12239628840818, 'original_std': 0.9572409682749269, 'sample_size': 6, 'efeature_name': 'voltage_after_stim', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'burst_number', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.6666666666666666, 'original_std': 0.9428090415820634, 'sample_size': 6, 'efeature_name': 'burst_number', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'number_initial_spikes', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1.6666666666666667, 'original_std': 0.4714045207910317, 'sample_size': 6, 'efeature_name': 'number_initial_spikes', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'irregularity_index', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 213.95798611189414, 'original_std': 278.960340993408, 'sample_size': 6, 'efeature_name': 'irregularity_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'adaptation_index', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.021810381821812972, 'original_std': 0.025904931541217195, 'sample_size': 6, 'efeature_name': 'adaptation_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'burst_mean_freq', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 6.189144214407903, 'original_std': 0.14984960695170724, 'sample_size': 2, 'efeature_name': 'burst_mean_freq', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'Spikecount', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 19.833333333333332, 'original_std': 3.4840908267278117, 'sample_size': 6, 'efeature_name': 'Spikecount', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_CV', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.07463775460612596, 'original_std': 0.028505952255165983, 'sample_size': 6, 'efeature_name': 'ISI_CV', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'doublet_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 44.208333333293126, 'original_std': 9.628967436268557, 'sample_size': 6, 'efeature_name': 'doublet_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'mean_frequency', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 10.189637299220186, 'original_std': 1.778840871822643, 'sample_size': 6, 'efeature_name': 'mean_frequency', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_first_spike', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 16.58333333295502, 'original_std': 3.72677996249626, 'sample_size': 6, 'efeature_name': 'time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_log_slope', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.03345638299091273, 'original_std': 0.05974524879883885, 'sample_size': 6, 'efeature_name': 'ISI_log_slope', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -82.656065689877, 'original_std': 0.4123618296597106, 'sample_size': 6, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_last_spike', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1946.500000005247, 'original_std': 17.939713858801507, 'sample_size': 6, 'efeature_name': 'time_to_last_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_time_to_first_spike', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 63.138076544811895, 'original_std': 12.691389369204195, 'sample_size': 6, 'efeature_name': 'inv_time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_first_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 23.61128698555237, 'original_std': 4.5663583102585985, 'sample_size': 6, 'efeature_name': 'inv_first_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_second_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 10.888408719900829, 'original_std': 1.9500846038754984, 'sample_size': 6, 'efeature_name': 'inv_second_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_third_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 9.988777902840466, 'original_std': 2.0044673370648782, 'sample_size': 6, 'efeature_name': 'inv_third_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_last_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 9.955328682497806, 'original_std': 2.3570371576647995, 'sample_size': 6, 'efeature_name': 'inv_last_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_depth', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 21.875388331111182, 'original_std': 1.6438123287417166, 'sample_size': 6, 'efeature_name': 'AHP_depth', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_time_from_peak', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 20.00731497674777, 'original_std': 1.747628947979076, 'sample_size': 6, 'efeature_name': 'AHP_time_from_peak', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'min_AHP_values', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -60.78067735876579, 'original_std': 1.919410899483439, 'sample_size': 6, 'efeature_name': 'min_AHP_values', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'depol_block_bool', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'depol_block_bool', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_after_stim', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -86.90101193311828, 'original_std': 0.5605484695404097, 'sample_size': 6, 'efeature_name': 'voltage_after_stim', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'burst_number', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'burst_number', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'number_initial_spikes', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 3.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'number_initial_spikes', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'irregularity_index', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 6.7059774196184, 'original_std': 1.583840773838753, 'sample_size': 6, 'efeature_name': 'irregularity_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'adaptation_index', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0005470381772781952, 'original_std': 0.0037812147886006962, 'sample_size': 6, 'efeature_name': 'adaptation_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'steady_state_voltage_stimend', 'protocol_name': 'RMPProtocol', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.38956665960032, 'original_std': 0.44319427048989085, 'sample_size': 9, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ohmic_input_resistance_vb_ssse', 'protocol_name': 'IV_0', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 40.381127264357104, 'original_std': 486.9230848650381, 'sample_size': 8, 'efeature_name': 'ohmic_input_resistance_vb_ssse', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'steady_state_voltage_stimend', 'protocol_name': 'SearchHoldingCurrent', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.1602795290849, 'original_std': 0.5732871465615677, 'sample_size': 9, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ohmic_input_resistance_vb_ssse', 'protocol_name': 'RinProtocol', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 46.144060948518096, 'original_std': 9.540253650631344, 'sample_size': 9, 'efeature_name': 'ohmic_input_resistance_vb_ssse', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IV_-100', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.10270958313072, 'original_std': 0.7408161829891378, 'sample_size': 9, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ohmic_input_resistance_vb_ssse', 'protocol_name': 'IV_-100', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 41.950232851373016, 'original_std': 7.7917698493471175, 'sample_size': 9, 'efeature_name': 'ohmic_input_resistance_vb_ssse', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AP_amplitude', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 85.63229179382726, 'original_std': 0.2739372661121643, 'sample_size': 3, 'efeature_name': 'AP_amplitude', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AP1_amp', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 91.24062410990176, 'original_std': 0.7323490865525268, 'sample_size': 3, 'efeature_name': 'AP1_amp', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AP_duration_half_width', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1.3916666666666269, 'original_std': 0.005892556509886723, 'sample_size': 3, 'efeature_name': 'AP_duration_half_width', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_depth', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 23.89942607879726, 'original_std': 0.4119030778088721, 'sample_size': 3, 'efeature_name': 'AHP_depth', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'mean_frequency', 'protocol_name': 'IDhyperpol_150', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.824314976680847, 'original_std': 0.8271130180600073, 'sample_size': 2, 'efeature_name': 'mean_frequency', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IDhyperpol_150', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.85849777183599, 'original_std': 0.5382516057115367, 'sample_size': 2, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'depol_block_bool', 'protocol_name': 'IDhyperpol_150', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1.0, 'original_std': 0.001, 'sample_size': 2, 'efeature_name': 'depol_block_bool', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'bpo_holding_current', 'protocol_name': 'SearchHoldingCurrent', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -0.09753230129913182, 'original_std': 0.039795330549849496, 'sample_size': None, 'efeature_name': None, 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True}}\n", - " {'efel_feature_name': 'bpo_threshold_current', 'protocol_name': 'SearchThresholdCurrent', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.33093749980131787, 'original_std': 0.06647122484201451, 'sample_size': None, 'efeature_name': None, 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True}}\n", - "\n" - ] - } - ], - "source": [ - "fitness_config = access_point.get_fitness_calculator_configuration()\n", - "\n", - "print(fitness_config)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.evaluation.efeature_configuration import EFeatureConfiguration\n", - "from bluepyemodel.evaluation.protocol_configuration import ProtocolConfiguration" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As an example, we add a new protocol called `Step_1000`" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "protocol_name = \"Step_1000\"\n", - "\n", - "stimuli = [{\n", - " \"amp\": None,\n", - " \"delay\": 700,\n", - " \"duration\": 4,\n", - " \"holding_current\": None,\n", - " \"thresh_perc\": 1000,\n", - " \"totduration\": 1000\n", - "}]\n", - "\n", - "recordings = [\n", - " {\n", - " \"type\": \"CompRecording\",\n", - " \"location\": \"soma\",\n", - " \"name\": f\"{protocol_name}.soma.v\",\n", - " \"variable\": \"v\"\n", - " },\n", - "]\n", - "\n", - "protocol_config = ProtocolConfiguration(\n", - " name=protocol_name,\n", - " stimuli=stimuli,\n", - " recordings=recordings,\n", - " validation=False\n", - ")\n", - "\n", - "fitness_config.protocols.append(protocol_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that the name of the protocol matters as BluePyEmodel will use it to detect which shape the protocol should have. For example, here, \"step\" is in the name and therefore a step protocol will be generated. A complete list of the eCode that can be used to create protocols can be seen [here](https://github.com/BlueBrain/BluePyEModel/blob/main/bluepyemodel/ecode/__init__.py). For more complex eCode, contact the package authors for an explanation of how to use them.§" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we add the feature associated with the newly defined protocol." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fitness Calculator Configuration:\n", - "\n", - "Protocols:\n", - " {'name': 'IDrest_130', 'stimuli': [{'delay': 700.5, 'amp': 0.4508745591995102, 'thresh_perc': 136.24160437248662, 'duration': 2000.25, 'totduration': 3000.0, 'holding_current': -0.09770833142101765}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IDrest_130.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IDrest_200', 'stimuli': [{'delay': 700.5, 'amp': 0.6552015231929614, 'thresh_perc': 197.9834632177735, 'duration': 2000.25, 'totduration': 3000.0, 'holding_current': -0.09765624813735485}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IDrest_200.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IV_0', 'stimuli': [{'delay': 20.0, 'amp': 0.00016210680283757512, 'thresh_perc': 0.048984114201291125, 'duration': 1000.0, 'totduration': 1320.0, 'holding_current': -0.09769097136126624}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IV_0.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IV_-100', 'stimuli': [{'delay': 20.0, 'amp': -0.3150840740753216, 'thresh_perc': -95.2095408542356, 'duration': 1000.0, 'totduration': 1320.0, 'holding_current': -0.097534721924199}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IV_-100.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'APWaveform_280', 'stimuli': [{'delay': 5.03, 'amp': 0.9239451116109707, 'thresh_perc': 279.1902133078517, 'duration': 51.24666666666667, 'totduration': 59.99, 'holding_current': -0.07718750089406967}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'APWaveform_280.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'IDhyperpol_150', 'stimuli': [{'delay': 100.0, 'tmid': 700.0000000000001, 'tmid2': 2700.0, 'toff': 2900.0, 'amp': 0.47654440222873184, 'long_amp': -0.22305531470241, 'thresh_perc': 143.99830859749372, 'duration': 2000.0, 'totduration': 3000.0, 'holding_current': -0.1074999999254942}], 'recordings_from_config': [{'type': 'CompRecording', 'name': 'IDhyperpol_150.soma.v', 'location': 'soma', 'variable': 'v'}], 'validation': True, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - " {'name': 'Step_1000', 'stimuli': [{'amp': None, 'delay': 700, 'duration': 4, 'holding_current': None, 'thresh_perc': 1000, 'totduration': 1000}], 'recordings_from_config': [{'type': 'CompRecording', 'location': 'soma', 'name': 'Step_1000.soma.v', 'variable': 'v'}], 'validation': False, 'protocol_type': 'ThresholdBasedProtocol', 'stochasticity': False}\n", - "EFeatures:\n", - " {'efel_feature_name': 'Spikecount', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 8.666666666666666, 'original_std': 3.496029493900505, 'sample_size': 6, 'efeature_name': 'Spikecount', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_CV', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.3702316995087826, 'original_std': 0.32566048327925706, 'sample_size': 6, 'efeature_name': 'ISI_CV', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'doublet_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 151.83333333319524, 'original_std': 34.61624939562405, 'sample_size': 6, 'efeature_name': 'doublet_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'mean_frequency', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.70922378383736, 'original_std': 1.6760068225673201, 'sample_size': 6, 'efeature_name': 'mean_frequency', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_first_spike', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 33.458333332939674, 'original_std': 7.231897899039166, 'sample_size': 6, 'efeature_name': 'time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_log_slope', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -0.16581691171366217, 'original_std': 0.8515890058904552, 'sample_size': 6, 'efeature_name': 'ISI_log_slope', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -82.78261651725562, 'original_std': 0.657020244055356, 'sample_size': 6, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_last_spike', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1806.9583333380724, 'original_std': 116.31793863080907, 'sample_size': 6, 'efeature_name': 'time_to_last_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_time_to_first_spike', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 31.44753263082006, 'original_std': 7.29210290109057, 'sample_size': 6, 'efeature_name': 'inv_time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_first_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 6.925399554807271, 'original_std': 1.5071612674614459, 'sample_size': 6, 'efeature_name': 'inv_first_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_second_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.985717646424224, 'original_std': 2.104516309965638, 'sample_size': 6, 'efeature_name': 'inv_second_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_third_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.840173932427828, 'original_std': 1.2897829185036143, 'sample_size': 6, 'efeature_name': 'inv_third_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_last_ISI', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.740281701448489, 'original_std': 1.4623481605488782, 'sample_size': 6, 'efeature_name': 'inv_last_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_depth', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 18.961034660480593, 'original_std': 1.1849839363452002, 'sample_size': 6, 'efeature_name': 'AHP_depth', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_time_from_peak', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 25.620500090252204, 'original_std': 2.638148646675542, 'sample_size': 6, 'efeature_name': 'AHP_time_from_peak', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'min_AHP_values', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -63.821581856775026, 'original_std': 1.5391678614966964, 'sample_size': 6, 'efeature_name': 'min_AHP_values', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'depol_block_bool', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'depol_block_bool', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_after_stim', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -87.12239628840818, 'original_std': 0.9572409682749269, 'sample_size': 6, 'efeature_name': 'voltage_after_stim', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'burst_number', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.6666666666666666, 'original_std': 0.9428090415820634, 'sample_size': 6, 'efeature_name': 'burst_number', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'number_initial_spikes', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1.6666666666666667, 'original_std': 0.4714045207910317, 'sample_size': 6, 'efeature_name': 'number_initial_spikes', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'irregularity_index', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 213.95798611189414, 'original_std': 278.960340993408, 'sample_size': 6, 'efeature_name': 'irregularity_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'adaptation_index', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.021810381821812972, 'original_std': 0.025904931541217195, 'sample_size': 6, 'efeature_name': 'adaptation_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'burst_mean_freq', 'protocol_name': 'IDrest_130', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 6.189144214407903, 'original_std': 0.14984960695170724, 'sample_size': 2, 'efeature_name': 'burst_mean_freq', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'Spikecount', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 19.833333333333332, 'original_std': 3.4840908267278117, 'sample_size': 6, 'efeature_name': 'Spikecount', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_CV', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.07463775460612596, 'original_std': 0.028505952255165983, 'sample_size': 6, 'efeature_name': 'ISI_CV', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'doublet_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 44.208333333293126, 'original_std': 9.628967436268557, 'sample_size': 6, 'efeature_name': 'doublet_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'mean_frequency', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 10.189637299220186, 'original_std': 1.778840871822643, 'sample_size': 6, 'efeature_name': 'mean_frequency', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_first_spike', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 16.58333333295502, 'original_std': 3.72677996249626, 'sample_size': 6, 'efeature_name': 'time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ISI_log_slope', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.03345638299091273, 'original_std': 0.05974524879883885, 'sample_size': 6, 'efeature_name': 'ISI_log_slope', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -82.656065689877, 'original_std': 0.4123618296597106, 'sample_size': 6, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'time_to_last_spike', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1946.500000005247, 'original_std': 17.939713858801507, 'sample_size': 6, 'efeature_name': 'time_to_last_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_time_to_first_spike', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 63.138076544811895, 'original_std': 12.691389369204195, 'sample_size': 6, 'efeature_name': 'inv_time_to_first_spike', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_first_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 23.61128698555237, 'original_std': 4.5663583102585985, 'sample_size': 6, 'efeature_name': 'inv_first_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_second_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 10.888408719900829, 'original_std': 1.9500846038754984, 'sample_size': 6, 'efeature_name': 'inv_second_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_third_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 9.988777902840466, 'original_std': 2.0044673370648782, 'sample_size': 6, 'efeature_name': 'inv_third_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'inv_last_ISI', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 9.955328682497806, 'original_std': 2.3570371576647995, 'sample_size': 6, 'efeature_name': 'inv_last_ISI', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_depth', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 21.875388331111182, 'original_std': 1.6438123287417166, 'sample_size': 6, 'efeature_name': 'AHP_depth', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_time_from_peak', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 20.00731497674777, 'original_std': 1.747628947979076, 'sample_size': 6, 'efeature_name': 'AHP_time_from_peak', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'min_AHP_values', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -60.78067735876579, 'original_std': 1.919410899483439, 'sample_size': 6, 'efeature_name': 'min_AHP_values', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'depol_block_bool', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'depol_block_bool', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_after_stim', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -86.90101193311828, 'original_std': 0.5605484695404097, 'sample_size': 6, 'efeature_name': 'voltage_after_stim', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'burst_number', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'burst_number', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'number_initial_spikes', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 3.0, 'original_std': 0.001, 'sample_size': 6, 'efeature_name': 'number_initial_spikes', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'irregularity_index', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 6.7059774196184, 'original_std': 1.583840773838753, 'sample_size': 6, 'efeature_name': 'irregularity_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'adaptation_index', 'protocol_name': 'IDrest_200', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.0005470381772781952, 'original_std': 0.0037812147886006962, 'sample_size': 6, 'efeature_name': 'adaptation_index', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'steady_state_voltage_stimend', 'protocol_name': 'RMPProtocol', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.38956665960032, 'original_std': 0.44319427048989085, 'sample_size': 9, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ohmic_input_resistance_vb_ssse', 'protocol_name': 'IV_0', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 40.381127264357104, 'original_std': 486.9230848650381, 'sample_size': 8, 'efeature_name': 'ohmic_input_resistance_vb_ssse', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'steady_state_voltage_stimend', 'protocol_name': 'SearchHoldingCurrent', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.1602795290849, 'original_std': 0.5732871465615677, 'sample_size': 9, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ohmic_input_resistance_vb_ssse', 'protocol_name': 'RinProtocol', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 46.144060948518096, 'original_std': 9.540253650631344, 'sample_size': 9, 'efeature_name': 'ohmic_input_resistance_vb_ssse', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IV_-100', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.10270958313072, 'original_std': 0.7408161829891378, 'sample_size': 9, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'ohmic_input_resistance_vb_ssse', 'protocol_name': 'IV_-100', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 41.950232851373016, 'original_std': 7.7917698493471175, 'sample_size': 9, 'efeature_name': 'ohmic_input_resistance_vb_ssse', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AP_amplitude', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 85.63229179382726, 'original_std': 0.2739372661121643, 'sample_size': 3, 'efeature_name': 'AP_amplitude', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AP1_amp', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 91.24062410990176, 'original_std': 0.7323490865525268, 'sample_size': 3, 'efeature_name': 'AP1_amp', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AP_duration_half_width', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1.3916666666666269, 'original_std': 0.005892556509886723, 'sample_size': 3, 'efeature_name': 'AP_duration_half_width', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'AHP_depth', 'protocol_name': 'APWaveform_280', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 23.89942607879726, 'original_std': 0.4119030778088721, 'sample_size': 3, 'efeature_name': 'AHP_depth', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'mean_frequency', 'protocol_name': 'IDhyperpol_150', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 4.824314976680847, 'original_std': 0.8271130180600073, 'sample_size': 2, 'efeature_name': 'mean_frequency', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'voltage_base', 'protocol_name': 'IDhyperpol_150', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -83.85849777183599, 'original_std': 0.5382516057115367, 'sample_size': 2, 'efeature_name': 'voltage_base', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'depol_block_bool', 'protocol_name': 'IDhyperpol_150', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 1.0, 'original_std': 0.001, 'sample_size': 2, 'efeature_name': 'depol_block_bool', 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True, 'Threshold': -20, 'interp_step': 0.025}}\n", - " {'efel_feature_name': 'bpo_holding_current', 'protocol_name': 'SearchHoldingCurrent', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': -0.09753230129913182, 'original_std': 0.039795330549849496, 'sample_size': None, 'efeature_name': None, 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True}}\n", - " {'efel_feature_name': 'bpo_threshold_current', 'protocol_name': 'SearchThresholdCurrent', 'recording_name': 'soma.v', 'threshold_efeature_std': 0.1, 'default_std_value': 0.001, 'mean': 0.33093749980131787, 'original_std': 0.06647122484201451, 'sample_size': None, 'efeature_name': None, 'weight': 1.0, 'efel_settings': {'strict_stiminterval': True}}\n", - " {'efel_feature_name': 'Spikecount', 'protocol_name': 'Step_1000', 'recording_name': 'soma.v', 'threshold_efeature_std': None, 'default_std_value': 0.001, 'mean': 1.0, 'original_std': 0.1, 'sample_size': None, 'efeature_name': None, 'weight': 1.0, 'efel_settings': {'strict_stiminterval': False}}\n", - "\n" - ] - } - ], - "source": [ - "efeature_def = EFeatureConfiguration(\n", - " efel_feature_name=\"Spikecount\",\n", - " protocol_name=protocol_name,\n", - " recording_name=\"soma.v\",\n", - " mean=1.0,\n", - " std=0.1,\n", - " efel_settings={\"strict_stiminterval\": False},\n", - " threshold_efeature_std=None,\n", - ")\n", - "\n", - "fitness_config.efeatures.append(efeature_def)\n", - "print(fitness_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, we save the edited FitnessCalculatorConfiguration back to Nexus" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " _deprecate_one\n", - " True\n", - " _register_one\n", - " True\n", - " _update_one\n", - " True\n" - ] - } - ], - "source": [ - "access_point.store_fitness_calculator_configuration(fitness_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To verify that the optimization works with the new FCC, run the following command:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python pipeline.py --step=test_optimise --emodel={emodel} --etype={etype} --iteration_tag={iteration_tag} --ttype={ttype}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv-bpem", - "language": "python", - "name": "venv-bpem" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/nexus/edit_neuron_model_configuration.ipynb b/examples/nexus/edit_neuron_model_configuration.ipynb deleted file mode 100644 index 7e4bb7b..0000000 --- a/examples/nexus/edit_neuron_model_configuration.ipynb +++ /dev/null @@ -1,319 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## How to edit an already existing Neuron Model Configuration" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook explains how to load an already existing neuron model configuration from nexus, edit it, and store it on nexus" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To run this notebook, you need to already have a NeuronModelConfiguration in Nexus." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading the NeuronModelConfiguration" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Please fill in the values to match the metadata of the model you want to edit." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "emodel = \"L5_TPC\"\n", - "etype = \"cAC\"\n", - "mtype = None\n", - "ttype = None\n", - "iteration_tag = \"XXXX-XX-XX\"\n", - "species = \"mouse\"\n", - "brain_region = \"SSCX\"\n", - "\n", - "nexus_project = \"\" # specify the nexus project to use\n", - "nexus_organisation = \"\" # specify the nexus organisation to use\n", - "nexus_endpoint = \"\" # specify the nexus endpoint to use\n", - "\n", - "# Advanced settings (only change if you know what you are doing)\n", - "forge_path = \"./forge.yml\"\n", - "forge_ontology_path = \"./nsg.yml\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This next cell will ask for your Nexus token:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.access_point.nexus import NexusAccessPoint\n", - "\n", - "access_point = NexusAccessPoint(\n", - " emodel=emodel,\n", - " species=species,\n", - " brain_region=brain_region,\n", - " project=nexus_project,\n", - " organisation=nexus_organisation,\n", - " endpoint=nexus_endpoint,\n", - " forge_path=forge_path,\n", - " forge_ontology_path=forge_ontology_path,\n", - " etype=etype,\n", - " mtype=mtype,\n", - " ttype=ttype,\n", - " iteration_tag=iteration_tag\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.model.model_configurator import ModelConfigurator\n", - "\n", - "configurator = ModelConfigurator(access_point)\n", - "configurator.load_configuration()\n", - "\n", - "print(configurator.configuration)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Adding Mechanisms, Distributions, Parameters or Morphology" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can now modify the neuron model configuration. Below is an example where a mechanism, a distribution, and a parameter are added to the existing mechanisms, distributions, and parameters of the neuron model configuration. Additionally, the morphology is updated to a different one." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.model.morphology_configuration import MorphologyConfiguration" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# add distribution\n", - "configurator.configuration.add_distribution(\n", - " distribution_name=\"decay\",\n", - " function=\"math.exp({distance}*{constant})*{value}\",\n", - " parameters=[\"constant\"],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# add two parameters\n", - "configurator.configuration.add_parameter(\n", - " parameter_name=\"constant\",\n", - " locations=\"distribution_decay\",\n", - " value=[-0.1,0.0],\n", - " mechanism=None,\n", - " distribution_name=None,\n", - " stochastic=None,\n", - ")\n", - "configurator.configuration.add_parameter(\n", - " parameter_name=\"gNaTgbar_NaTg\",\n", - " locations=\"apical\",\n", - " value=[0.0, 0.1],\n", - " mechanism=\"NaTg\",\n", - " distribution_name=\"decay\",\n", - " stochastic=None,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# add mechanism\n", - "configurator.configuration.add_mechanism(\n", - " mechanism_name=\"NaTg\",\n", - " locations=\"apical\",\n", - " stochastic=None,\n", - " version=None,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# change morphology\n", - "configurator.configuration.morphology = MorphologyConfiguration(name=\"ak171123_C_idA\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Removing Parameters, Mechanisms, and Distributions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here, we will see how to remove parameters, mechanisms and distributions. For the sake of simplicity, we will remove the ones that we just added." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# remove parameter\n", - "configurator.configuration.remove_parameter(parameter_name=\"constant\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When a mechanism is removed, it also removes all associated parameters. Here, 'gNaTgbar_NaTg' parameter will be removed as well" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# remove mechanism\n", - "configurator.configuration.remove_mechanism(mechanism_name=\"NaTg\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There is no function to remove a distribution, but if you need to remove one, you can use the code below." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# remove a distribution\n", - "distribution_name = \"decay\"\n", - "configurator.configuration.distributions = [d for d in configurator.configuration.distributions if d.name != distribution_name]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Saving the NeuronModelConfiguration" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If there is no optimised model in your access point, you can store the new neuron model configuration as it is.\n", - "\n", - "However, if there is an optimised model in your access point, it is advisable to use a new access point with an unused iteration tag to avoid conflicts with the existing model.\n", - "\n", - "In this example, since we have just created the access point without optimising any model yet, we will keep it for storing the new neuron model configuration. Note that the old NeuronModelConfiguration will be deprecated." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# store new model config\n", - "configurator.save_configuration()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To verify that the optimization works, run the following commands:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python pipeline.py --step=extract --emodel={emodel} --etype={etype} --iteration_tag={iteration_tag} --ttype={ttype}\n", - "!python pipeline.py --step=test_optimise --emodel={emodel} --etype={etype} --iteration_tag={iteration_tag} --ttype={ttype}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv-bpem", - "language": "python", - "name": "venv-bpem" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/nexus/exploit_model.ipynb b/examples/nexus/exploit_model.ipynb deleted file mode 100644 index cdd80fe..0000000 --- a/examples/nexus/exploit_model.ipynb +++ /dev/null @@ -1,277 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Accessing the emodels and running custom protocols" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook shows how to access the optimised emodels and how to run custom protocols on them." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Instantiating an access point to Nexus" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Please fill in the values to match the metadata of the model you want to access" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "emodel = \"L5_TPC\"\n", - "etype = \"cAC\"\n", - "mtype = None\n", - "ttype = None\n", - "iteration_tag = \"XXXX-XX-XX\"\n", - "species = \"mouse\"\n", - "brain_region = \"SSCX\"\n", - "\n", - "nexus_project = \"\" # specify the nexus project to use\n", - "nexus_organisation = \"\" # specify the nexus organisation to use\n", - "nexus_endpoint = \"\" # specify the nexus endpoint to use\n", - "\n", - "# Advanced settings (only change if you know what you are doing)\n", - "forge_path = \"./forge.yml\"\n", - "forge_ontology_path = \"./nsg.yml\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This next cell will ask for your Nexus token:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.access_point.nexus import NexusAccessPoint\n", - "\n", - "access_point = NexusAccessPoint(\n", - " emodel=emodel,\n", - " species=species,\n", - " brain_region=brain_region,\n", - " project=nexus_project,\n", - " organisation=nexus_organisation,\n", - " endpoint=nexus_endpoint,\n", - " forge_path=forge_path,\n", - " etype=etype,\n", - " mtype=mtype,\n", - " ttype=ttype,\n", - " iteration_tag=iteration_tag,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieving the emodels" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "emodels = access_point.get_emodels()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pprint\n", - "\n", - "print(f\"Found {len(emodels)} emodels\")\n", - "print(\"\\nExample of an emodel:\\n\")\n", - "pprint.pprint(emodels[0].as_dict())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Running protocols and the emodel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Instantiate the evaluator which contains the model and the protocols used for optimisation:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.evaluation.evaluation import get_evaluator_from_access_point\n", - "\n", - "evaluator = get_evaluator_from_access_point(\n", - " access_point,\n", - " stochasticity=False,\n", - " include_validation_protocols=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To rerun the protocols that were used for optimisation on the first of the models, do:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "responses = evaluator.run_protocols(protocols=evaluator.fitness_protocols.values(), param_values=emodels[0].parameters)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you wish to run additional protocols, you can first define a new protocol:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.evaluation.protocol_configuration import ProtocolConfiguration\n", - "from bluepyemodel.evaluation.evaluator import define_protocol\n", - "\n", - "protocol_name = \"Step_test\"\n", - "\n", - "stimuli = [{\n", - " \"holding_current\": -0.08,\n", - " \"threshold\": 0.16,\n", - " \"amp\": 300,\n", - " \"delay\": 100,\n", - " \"duration\": 1000,\n", - " \"totduration\": 1200\n", - "}]\n", - "\n", - "recordings = [\n", - " {\n", - " \"type\": \"CompRecording\",\n", - " \"name\": f\"{protocol_name}.soma.v\",\n", - " \"location\": \"soma\",\n", - " \"variable\": \"v\",\n", - " }\n", - "]\n", - "\n", - "my_protocol_configuration = ProtocolConfiguration(\n", - " name=protocol_name,\n", - " stimuli=stimuli,\n", - " recordings=recordings,\n", - " validation=False\n", - ")\n", - "\n", - "my_protocol = define_protocol(my_protocol_configuration)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that the name of the protocol matters as BluePyEmodel will use it to detect which shape the protocol should have. For example, here, \"step\" is in the name and therefore a step protocol will be generated. A complete list of the eCode that can be used to create protocols can be seen [here](https://github.com/BlueBrain/BluePyEModel/blob/main/bluepyemodel/ecode/__init__.py). For more complex eCode, contact the package authors for an explanation of how to use them." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once the protocol is created, it can be added to the protocols already in the evaluator:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "evaluator.fitness_protocols[\"main_protocol\"].protocols[protocol_name] = my_protocol\n", - "evaluator.fitness_protocols[\"main_protocol\"].execution_order = (\n", - " evaluator.fitness_protocols[\"main_protocol\"].compute_execution_order()\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, the responses can be plotted and saved in a pdf in the current working directory:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from bluepyemodel.emodel_pipeline import plotting\n", - "from bluepyemodel.emodel_pipeline.plotting import get_recording_names\n", - "\n", - "responses = evaluator.run_protocols(protocols=evaluator.fitness_protocols.values(), param_values=emodels[0].parameters)\n", - "stimuli = evaluator.fitness_protocols[\"main_protocol\"].protocols\n", - "recording_names = get_recording_names(\n", - " access_point.get_fitness_calculator_configuration().protocols,\n", - " stimuli,\n", - ")\n", - "plotting.traces(model=emodels[0], responses=responses, recording_names=recording_names, stimuli=stimuli, figures_dir=\"./\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv-bpem", - "language": "python", - "name": "venv-bpem" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/nexus/forge.yml b/examples/nexus/forge.yml deleted file mode 100644 index 8950993..0000000 --- a/examples/nexus/forge.yml +++ /dev/null @@ -1,73 +0,0 @@ -Model: - name: RdfModel - origin: store - source: BlueBrainNexus - context: - iri: "https://bbp.neuroshapes.org" - bucket: "neurosciencegraph/datamodels" - -Store: - name: BlueBrainNexus - endpoint: https://openbluebrain.com/api/nexus/v1 - model: - name: RdfModel - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - elastic: - endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" - mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" - default_str_keyword_field: "keyword" - vocabulary: - metadata: - iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" - local_iri: "https://bluebrainnexus.io/contexts/metadata.json" - namespace: "https://bluebrain.github.io/nexus/vocabulary/" - deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" - project_property: "https://bluebrain.github.io/nexus/vocabulary/project" - max_connection: 5 - versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" - file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson - -Resolvers: - ontology: - - resolver: OntologyResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: terms - bucket: neurosciencegraph/datamodels - - identifier: CellType - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainCellType - - identifier: BrainRegion - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainRegion - - identifier: Species - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: Species - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson - agent: - - resolver: AgentResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: agents - bucket: bbp/agents - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson - -Formatters: - identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} - identifier_bbn_self: https://bbp.epfl.ch/resources/{}/{}/{}/{} # https://bbp.epfl.ch/nexus/v1/resources/{organization}/{project}/{schema}/{id} diff --git a/examples/nexus/img_nexus_id.png b/examples/nexus/img_nexus_id.png deleted file mode 100644 index 7429523..0000000 Binary files a/examples/nexus/img_nexus_id.png and /dev/null differ diff --git a/examples/nexus/img_nexus_token.png b/examples/nexus/img_nexus_token.png deleted file mode 100644 index 4541c13..0000000 Binary files a/examples/nexus/img_nexus_token.png and /dev/null differ diff --git a/examples/nexus/launch_luigi.sh b/examples/nexus/launch_luigi.sh deleted file mode 100755 index 751aa76..0000000 --- a/examples/nexus/launch_luigi.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -##################################################################### -# Copyright 2024 Blue Brain Project / EPFL - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -##################################################################### - -deactivate -module purge all -source myvenv/bin/activate - -emodel=EMODEL_NAME # e.g. "L5_TPC" -etype=ETYPE_NAME # e.g. "cAC" -mtype=MTYPE_NAME -ttype=TTYPE_NAME -species="mouse" -brain_region="SSCX" -iteration="XXXX-XX-XX" - -mtype_formatted=${mtype// /__} -ttype_formatted=${ttype// /__} - -bbp-workflow launch-bb5 -f --config=luigi.cfg bluepyemodel.tasks.emodel_creation.optimisation EModelCreation emodel=$emodel species=$species brain-region=$brain_region iteration-tag=$iteration etype=$etype mtype=$mtype_formatted ttype=$ttype_formatted \ No newline at end of file diff --git a/examples/nexus/luigi.cfg b/examples/nexus/luigi.cfg deleted file mode 100644 index 0a22b7c..0000000 --- a/examples/nexus/luigi.cfg +++ /dev/null @@ -1,42 +0,0 @@ -[DEFAULT] -account=PROJECT_NAME -module-archive=unstable -modules= -virtual-env=PATH_TO_VIRTUALENV -chdir=PATH_TO_YOUR_WORKSPACE -workflows-sync=/gpfs/bbp.cscs.ch/home/${USER}/workflows -enable-internet=True -workers=1 -time=24:00:00 - -[ExtractEFeatures] -enable-internet=True -modules= - -[Optimise] -enable-internet=True -modules= -nodes=2 -time=20:00:00 -continue_unfinished_optimisation=True - -[Validation] -enable-internet=True -modules= - -[ExportHoc] -modules= - -[core] -log_level=INFO - -[parallel] -backend=ipyparallel - -[EmodelAPIConfig] -api=nexus -forge_path=./forge.yml -forge_ontology_path=./nsg.yml -nexus_project=NEXUS_PROJECT -nexus_organisation=NEXUS_ORG -nexus_endpoint=https://bbp.epfl.ch/nexus/v1 diff --git a/examples/nexus/nsg.yml b/examples/nexus/nsg.yml deleted file mode 100644 index 33fb019..0000000 --- a/examples/nexus/nsg.yml +++ /dev/null @@ -1,73 +0,0 @@ -Model: - name: RdfModel - origin: store - source: BlueBrainNexus - context: - iri: "https://neuroshapes.org" - bucket: "neurosciencegraph/datamodels" - -Store: - name: BlueBrainNexus - endpoint: https://openbluebrain.com/api/nexus/v1 - model: - name: RdfModel - searchendpoints: - sparql: - endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/62529364-b584-4cc9-82ce-36efd690b111" - elastic: - endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" - mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" - default_str_keyword_field: "keyword" - vocabulary: - metadata: - iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" - local_iri: "https://bluebrainnexus.io/contexts/metadata.json" - namespace: "https://bluebrain.github.io/nexus/vocabulary/" - deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" - project_property: "https://bluebrain.github.io/nexus/vocabulary/project" - max_connection: 5 - versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" - file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson - -Resolvers: - ontology: - - resolver: OntologyResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: terms - bucket: neurosciencegraph/datamodels - - identifier: CellType - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainCellType - - identifier: BrainRegion - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainRegion - - identifier: Species - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: Species - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson - agent: - - resolver: AgentResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: agents - bucket: bbp/agents - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson - -Formatters: - identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} - identifier_bbn_self: https://bbp.epfl.ch/resources/{}/{}/{}/{} # https://bbp.epfl.ch/nexus/v1/resources/{organization}/{project}/{schema}/{id} diff --git a/examples/nexus/pipeline.py b/examples/nexus/pipeline.py deleted file mode 100644 index 7f5063e..0000000 --- a/examples/nexus/pipeline.py +++ /dev/null @@ -1,199 +0,0 @@ -""" -Copyright 2024, EPFL/Blue Brain Project - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import argparse -import json -import logging - -from bluepyemodel.model.model_configurator import ModelConfigurator -from bluepyemodel.emodel_pipeline.emodel_pipeline import EModel_pipeline -from bluepyemodel.emodel_pipeline.emodel_settings import EModelPipelineSettings -from bluepyemodel.efeatures_extraction.targets_configurator import TargetsConfigurator -from targets import filenames, ecodes_metadata, targets, protocols_rheobase - -logger = logging.getLogger() - - -def get_parser(): - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="Pipeline for Nexus", - ) - - parser.add_argument( - "--step", - type=str, - required=True, - choices=[ - "configure_nexus", - "configure_model_from_gene", - "configure_model_from_json", - "extract", - "test_optimise", - "test_analyze", - ], - ) - parser.add_argument("--emodel", type=str, required=True) - parser.add_argument("--ttype", type=str, required=False) - parser.add_argument("--etype", type=str, required=True) - parser.add_argument("--mtype", type=str, required=False) - parser.add_argument("--iteration_tag", type=str, default="test") - parser.add_argument("--use_ipyparallel", action="store_true", default=True) - parser.add_argument("--use_multiprocessing", action="store_true", default=False) - parser.add_argument("-v", "--verbose", action="count", dest="verbosity", default=1) - parser.add_argument("--seed", type=int, required=False, default=1) - - return parser - - -def configure_targets(access_point): - - files_metadata = [] - for filename in filenames: - files_metadata.append({"cell_name": filename, "ecodes": ecodes_metadata}) - - targets_formated = [] - for ecode in targets: - for amplitude in targets[ecode]["amplitudes"]: - for efeature in targets[ecode]["efeatures"]: - targets_formated.append( - { - "efeature": efeature, - "protocol": ecode, - "amplitude": amplitude, - "tolerance": 10.0, - } - ) - - configurator = TargetsConfigurator(access_point) - configurator.new_configuration(files_metadata, targets_formated, protocols_rheobase) - configurator.save_configuration() - print(configurator.configuration.as_dict()) - - -def store_pipeline_settings(access_point): - - pipeline_settings = EModelPipelineSettings( - extraction_threshold_value_save=1, - stochasticity=False, - optimiser="SO-CMA", - optimisation_params={"offspring_size": 20}, - optimisation_timeout=600.0, - threshold_efeature_std=0.05, - max_ngen=250, - validation_threshold=10.0, - optimisation_batch_size=10, - max_n_batch=10, - n_model=1, - name_gene_map="Mouse_met_types_ion_channel_expression", - plot_extraction=False, - plot_optimisation=True, - compile_mechanisms=True, - name_Rin_protocol="IV_-40", - name_rmp_protocol="IV_0", - validation_protocols=["IDhyperpol_150"], - morph_modifiers=[], - ) - - access_point.store_pipeline_settings(pipeline_settings) - - -def configure(pipeline): - pipeline.access_point.access_point.deprecate_all( - {"iteration": pipeline.access_point.emodel_metadata.iteration, - "eModel": pipeline.access_point.emodel_metadata.emodel} - ) - configure_targets(pipeline.access_point) - store_pipeline_settings(pipeline.access_point) - - -def configure_model(pipeline, legacy_conf_json_file, morphology_name, morphology_format=None): - configurator = ModelConfigurator(pipeline.access_point) - configurator.new_configuration() - - with open(legacy_conf_json_file, "r") as f: - parameters = json.load(f) - - configurator.configuration.init_from_legacy_dict(parameters, {"name": morphology_name, "format": morphology_format}) - configurator.save_configuration() - print(configurator.configuration) - - -if __name__ == "__main__": - - args = get_parser().parse_args() - - species = "mouse" - - brain_region = "SSCX" - - morphology = "C060114A5" - morphology_format = "asc" # specify the format of the morphology file, if not specified, the first availabe format will be used - - nexus_project = "" # specify the project name in nexus - nexus_organisation = "" # specify the organisation name in nexus - nexus_endpoint = "prod" - forge_path = "./forge.yml" - forge_ontology_path = "./nsg.yml" - - legacy_conf_json_file = "./../L5PC/config/params/pyr.json" # specify the path to the json file containing the model configuration if the model is not configured from gene data - - logging.basicConfig( - level=(logging.WARNING, logging.INFO, logging.DEBUG)[1], - handlers=[logging.StreamHandler()], - ) - - pipeline = EModel_pipeline( - emodel=args.emodel, - ttype=args.ttype, - iteration_tag=args.iteration_tag, - species=species, - brain_region=brain_region, - forge_path=forge_path, - forge_ontology_path=forge_ontology_path, - nexus_organisation=nexus_organisation, - nexus_project=nexus_project, - nexus_endpoint=nexus_endpoint, - use_ipyparallel=args.use_ipyparallel, - use_multiprocessing=args.use_multiprocessing, - mtype=args.mtype, - etype=args.etype, - data_access_point="nexus", - ) - - if args.step == "configure_nexus": - configure(pipeline) - elif args.step == "configure_model_from_gene": - pipeline.configure_model(morphology_name=morphology, morphology_format=morphology_format, use_gene_data=True) - elif args.step == "configure_model_from_json": - configure_model(pipeline, legacy_conf_json_file, morphology, morphology_format) - elif args.step == "extract": - pipeline.extract_efeatures() - elif args.step == "test_optimise": - logger.warning( - "test_optimise is only to check that the optimisation works. To " - "optimise the models, please use launch_luigi.sh" - ) - pipeline.optimise(seed=args.seed) - elif args.step == "test_analyze": - logger.warning( - "test_analyze is only to check that the validation and storing " - "of the models work. For real validation and storing, please use launch_luigi.sh" - ) - pipeline.store_optimisation_results() - pipeline.validation() - pipeline.plot() diff --git a/examples/nexus/run_pipeline.ipynb b/examples/nexus/run_pipeline.ipynb deleted file mode 100644 index 310d4ef..0000000 --- a/examples/nexus/run_pipeline.ipynb +++ /dev/null @@ -1,269 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run the pipeline with Nexus/Luigi/bbp-workflow" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook provides an example of how to run the pipeline using the Nexus access point, Luigi and bbp-workflow. For more details, please refer to the [README](./README.md)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "from bluepyemodel.model.model_configurator import ModelConfigurator\n", - "from bluepyemodel.emodel_pipeline.emodel_pipeline import EModel_pipeline\n", - "from bluepyemodel.emodel_pipeline.emodel_settings import EModelPipelineSettings\n", - "from bluepyemodel.efeatures_extraction.targets_configurator import TargetsConfigurator" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "emodel = \"L5_TPC\"\n", - "etype = \"cAC\"\n", - "mtype = None\n", - "ttype = None\n", - "iteration_tag = \"XXXX-XX-XX\"\n", - "species = \"mouse\"\n", - "brain_region = \"SSCX\"\n", - "iteration=\"test-bpem-nexus\"\n", - "morphology = \"C060114A5\"\n", - "morphology_format = \"asc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Nexus settings\n", - "nexus_project = \"\" # specify the nexus project\n", - "nexus_organisation = \"\" # specify the nexus organisation\n", - "nexus_endpoint = \"prod\"\n", - "forge_path = \"./forge.yml\"\n", - "forge_ontology_path = \"./nsg.yml\"\n", - "legacy_conf_json_file = \"./../L5PC/config/params/pyr.json\" # specify the path to the json file containing the model configuration if the model is not configured from gene data" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "logger = logging.getLogger()\n", - "logging.basicConfig(\n", - " level=logging.INFO,\n", - " handlers=[logging.StreamHandler()],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline = EModel_pipeline(\n", - " emodel=emodel,\n", - " etype=etype,\n", - " mtype=mtype,\n", - " ttype=ttype,\n", - " iteration_tag=iteration,\n", - " species=species,\n", - " brain_region=brain_region,\n", - " forge_path=forge_path,\n", - " forge_ontology_path=forge_ontology_path,\n", - " nexus_organisation=nexus_organisation,\n", - " nexus_project=nexus_project,\n", - " nexus_endpoint=nexus_endpoint,\n", - " use_ipyparallel=True,\n", - " use_multiprocessing=False,\n", - " data_access_point=\"nexus\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To begin, we need to initialize the ``TargetsConfigurator`` using the provided ``targets.py`` file. This file contains predefined configurations for targets and file metadata needed for the feature extraction process, and it can be customized as needed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from targets import filenames, ecodes_metadata, targets, protocols_rheobase\n", - "\n", - "files_metadata = []\n", - "for filename in filenames:\n", - " files_metadata.append({\"cell_name\": filename, \"ecodes\": ecodes_metadata})\n", - "\n", - "targets_formated = []\n", - "for ecode in targets:\n", - " for amplitude in targets[ecode][\"amplitudes\"]:\n", - " for efeature in targets[ecode][\"efeatures\"]:\n", - " targets_formated.append(\n", - " {\n", - " \"efeature\": efeature,\n", - " \"protocol\": ecode,\n", - " \"amplitude\": amplitude,\n", - " \"tolerance\": 10.0,\n", - " }\n", - " )\n", - "\n", - "configurator = TargetsConfigurator(pipeline.access_point)\n", - "configurator.new_configuration(files_metadata, targets_formated, protocols_rheobase)\n", - "configurator.save_configuration()\n", - "print(configurator.configuration.as_dict())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we need to store the ``EModelPipelineSettings`` (EMPS). The following cell creates an instance of EMPS with various parameters, which can be customized according to your needs:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " _deprecate_one\n", - " True\n", - " _register_one\n", - " True\n" - ] - } - ], - "source": [ - "pipeline_settings = EModelPipelineSettings(\n", - " stochasticity=False,\n", - " optimiser=\"SO-CMA\",\n", - " optimisation_params={\"offspring_size\": 20},\n", - " optimisation_timeout=600.0,\n", - " threshold_efeature_std=0.1,\n", - " max_ngen=100,\n", - " validation_threshold=10.0,\n", - " plot_extraction=True,\n", - " plot_optimisation=True,\n", - " compile_mechanisms=True,\n", - " name_Rin_protocol=\"IV_-40\",\n", - " name_rmp_protocol=\"IV_0\",\n", - " efel_settings={\n", - " \"strict_stiminterval\": True,\n", - " \"Threshold\": -20,\n", - " \"interp_step\": 0.025,\n", - " },\n", - " strict_holding_bounds=True,\n", - " validation_protocols=[\"IDhyperpol_150\"],\n", - " morph_modifiers=[],\n", - " n_model=3,\n", - " optimisation_batch_size=3,\n", - " max_n_batch=5,\n", - " name_gene_map=\"Mouse_met_types_ion_channel_expression\",\n", - " )\n", - "\n", - "pipeline.access_point.store_pipeline_settings(pipeline_settings)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the next cell, we will use ``ICSelector`` to generate the ``EModelConfiguration`` (EMC) based on gene data. This process will use the provided ``etype``, ``mtype``, and ``ttype`` to select appropriate ion channel resources from Nexus." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline.configure_model(morphology_name=morphology, morphology_format=morphology_format, use_gene_data=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before running the Luigi pipeline, we need to configure the [launch script](./launch_luigi.sh) and the [Luigi configuration script](./luigi.cfg). See sections 2 and 4, respectively, in the following [README](./README.md) for more details.\n", - "\n", - "Once the coniguration is done, we can run the Luigi pipeline in the terminal, by following these steps:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "Activate the virtual environment:\n", - "\n", - "``source myvenv/bin/activate``" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Initialize Kerberos authentication: \n", - "\n", - "``kinit``\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Launch the Luigi pipeline:\n", - "\n", - "``./launch_luigi.sh``" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv-bpem", - "language": "python", - "name": "venv-bpem" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/nexus/targets.py b/examples/nexus/targets.py deleted file mode 100644 index 136c7f6..0000000 --- a/examples/nexus/targets.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Copyright 2024, EPFL/Blue Brain Project - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - - - -filenames = [ - "C060116A6-SR-C1", - "C060112A4-SR-C1", - "C060116A1-SR-C1", -] - -ecodes_metadata = { - "IDthresh": { - "ljp": 14.0, - }, - "IDrest": { - "ljp": 14.0, - }, - "IV": {"ljp": 14.0, "ton": 20, "toff": 1020}, - "APWaveform": { - "ljp": 14.0, - }, - "IDhyperpol": {"ljp": 14.0, "ton": 100, "tmid": 700, "tmid2": 2700, "toff": 2900}, -} - - -protocols_rheobase = ["IDthresh"] - -targets = { - "IDrest": { - "amplitudes": [130, 200], - "efeatures": [ - "Spikecount", - "ISI_CV", - "doublet_ISI", - "mean_frequency", - "time_to_first_spike", - "ISI_log_slope", - "voltage_base", - "time_to_last_spike", - "inv_time_to_first_spike", - "inv_first_ISI", - "inv_second_ISI", - "inv_third_ISI", - "inv_last_ISI", - "AHP_depth", - "AHP_time_from_peak", - "min_AHP_values", - "depol_block_bool", - "voltage_after_stim", - "burst_number", - "number_initial_spikes", - "irregularity_index", - "adaptation_index", - "burst_mean_freq", - ], - }, - "IV": { - "amplitudes": [0, -40, -100], - "efeatures": [ - "voltage_base", - "ohmic_input_resistance_vb_ssse", - ], - }, - "APWaveform": { - "amplitudes": [280], - "efeatures": ["AP_amplitude", "AP1_amp", "AP_duration_half_width", "AHP_depth"], - }, - "IDhyperpol": { - "amplitudes": [150], - "efeatures": ["mean_frequency", "voltage_base", "depol_block_bool"], - }, -} diff --git a/pyproject.toml b/pyproject.toml index 226048b..2ac256a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,17 +65,9 @@ test = [ "pytest>=6.2", "dictdiffer>=0.8", ] -nexus = [ - "nexusforge>=0.8.2", - "entity_management>=1.2", - "pyJWT>=2.1.0", -] all = [ "luigi>=3.0", "luigi-tools>=0.0.12", - "nexusforge>=0.8.2", - "entity_management>=1.2", - "pyJWT>=2.1.0", "pytest>=6.2", "dictdiffer>=0.8", ] diff --git a/tests/unit_tests/test_emodel_pipeline.py b/tests/unit_tests/test_emodel_pipeline.py index 8302127..1e65306 100644 --- a/tests/unit_tests/test_emodel_pipeline.py +++ b/tests/unit_tests/test_emodel_pipeline.py @@ -17,10 +17,8 @@ import pytest from bluepyemodel.access_point.local import LocalAccessPoint -from bluepyemodel.access_point.nexus import NexusAccessPoint from bluepyemodel.emodel_pipeline.emodel_pipeline import EModel_pipeline from tests.utils import DATA -from unittest.mock import patch @pytest.fixture @@ -41,18 +39,3 @@ def pipeline(): def test_init_local(pipeline): assert isinstance(pipeline.access_point, LocalAccessPoint) - - -def test_init_nexus_missing_project(): - with pytest.raises( - ValueError, - match="Nexus project name is required for Nexus access point.", - ): - _ = EModel_pipeline( - emodel="cADpyr_L5TPC", - recipes_path=DATA / "config/recipes.json", - ttype="test", - species="mouse", - brain_region="SSCX", - data_access_point="nexus", - ) diff --git a/tests/unit_tests/test_nexus_access_point.py b/tests/unit_tests/test_nexus_access_point.py deleted file mode 100644 index 9cd8e6f..0000000 --- a/tests/unit_tests/test_nexus_access_point.py +++ /dev/null @@ -1,276 +0,0 @@ -""" -Copyright 2023-2024 Blue Brain Project / EPFL - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import pytest -from unittest.mock import PropertyMock, patch, Mock -from bluepyemodel.access_point.forge_access_point import NexusForgeAccessPoint, AccessPointException -from bluepyemodel.access_point.nexus import NexusAccessPoint -from datetime import datetime, timezone, timedelta -import logging - - -def jwt_payload(): - return { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": (datetime.now(timezone.utc) + timedelta(hours=1)).timestamp(), - } - - -@pytest.fixture(autouse=True) -def mock_jwt_decode(): - with patch("jwt.decode") as mock_jwt: - mock_jwt.return_value = jwt_payload() - yield mock_jwt - - -@pytest.fixture -def mock_nexus_access_point(): - with patch( - "bluepyemodel.access_point.forge_access_point.NexusForgeAccessPoint.refresh_token" - ) as mock_refresh, patch.object( - NexusForgeAccessPoint, "forge", new_callable=PropertyMock - ) as mock_forge_prop, patch( - "bluepyemodel.access_point.nexus.get_brain_region_notation", return_value="SS" - ) as mock_brain_region, patch( - "bluepyemodel.access_point.nexus.NexusAccessPoint.get_pipeline_settings", - return_value=Mock(), - ) as mock_pipeline_settings, patch( - "bluepyemodel.access_point.nexus.NexusAccessPoint.build_ontology_based_metadata" - ) as mock_build_metadata: - - mock_refresh.return_value = (datetime.now(timezone.utc) + timedelta(hours=1)).timestamp() - - mock_forge = Mock() - mock_resource = Mock() - mock_resource.id = "0" - mock_forge.resolve.return_value = mock_resource - mock_forge_prop.return_value = mock_forge - - mock_nexus_forge_access_point = NexusForgeAccessPoint( - project="test", - organisation="demo", - endpoint="https://openbluebrain.com/api/nexus/v1", - forge_path=None, - access_token="test_token", - ) - - with patch( - "bluepyemodel.access_point.forge_access_point.NexusForgeAccessPoint", - return_value=mock_nexus_forge_access_point, - ): - yield NexusAccessPoint( - emodel="L5_TPC", - etype="cAC", - ttype="189_L4/5 IT CTX", - mtype="L5_TPC:B", - species="mouse", - brain_region="SSCX", - iteration_tag="v0", - project="test", - organisation="demo", - endpoint="https://openbluebrain.com/api/nexus/v1", - forge_path=None, - forge_ontology_path=None, - access_token="test_token", - sleep_time=0, - ) - - -@pytest.fixture -def nexus_patches(): - with patch( - "bluepyemodel.access_point.nexus.get_brain_region_notation", return_value="SS" - ) as mock_brain_region, patch( - "bluepyemodel.access_point.nexus.NexusAccessPoint.get_pipeline_settings", - return_value=Mock(), - ) as mock_pipeline_settings, patch( - "bluepyemodel.access_point.nexus.NexusAccessPoint.build_ontology_based_metadata" - ) as mock_build_metadata: - yield mock_brain_region, mock_pipeline_settings, mock_build_metadata - - -def test_init(mock_nexus_access_point): - """ - Test the initialization of the NexusAccessPoint. - """ - emodel_metadata = mock_nexus_access_point.emodel_metadata - assert emodel_metadata.emodel == "L5_TPC" - assert emodel_metadata.etype == "cAC" - assert emodel_metadata.ttype == "189_L4/5 IT CTX" - assert emodel_metadata.mtype == "L5_TPC:B" - assert emodel_metadata.species == "mouse" - assert emodel_metadata.brain_region == "SSCX" - assert emodel_metadata.allen_notation == "SS" - assert emodel_metadata.iteration == "v0" - assert mock_nexus_access_point.forge_ontology_path is None - assert mock_nexus_access_point.sleep_time == 0 - - resolved_resource = mock_nexus_access_point.access_point.forge.resolve() - assert resolved_resource.id == "0" - - -@pytest.fixture -def mock_available_etypes(): - with patch.object( - NexusForgeAccessPoint, "available_etypes", new_callable=PropertyMock - ) as mock_etypes: - mock_etypes.return_value = ["0", "1", "2"] - yield mock_etypes - - -@pytest.fixture -def mock_available_mtypes(): - with patch.object( - NexusForgeAccessPoint, "available_mtypes", new_callable=PropertyMock - ) as mock_mtypes: - mock_mtypes.return_value = ["0", "1", "2"] - yield mock_mtypes - - -@pytest.fixture -def mock_available_ttypes(): - with patch.object( - NexusForgeAccessPoint, "available_ttypes", new_callable=PropertyMock - ) as mock_ttypes: - mock_ttypes.return_value = ["0", "1", "2"] - yield mock_ttypes - - -@pytest.fixture -def mock_check_mettypes_dependencies(): - with patch( - "bluepyemodel.access_point.nexus.ontology_forge_access_point" - ) as mock_ontology_forge, patch( - "bluepyemodel.access_point.nexus.check_resource" - ) as mock_check_resource: - mock_ontology_forge.return_value = Mock() - yield mock_ontology_forge, mock_check_resource - - -def test_check_mettypes( - mock_nexus_access_point, - mock_available_etypes, - mock_available_mtypes, - mock_available_ttypes, - mock_check_mettypes_dependencies, - caplog, -): - """ - Test the check_mettypes function of the NexusAccessPoint. - """ - mock_ontology_forge, mock_check_resource = mock_check_mettypes_dependencies - - mock_nexus_access_point.emodel_metadata.etype = "cAC" - mock_nexus_access_point.emodel_metadata.mtype = None - mock_nexus_access_point.emodel_metadata.ttype = None - - with caplog.at_level(logging.INFO): - mock_nexus_access_point.check_mettypes() - - assert "Checking if etype cAC is present on nexus..." in caplog.text - assert "Etype checked" in caplog.text - assert "Mtype is None, its presence on Nexus is not being checked." in caplog.text - assert "Ttype is None, its presence on Nexus is not being checked." in caplog.text - - mock_ontology_forge.assert_called_once_with( - mock_nexus_access_point.access_point.access_token, - mock_nexus_access_point.forge_ontology_path, - mock_nexus_access_point.access_point.endpoint, - ) - - mock_check_resource.assert_any_call( - "cAC", - "etype", - access_point=mock_ontology_forge.return_value, - access_token=mock_nexus_access_point.access_point.access_token, - forge_path=mock_nexus_access_point.forge_ontology_path, - endpoint=mock_nexus_access_point.access_point.endpoint, - ) - - -def test_get_nexus_subject_none(mock_nexus_access_point): - """ - Test get_nexus_subject with None as input. - """ - assert mock_nexus_access_point.get_nexus_subject(None) is None - - -def test_get_nexus_subject_human(mock_nexus_access_point): - """ - Test get_nexus_subject with 'human' as input. - """ - expected_subject = { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_9606", - "label": "Homo sapiens", - }, - } - assert mock_nexus_access_point.get_nexus_subject("human") == expected_subject - - -@pytest.mark.parametrize( - "species, expected_subject", - [ - (None, None), - ( - "human", - { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_9606", - "label": "Homo sapiens", - }, - }, - ), - ( - "mouse", - { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_10090", - "label": "Mus musculus", - }, - }, - ), - ( - "rat", - { - "type": "Subject", - "species": { - "id": "http://purl.obolibrary.org/obo/NCBITaxon_10116", - "label": "Rattus norvegicus", - }, - }, - ), - ], -) -def test_get_nexus_subject_parametrized(mock_nexus_access_point, species, expected_subject): - """ - Parametrized test for get_nexus_subject with different species inputs. - """ - assert mock_nexus_access_point.get_nexus_subject(species) == expected_subject - - -def test_get_nexus_subject_unknown_species(mock_nexus_access_point): - """ - Test get_nexus_subject with an unknown species input. - """ - with pytest.raises(ValueError, match="Unknown species unknown_species."): - mock_nexus_access_point.get_nexus_subject("unknown_species") diff --git a/tests/unit_tests/test_nexus_forge_access_point.py b/tests/unit_tests/test_nexus_forge_access_point.py deleted file mode 100644 index 6d8838c..0000000 --- a/tests/unit_tests/test_nexus_forge_access_point.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -Copyright 2023-2024 Blue Brain Project / EPFL - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import math -import pytest -from unittest.mock import Mock, patch -from bluepyemodel.access_point.forge_access_point import ( - AccessPointException, - NexusForgeAccessPoint, - get_brain_region, - get_brain_region_dict, -) -from datetime import datetime, timezone, timedelta - - -@pytest.fixture -def mock_forge_access_point(): - with patch("jwt.decode") as mock_jwt_decode: - mock_jwt_decode.return_value = { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": (datetime.now(timezone.utc) + timedelta(seconds=300)).timestamp(), - } - with patch( - "bluepyemodel.access_point.forge_access_point.NexusForgeAccessPoint.refresh_token" - ) as mock_refresh_token: - mock_refresh_token.return_value = ( - datetime.now(timezone.utc) + timedelta(seconds=300) - ).timestamp() - with patch( - "bluepyemodel.access_point.forge_access_point.KnowledgeGraphForge" - ) as mock_kg_forge: - return NexusForgeAccessPoint( - project="test", - organisation="demo", - endpoint="https://openbluebrain.com/api/nexus/v1", - forge_path=None, - access_token="test_token", - ) - - -def test_nexus_forge_access_point_init(mock_forge_access_point): - """ - Test the initialization of NexusForgeAccessPoint. - """ - assert mock_forge_access_point.bucket == "demo/test" - assert mock_forge_access_point.endpoint == "https://openbluebrain.com/api/nexus/v1" - assert mock_forge_access_point.access_token == "test_token" - assert ( - mock_forge_access_point.agent.id - == "https://bbp.epfl.ch/nexus/v1/realms/bbp/users/test_user" - ) - - -def test_refresh_token_not_expired(mock_forge_access_point): - """ - Test refresh_token method when the token is not expired. - """ - future_exp = (datetime.now(timezone.utc) + timedelta(seconds=300)).timestamp() - with patch("jwt.decode") as mock_jwt_decode: - mock_jwt_decode.return_value = { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": future_exp, - } - exp_timestamp = mock_forge_access_point.refresh_token() - assert math.isclose(exp_timestamp, future_exp, abs_tol=0.1) - - -def test_refresh_token_expired_offset(mock_forge_access_point, caplog): - """ - Test refresh_token method when the token is about to expire. - """ - future_exp = (datetime.now(timezone.utc) + timedelta(seconds=29)).timestamp() - new_future_exp = (datetime.now(timezone.utc) + timedelta(seconds=300)).timestamp() - with patch("jwt.decode") as mock_jwt_decode: - mock_jwt_decode.side_effect = [ - { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": future_exp, - }, - { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": new_future_exp, - }, - ] - with patch.object( - mock_forge_access_point, "get_access_token", return_value="new_test_token" - ): - with caplog.at_level("INFO"): - exp_timestamp = mock_forge_access_point.refresh_token() - assert math.isclose(exp_timestamp, new_future_exp, abs_tol=0.1) - assert "Nexus access token has expired, refreshing token..." in caplog.text - - -def test_refresh_token_expired(mock_forge_access_point, caplog): - """ - Test refresh_token method when the token has expired. - """ - past_exp = (datetime.now(timezone.utc) - timedelta(seconds=1)).timestamp() - new_future_exp = (datetime.now(timezone.utc) + timedelta(seconds=300)).timestamp() - with patch("jwt.decode") as mock_jwt_decode: - mock_jwt_decode.side_effect = [ - { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": past_exp, - }, - { - "preferred_username": "test_user", - "name": "Test User", - "email": "test_user@example.com", - "sub": "test_sub", - "exp": new_future_exp, - }, - ] - with patch.object( - mock_forge_access_point, "get_access_token", return_value="new_test_token" - ): - with caplog.at_level("INFO"): - exp_timestamp = mock_forge_access_point.refresh_token() - assert math.isclose(exp_timestamp, new_future_exp, abs_tol=0.1) - assert "Nexus access token has expired, refreshing token..." in caplog.text - - -@pytest.fixture -def mock_get_brain_region_resolve(): - with patch( - "bluepyemodel.access_point.forge_access_point.ontology_forge_access_point" - ) as mock_ontology_access_point: - mock_access_point = Mock() - mock_ontology_access_point.return_value = mock_access_point - - def resolve(brain_region, strategy, **kwargs): - if brain_region.lower() in [ - "somatosensory areas", - "basic cell groups and regions", - "mock_brain_region", - ]: - mock_resource = Mock() - mock_resource.id = "mock_id" - mock_resource.label = "mock_label" - return mock_resource - return None - - mock_access_point.resolve = resolve - yield mock_ontology_access_point, mock_access_point - - -def test_get_brain_region_found(mock_get_brain_region_resolve): - """ - Test get_brain_region function when the brain region is found. - """ - resource = get_brain_region("SSCX", access_token="test_token") - assert resource.id == "mock_id" - assert resource.label == "mock_label" - - -def test_get_brain_region_not_found(mock_get_brain_region_resolve): - """ - Test get_brain_region function when the brain region is not found. - """ - with pytest.raises( - AccessPointException, match=r"Could not find any brain region with name UnknownRegion" - ): - get_brain_region("UnknownRegion", access_token="test_token") - - -def test_get_brain_region_dict(mock_get_brain_region_resolve): - """ - Test get_brain_region_dict function to ensure it returns the correct dictionary. - """ - _, mock_access_point = mock_get_brain_region_resolve - - mock_access_point.forge.as_json.return_value = {"id": "mock_id", "label": "mock_label"} - - result = get_brain_region_dict("SSCX", access_token="test_token") - assert result["id"] == "mock_id" - assert result["label"] == "mock_label"