From cf4eff9b254e04c006638b2c5c5b88ead5089cb8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 3 Dec 2024 14:47:38 +0100 Subject: [PATCH 01/67] Add an experimental non-cheetah way of building command lines The primary benefit is that the command section does not have access the app or any dangling reference to the database connection or any other secrets. There are two flavors here, one uses base_command and arguments, and allows building up an (escaped) argv list, the other is a shortcut for writing shell scripts and feels maybe a bit more like writing a very simple cheetah section. base_command: ```yml name: base_command tool class: GalaxyTool version: 1.0.0 base_command: cat arguments: - $(inputs.input.path) - '>' - output.fastq inputs: - type: data name: input outputs: output: type: data from_work_dir: output.fastq name: output ``` shell_command style: ```yml name: shell_command tool class: GalaxyTool version: 1.0.0 shell_command: cat '$(inputs.input.path)' > output.fastq inputs: - type: data name: input outputs: output: type: data from_work_dir: output.fastq name: output ``` --- lib/galaxy/jobs/__init__.py | 8 +- lib/galaxy/tool_util/parser/interface.py | 12 +++ lib/galaxy/tool_util/parser/yaml.py | 11 +++ lib/galaxy/tools/__init__.py | 27 ++++-- lib/galaxy/tools/evaluation.py | 97 ++++++++++++++++++---- lib/galaxy/tools/expressions/evaluation.py | 8 +- lib/galaxy/workflow/modules.py | 20 +++-- lib/galaxy_test/api/test_tools.py | 73 ++++++++++++++++ 8 files changed, 224 insertions(+), 32 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 8dafd0d8b365..c77a07f7da3f 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -86,6 +86,7 @@ from galaxy.tools.evaluation import ( PartialToolEvaluator, ToolEvaluator, + UserToolEvaluator, ) from galaxy.util import ( parse_xml_string, @@ -1374,7 +1375,12 @@ def _load_job(self): return job def _get_tool_evaluator(self, job): - klass = PartialToolEvaluator if self.remote_command_line else ToolEvaluator + if self.remote_command_line: + klass = PartialToolEvaluator + elif self.tool.base_command or self.tool.shell_command: + klass = UserToolEvaluator + else: + klass = ToolEvaluator tool_evaluator = klass( app=self.app, job=job, diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 21db34d203f8..44fe7653bebc 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -226,6 +226,18 @@ def parse_request_param_translation_elem(self): def parse_command(self): """Return string contianing command to run.""" + def parse_shell_command(self) -> Optional[str]: + """Return string that after input binding can be executed.""" + return None + + def parse_base_command(self) -> Optional[List[str]]: + """Return string containing script entrypoint.""" + return None + + def parse_arguments(self) -> Optional[List[str]]: + """Return list of strings to append to base_command.""" + return None + def parse_expression(self): """Return string contianing command to run.""" return None diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index 88be9c72846a..e5a745e8756f 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -16,6 +16,7 @@ DEFAULT_DELTA_FRAC, DEFAULT_SORT, ) +from galaxy.util import listify from .interface import ( AssertionDict, AssertionList, @@ -97,6 +98,16 @@ def parse_command(self): def parse_expression(self): return self.root_dict.get("expression") + def parse_shell_command(self) -> Optional[str]: + return self.root_dict.get("shell_command") + + def parse_base_command(self) -> Optional[List[str]]: + """Return string containing script entrypoint.""" + return listify(self.root_dict.get("base_command")) + + def parse_arguments(self) -> Optional[List[str]]: + return self.root_dict.get("arguments") + def parse_environment_variables(self): return [] diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a278c13e3b80..1dde1bc75091 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -849,6 +849,10 @@ def __init__( self.tool_source = tool_source self.outputs: Dict[str, ToolOutput] = {} self.output_collections: Dict[str, ToolOutputCollection] = {} + self.command: Optional[str] = None + self.base_command: Optional[List[str]] = None + self.arguments: Optional[List[str]] = [] + self.shell_command: Optional[str] = None self._is_workflow_compatible = None self.__help = None self.__tests: Optional[str] = None @@ -1091,6 +1095,9 @@ def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bo self.input_translator = None self.parse_command(tool_source) + self.parse_shell_command(tool_source) + self.parse_base_command(tool_source) + self.parse_arguments(tool_source) self.environment_variables = self.parse_environment_variables(tool_source) self.tmp_directory_vars = tool_source.parse_tmp_directory_vars() @@ -1421,6 +1428,15 @@ def parse_command(self, tool_source): self.command = "" self.interpreter = None + def parse_shell_command(self, tool_source: ToolSource): + self.shell_command = tool_source.parse_shell_command() + + def parse_base_command(self, tool_source: ToolSource): + self.base_command = tool_source.parse_base_command() + + def parse_arguments(self, tool_source: ToolSource): + self.arguments = tool_source.parse_arguments() + def parse_environment_variables(self, tool_source): return tool_source.parse_environment_variables() @@ -2453,11 +2469,12 @@ def to_archive(self): tool_tup = (os.path.abspath(self.config_file), os.path.split(self.config_file)[-1]) tarball_files.append(tool_tup) # TODO: This feels hacky. - tool_command = self.command.strip().split()[0] - tool_path = os.path.dirname(os.path.abspath(self.config_file)) - # Add the tool XML to the tuple that will be used to populate the tarball. - if os.path.exists(os.path.join(tool_path, tool_command)): - tarball_files.append((os.path.join(tool_path, tool_command), tool_command)) + if self.command: + tool_command = self.command.strip().split()[0] + tool_path = os.path.dirname(os.path.abspath(self.config_file)) + # Add the tool XML to the tuple that will be used to populate the tarball. + if os.path.exists(os.path.join(tool_path, tool_command)): + tarball_files.append((os.path.join(tool_path, tool_command), tool_command)) # Find and add macros and code files. for external_file in self.get_externally_referenced_paths(os.path.abspath(self.config_file)): external_file_abspath = os.path.abspath(os.path.join(tool_path, external_file)) diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 1d1a11914fc1..75385b166ee4 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -34,6 +34,7 @@ ) from galaxy.tool_util.data import TabularToolDataTable from galaxy.tools.actions import determine_output_format +from galaxy.tools.expressions import do_eval from galaxy.tools.parameters import ( visit_input_values, wrapped_json, @@ -131,6 +132,7 @@ class ToolEvaluator: app: MinimalToolApp job: model.Job materialize_datasets: bool = True + param_dict_style = "regular" def __init__(self, app: MinimalToolApp, tool: "Tool", job, local_working_directory): self.app = app @@ -173,25 +175,35 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s # replace materialized objects back into tool input parameters self._replaced_deferred_objects(inp_data, incoming, materialized_objects) - if get_special: - special = get_special() - if special: - out_data["output_file"] = special - - # These can be passed on the command line if wanted as $__user_*__ - incoming.update(model.User.user_template_environment(self._user)) - - # Build params, done before hook so hook can use - self.param_dict = self.build_param_dict( - incoming, - inp_data, - out_data, - output_collections=out_collections, - ) # late update of format_source outputs self._eval_format_source(job, inp_data, out_data) - self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming) + if self.param_dict_style == "regular": + if get_special: + special = get_special() + if special: + out_data["output_file"] = special + + # These can be passed on the command line if wanted as $__user_*__ + + incoming.update(model.User.user_template_environment(self._user)) + + # Build params, done before hook so hook can use + self.param_dict = self.build_param_dict( + incoming, + inp_data, + out_data, + output_collections=out_collections, + ) + self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming) + + else: + self.param_dict = self.build_param_dict( + incoming, + inp_data, + out_data, + output_collections=out_collections, + ) def execute_tool_hooks(self, inp_data, out_data, incoming): # Certain tools require tasks to be completed prior to job execution @@ -909,6 +921,59 @@ def build(self): ) +class UserToolEvaluator(ToolEvaluator): + + param_dict_style = "json" + + def _build_config_files(self): + pass + + def _build_param_file(self): + pass + + def _build_version_command(self): + pass + + def __sanitize_param_dict(self, param_dict): + pass + + def build_param_dict(self, incoming, input_datasets, output_datasets, output_collections): + """ + Build the dictionary of parameters for substituting into the command + line. We're effecively building the CWL job object here. + """ + compute_environment = self.compute_environment + job_working_directory = compute_environment.working_directory() + from galaxy.workflow.modules import to_cwl + + hda_references: List[model.HistoryDatasetAssociation] = [] + cwl_style_inputs = to_cwl(incoming, hda_references=hda_references) + return {"inputs": cwl_style_inputs, "outdir": job_working_directory} + + def _build_command_line(self): + if self.tool.base_command: + base_command = self.tool.base_command + arguments = self.tool.arguments + bound_arguments = [*base_command] + for argument in arguments: + if ( + bound_argument := do_eval(argument, self.param_dict["inputs"], outdir=self.param_dict["outdir"]) + ) != argument: + # variables will be shell-escaped, but you can of course still + # write invalid things into the literal portion of the arguments. + # The upside is that we can use `>`, `|`. + # Maybe we should wrap this in `sh -c` or something like that though. + bound_argument = shlex.quote(str(bound_argument)) + if bound_argument is not None: + bound_arguments.append(bound_argument) + command_line = " ".join(bound_arguments) + elif self.tool.shell_command: + command_line = do_eval(self.tool.shell_command, self.param_dict["inputs"], outdir=self.param_dict["outdir"]) + else: + raise Exception("Tool must define shell_command or base_command") + self.command_line = command_line + + class RemoteToolEvaluator(ToolEvaluator): """ToolEvaluator that skips unnecessary steps already executed during job setup.""" diff --git a/lib/galaxy/tools/expressions/evaluation.py b/lib/galaxy/tools/expressions/evaluation.py index e5ec9636f2c3..13e195669dec 100644 --- a/lib/galaxy/tools/expressions/evaluation.py +++ b/lib/galaxy/tools/expressions/evaluation.py @@ -20,7 +20,13 @@ NODE_ENGINE = os.path.join(FILE_DIRECTORY, "cwlNodeEngine.js") -def do_eval(expression: str, jobinput: "CWLObjectType", context: Optional["CWLOutputType"] = None): +def do_eval( + expression: str, + jobinput: "CWLObjectType", + outdir: Optional[str] = None, + tmpdir: Optional[str] = None, + context: Optional["CWLOutputType"] = None, +): return _do_eval( expression, jobinput, diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 08d65f6206cc..b174a8d2f57f 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -135,7 +135,7 @@ class ConditionalStepWhen(BooleanToolParameter): pass -def to_cwl(value, hda_references, step): +def to_cwl(value, hda_references, step: Optional[WorkflowStep] = None): element_identifier = None if isinstance(value, model.HistoryDatasetCollectionAssociation): value = value.collection @@ -145,15 +145,16 @@ def to_cwl(value, hda_references, step): if isinstance(value, model.HistoryDatasetAssociation): # I think the following two checks are needed but they may # not be needed. - if not value.dataset.in_ready_state(): - why = f"dataset [{value.id}] is needed for valueFrom expression and is non-ready" - raise DelayedWorkflowEvaluation(why=why) - if not value.is_ok: - raise FailWorkflowEvaluation( - why=InvocationFailureDatasetFailed( - reason=FailureReason.dataset_failed, hda_id=value.id, workflow_step_id=step.id + if step: + if not value.dataset.in_ready_state(): + why = f"dataset [{value.id}] is needed for valueFrom expression and is non-ready" + raise DelayedWorkflowEvaluation(why=why) + if not value.is_ok: + raise FailWorkflowEvaluation( + why=InvocationFailureDatasetFailed( + reason=FailureReason.dataset_failed, hda_id=value.id, workflow_step_id=step.id + ) ) - ) if value.ext == "expression.json": with open(value.get_file_name()) as f: # OUR safe_loads won't work, will not load numbers, etc... @@ -163,6 +164,7 @@ def to_cwl(value, hda_references, step): properties = { "class": "File", "location": f"step_input://{len(hda_references)}", + "path": value.get_file_name(), } set_basename_and_derived_properties( properties, value.dataset.created_from_basename or element_identifier or value.name diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 1b50605b39e9..0e8255e75c2a 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -57,6 +57,45 @@ output1=dict(format="txt"), ), } +TOOL_WITH_BASE_COMMAND = { + "name": "Base command tool", + "class": "GalaxyTool", + "version": "1.0.0", + "base_command": "cat", + "arguments": ["$(inputs.input.path)", ">", "output.fastq"], + "inputs": [ + { + "type": "data", + "name": "input", + } + ], + "outputs": { + "output": { + "type": "data", + "from_work_dir": "output.fastq", + "name": "output", + } + }, +} +TOOL_WITH_SHELL_COMMAND = { + "name": "Base command tool", + "class": "GalaxyTool", + "version": "1.0.0", + "shell_command": "cat '$(inputs.input.path)' > output.fastq", + "inputs": [ + { + "type": "data", + "name": "input", + } + ], + "outputs": { + "output": { + "type": "data", + "from_work_dir": "output.fastq", + "name": "output", + } + }, +} class TestsTools: @@ -1394,6 +1433,40 @@ def test_dynamic_tool_no_id(self): output_content = self.dataset_populator.get_history_dataset_content(history_id) assert output_content == "Hello World 2\n" + def test_dynamic_tool_base_command(self): + tool_response = self.dataset_populator.create_tool(TOOL_WITH_BASE_COMMAND) + self._assert_has_keys(tool_response, "uuid") + + # Run tool. + history_id = self.dataset_populator.new_history() + dataset = self.dataset_populator.new_dataset(history_id=history_id, content="abc") + self._run( + history_id=history_id, + tool_uuid=tool_response["uuid"], + inputs={"input": {"src": "hda", "id": dataset["id"]}}, + ) + + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + assert output_content == "abc\n" + + def test_dynamic_tool_shell_command(self): + tool_response = self.dataset_populator.create_tool(TOOL_WITH_SHELL_COMMAND) + self._assert_has_keys(tool_response, "uuid") + + # Run tool. + history_id = self.dataset_populator.new_history() + dataset = self.dataset_populator.new_dataset(history_id=history_id, content="abc") + self._run( + history_id=history_id, + tool_uuid=tool_response["uuid"], + inputs={"input": {"src": "hda", "id": dataset["id"]}}, + ) + + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + assert output_content == "abc\n" + def test_show_dynamic_tools(self): # Create tool. original_list = self.dataset_populator.list_dynamic_tools() From b07561673d89cedf94180e936275737f919b5ee6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 4 Dec 2024 14:05:56 +0100 Subject: [PATCH 02/67] Create UserToolSource pydantic model --- lib/galaxy/managers/tools.py | 16 +-- lib/galaxy/tool_util/models.py | 20 ++++ lib/galaxy/tool_util/parameters/__init__.py | 2 + lib/galaxy/tool_util/parameters/factory.py | 7 ++ lib/galaxy/tool_util/parameters/models.py | 7 ++ lib/galaxy/tool_util/parser/factory.py | 2 +- lib/galaxy/tool_util/parser/output_models.py | 73 +++++++++--- lib/galaxy/tool_util/parser/output_objects.py | 1 + lib/galaxy/tool_util/parser/yaml.py | 11 +- .../webapps/galaxy/api/dynamic_tools.py | 107 +++++++----------- lib/galaxy/webapps/galaxy/buildapp.py | 1 - lib/galaxy_test/api/test_tools.py | 12 +- lib/galaxy_test/base/populators.py | 6 +- 13 files changed, 165 insertions(+), 100 deletions(-) diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index 25ce50f9fa04..ab770043e981 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -57,7 +57,7 @@ def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any] ) dynamic_tool = None - uuid_str = tool_payload.get("uuid") + uuid_str = tool_payload.uuid # Convert uuid_str to UUID or generate new if None uuid = model.get_uuid(uuid_str) if uuid_str: @@ -69,14 +69,14 @@ def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any] raise DuplicatedIdentifierException(dynamic_tool.id) assert dynamic_tool.uuid == uuid if not dynamic_tool: - src = tool_payload.get("src", "representation") + src = tool_payload.src is_path = src == "from_path" if is_path: tool_format, representation, _ = artifact_class(None, tool_payload) else: assert src == "representation" - representation = tool_payload.get("representation") + representation = tool_payload.representation.dict(by_alias=True, exclude_unset=True) if not representation: raise exceptions.ObjectAttributeMissingException("A tool 'representation' is required.") @@ -84,9 +84,9 @@ def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any] if not tool_format: raise exceptions.ObjectAttributeMissingException("Current tool representations require 'class'.") - tool_path = tool_payload.get("path") - tool_directory = tool_payload.get("tool_directory") - if tool_format == "GalaxyTool": + tool_path = tool_payload.path + tool_directory = tool_payload.tool_directory + if tool_format in ("GalaxyTool", "GalaxyUserTool"): tool_id = representation.get("id") if not tool_id: tool_id = str(uuid) @@ -113,8 +113,8 @@ def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any] tool_path=tool_path, tool_directory=tool_directory, uuid=uuid, - active=tool_payload.get("active"), - hidden=tool_payload.get("hidden"), + active=tool_payload.active, + hidden=tool_payload.hidden, value=representation, ) self.app.toolbox.load_dynamic_tool(dynamic_tool) diff --git a/lib/galaxy/tool_util/models.py b/lib/galaxy/tool_util/models.py index 858b50938942..f6bb9058f068 100644 --- a/lib/galaxy/tool_util/models.py +++ b/lib/galaxy/tool_util/models.py @@ -28,6 +28,7 @@ ) from .parameters import ( + GalaxyParameterT, input_models_for_tool_source, ToolParameterT, ) @@ -40,11 +41,30 @@ ) from .parser.output_models import ( from_tool_source, + IncomingToolOutput, ToolOutput, ) from .verify.assertion_models import assertions +class UserToolSource(BaseModel): + class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")] + name: str + container: str + version: str + description: Optional[str] = None + shell_command: str + inputs: List[GalaxyParameterT] + outputs: List[IncomingToolOutput] + citations: Optional[List[Citation]] = None + license: Optional[str] = None + profile: Optional[str] = None + edam_operations: Optional[List[str]] = None + edam_topics: Optional[List[str]] = None + xrefs: Optional[List[XrefDict]] = None + help: Optional[HelpContent] = None + + class ParsedTool(BaseModel): id: str version: Optional[str] diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py index 22dd7e6053aa..d6d2b9b8431f 100644 --- a/lib/galaxy/tool_util/parameters/__init__.py +++ b/lib/galaxy/tool_util/parameters/__init__.py @@ -41,6 +41,7 @@ DataRequestInternalHda, DataRequestUri, FloatParameterModel, + GalaxyParameterT, HiddenParameterModel, IntegerParameterModel, LabelValue, @@ -112,6 +113,7 @@ "CwlUnionParameterModel", "CwlBooleanParameterModel", "CwlDirectoryParameterModel", + "GalaxyParameterT", "TextParameterModel", "FloatParameterModel", "HiddenParameterModel", diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index 932195effb9d..7cc60778eb27 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -107,6 +107,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool min_int = int(min_raw) if min_raw is not None else None max_int = int(max_raw) if max_raw is not None else None return IntegerParameterModel( + type="integer", name=input_source.parse_name(), optional=optional, value=int_value, @@ -118,6 +119,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool nullable = input_source.parse_optional() value = input_source.get_bool_or_none("checked", None if nullable else False) return BooleanParameterModel( + type="boolean", name=input_source.parse_name(), optional=nullable, value=value, @@ -126,6 +128,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool optional, optionality_inferred = text_input_is_optional(input_source) text_validators: List[TextCompatiableValidators] = _text_validators(input_source) return TextParameterModel( + type="text", name=input_source.parse_name(), optional=optional, validators=text_validators, @@ -150,6 +153,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool min_float = float(min_raw) if min_raw is not None else None max_float = float(max_raw) if max_raw is not None else None return FloatParameterModel( + type="float", name=input_source.parse_name(), optional=optional, value=float_value, @@ -170,6 +174,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool elif param_type == "color": optional = input_source.parse_optional() return ColorParameterModel( + type="color", name=input_source.parse_name(), optional=optional, value=get_color_value(input_source), @@ -182,6 +187,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool optional = input_source.parse_optional() multiple = input_source.get_bool("multiple", False) return DataParameterModel( + type="data", name=input_source.parse_name(), optional=optional, multiple=multiple, @@ -190,6 +196,7 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool optional = input_source.parse_optional() default_value = input_source.parse_default() return DataCollectionParameterModel( + type="data_collection", name=input_source.parse_name(), optional=optional, value=default_value, diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 1feae64168f2..1b0168bc61de 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -238,6 +238,7 @@ def validator(v: Any) -> Any: class TextParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_text"] = "gx_text" + type: Literal["text"] area: bool = False default_value: Optional[str] = Field(default=None, alias="value") default_options: List[LabelValue] = [] @@ -266,6 +267,7 @@ def request_requires_value(self) -> bool: class IntegerParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_integer"] = "gx_integer" + type: Literal["integer"] optional: bool value: Optional[int] = None min: Optional[int] = None @@ -298,6 +300,7 @@ def request_requires_value(self) -> bool: class FloatParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_float"] = "gx_float" + type: Literal["float"] value: Optional[float] = None min: Optional[float] = None max: Optional[float] = None @@ -441,6 +444,7 @@ class BatchDataInstanceInternal(StrictModel): class DataParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_data"] = "gx_data" + type: Literal["data"] extensions: List[str] = ["data"] multiple: bool = False min: Optional[int] = None @@ -529,6 +533,7 @@ class DataCollectionRequestInternal(StrictModel): class DataCollectionParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_data_collection"] = "gx_data_collection" + type: Literal["data_collection"] collection_type: Optional[str] = None extensions: List[str] = ["data"] value: Optional[Dict[str, Any]] @@ -616,6 +621,7 @@ def ensure_color_valid(value: Optional[Any]): class ColorParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_color"] = "gx_color" + type: Literal["color"] value: Optional[str] = None @property @@ -665,6 +671,7 @@ def request_requires_value(self) -> bool: class BooleanParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_boolean"] = "gx_boolean" + type: Literal["boolean"] value: Optional[bool] = False truevalue: Optional[str] = None falsevalue: Optional[str] = None diff --git a/lib/galaxy/tool_util/parser/factory.py b/lib/galaxy/tool_util/parser/factory.py index 8beb0dc0432f..3f37c57c7370 100644 --- a/lib/galaxy/tool_util/parser/factory.py +++ b/lib/galaxy/tool_util/parser/factory.py @@ -112,7 +112,7 @@ def get_tool_source( def get_tool_source_from_representation(tool_format: Optional[str], tool_representation: Dict[str, Any]): # TODO: make sure whatever is consuming this method uses ordered load. log.info("Loading dynamic tool - this is experimental - tool may not function in future.") - if tool_format == "GalaxyTool": + if tool_format in ("GalaxyTool", "GalaxyUserTool"): if "version" not in tool_representation: tool_representation["version"] = "1.0.0" # Don't require version for embedded tools. return YamlToolSource(tool_representation) diff --git a/lib/galaxy/tool_util/parser/output_models.py b/lib/galaxy/tool_util/parser/output_models.py index d72c653de28c..453fb56c7e80 100644 --- a/lib/galaxy/tool_util/parser/output_models.py +++ b/lib/galaxy/tool_util/parser/output_models.py @@ -6,6 +6,7 @@ """ from typing import ( + Generic, List, Optional, Sequence, @@ -19,39 +20,70 @@ from typing_extensions import ( Annotated, Literal, + TypeVar, ) from .interface import ToolSource +AnyT = TypeVar("AnyT") +NotRequired = Annotated[Optional[AnyT], Field(None)] +IncomingNotRequiredBoolT = TypeVar("IncomingNotRequiredBoolT") +IncomingNotRequiredStringT = TypeVar("IncomingNotRequiredStringT") +IncomingNotRequiredDatasetCollectionDescriptionT = TypeVar("IncomingNotRequiredDatasetCollectionDescriptionT") +IncomingNotRequiredDatasetCollectionDescription = NotRequired[List["DatasetCollectionDescriptionT"]] -class ToolOutputBaseModel(BaseModel): - name: str - label: Optional[str] - hidden: bool +# Use IncomingNotRequired when concrete key: Optional[str] = None would be incorrect -class ToolOutputDataset(ToolOutputBaseModel): +class ToolOutputBaseModelG(BaseModel, Generic[IncomingNotRequiredBoolT, IncomingNotRequiredStringT]): + name: IncomingNotRequiredStringT + label: Optional[str] = None + hidden: IncomingNotRequiredBoolT + + +IncomingToolOutputBaseModel = ToolOutputBaseModelG[NotRequired[bool], NotRequired[str]] + + +class ToolOutputDatasetG( + ToolOutputBaseModelG[IncomingNotRequiredBoolT, IncomingNotRequiredStringT], + Generic[IncomingNotRequiredBoolT, IncomingNotRequiredStringT], +): type: Literal["data"] - format: str - format_source: Optional[str] - metadata_source: Optional[str] - discover_datasets: Optional[List["DatasetCollectionDescriptionT"]] + format: IncomingNotRequiredStringT + format_source: Optional[str] = None + metadata_source: Optional[str] = None + discover_datasets: Optional[List["DatasetCollectionDescriptionT"]] = None + from_work_dir: Optional[str] = None + + +ToolOutputDataset = ToolOutputDatasetG[bool, str] +IncomingToolOutputDataset = ToolOutputDatasetG[ + NotRequired[bool], + NotRequired[str], +] class ToolOutputCollectionStructure(BaseModel): - collection_type: Optional[str] - collection_type_source: Optional[str] - collection_type_from_rules: Optional[str] - structured_like: Optional[str] - discover_datasets: Optional[List["DatasetCollectionDescriptionT"]] + collection_type: Optional[str] = None + collection_type_source: Optional[str] = None + collection_type_from_rules: Optional[str] = None + structured_like: Optional[str] = None + discover_datasets: Optional[List["DatasetCollectionDescriptionT"]] = None -class ToolOutputCollection(ToolOutputBaseModel): +class ToolOutputCollectionG( + ToolOutputBaseModelG[IncomingNotRequiredBoolT, IncomingNotRequiredStringT], + Generic[IncomingNotRequiredBoolT, IncomingNotRequiredStringT], +): type: Literal["collection"] structure: ToolOutputCollectionStructure -class ToolOutputSimple(ToolOutputBaseModel): +ToolOutputCollection = ToolOutputCollectionG[bool, str] +IncomingToolOutputCollection = ToolOutputCollectionG[NotRequired[bool], NotRequired[str]] + + +class ToolOutputSimple(ToolOutputBaseModelG): pass @@ -100,6 +132,15 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription): DatasetCollectionDescriptionT = Union[FilePatternDatasetCollectionDescription, ToolProvidedMetadataDatasetCollection] +IncomingToolOutputT = Union[ + IncomingToolOutputDataset, + IncomingToolOutputCollection, + ToolOutputText, + ToolOutputInteger, + ToolOutputFloat, + ToolOutputBoolean, +] +IncomingToolOutput = Annotated[IncomingToolOutputT, Field(discriminator="type")] ToolOutputT = Union[ ToolOutputDataset, ToolOutputCollection, ToolOutputText, ToolOutputInteger, ToolOutputFloat, ToolOutputBoolean ] diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index bedff968f89a..047b25d8480e 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -160,6 +160,7 @@ def to_model(self) -> ToolOutputDataModel: format_source=self.format_source, metadata_source=self.metadata_source or None, # model is decorated as Optional discover_datasets=[d.to_model() for d in self.dataset_collector_descriptions], + from_work_dir=self.from_work_dir, ) @staticmethod diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index e5a745e8756f..192b0f5b0603 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -1,4 +1,5 @@ import json +from collections.abc import MutableMapping from typing import ( Any, Dict, @@ -148,11 +149,17 @@ def parse_help(self) -> Optional[HelpContent]: return None def parse_outputs(self, tool): - outputs = self.root_dict.get("outputs", {}) + outputs = self.root_dict.get("outputs", []) + if isinstance(outputs, MutableMapping): + for name, output_dict in outputs.items(): + output_dict["name"] = name + outputs = outputs.values() + output_defs = [] output_collection_defs = [] - for name, output_dict in outputs.items(): + for output_dict in outputs: output_type = output_dict.get("type", "data") + name = output_dict["name"] if output_type == "data": output_defs.append(self._parse_output(tool, name, output_dict)) elif output_type == "collection": diff --git a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py index 7866e79c6564..e26a56607ab0 100644 --- a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py +++ b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py @@ -1,86 +1,67 @@ import logging - -from galaxy import ( - util, - web, +from typing import ( + Literal, + Optional, ) + +from pydantic.main import BaseModel + from galaxy.exceptions import ObjectNotFound -from galaxy.web import ( - expose_api, - expose_api_anonymous_and_sessionless, +from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.tools import DynamicToolManager +from galaxy.schema.fields import DecodedDatabaseIdField +from galaxy.tool_util.models import UserToolSource +from . import ( + depends, + DependsOnTrans, + Router, ) -from . import BaseGalaxyAPIController -log = logging.getLogger(__name__) +class DynamicToolCreatePayload(BaseModel): + allow_load: bool = True + uuid: Optional[str] = None + src: Literal["representation", "path"] = "representation" + representation: UserToolSource + active: Optional[bool] = None + hidden: Optional[bool] = None + path: Optional[str] = None + tool_directory: Optional[str] = None -class DynamicToolsController(BaseGalaxyAPIController): - """ - RESTful controller for interactions with dynamic tools. - Dynamic tools are tools defined in the database. Use the tools controller - to run these tools and view functional information. - """ +log = logging.getLogger(__name__) - @expose_api_anonymous_and_sessionless - def index(self, trans, **kwds): - """ - GET /api/dynamic_tools +router = Router(tags=["dynamic_tools"]) - This returns meta-information about the dynamic tool, such as - tool_uuid. To use the tool or view funtional information such as - inputs and outputs, use the standard tools API indexed by the - ID (and optionally version) returned from this endpoint. - """ - manager = self.app.dynamic_tool_manager - return [t.to_dict() for t in manager.list_tools()] - @expose_api_anonymous_and_sessionless - def show(self, trans, id, **kwd): - """ - GET /api/dynamic_tools/{encoded_dynamic_tool_id|tool_uuid} - """ - return self._get_dynamic_tool(trans, id).to_dict() +@router.cbv +class DynamicToolApi: + dynamic_tools_manager: DynamicToolManager = depends(DynamicToolManager) - @web.require_admin - @expose_api - def create(self, trans, payload, **kwd): - """ - POST /api/dynamic_tools + @router.get("/api/dynamic_tools") + def index(self): + return [t.to_dict() for t in self.dynamic_tools_manager.list_tools()] - The payload is expected to be a tool definition to dynamically load - into Galaxy's toolbox. + @router.get("/api/dynamic_tools/{dynamic_tool_id}") + def show(self, dynamic_tool_id: DecodedDatabaseIdField): + dynamic_tool = self.dynamic_tools_manager.get_tool_by_id(dynamic_tool_id) + if dynamic_tool is None: + raise ObjectNotFound() + return dynamic_tool.to_dict() - :type representation: dict - :param representation: a JSON-ified tool description to load - :type uuid: str - :param uuid: the uuid to associate with the tool being created - """ - dynamic_tool = self.app.dynamic_tool_manager.create_tool( - trans, payload, allow_load=util.asbool(kwd.get("allow_load", True)) - ) + @router.post("/api/dynamic_tools", require_admin=True) + def create(self, payload: DynamicToolCreatePayload, trans: ProvidesUserContext = DependsOnTrans): + dynamic_tool = self.dynamic_tools_manager.create_tool(trans, payload, allow_load=payload.allow_load) return dynamic_tool.to_dict() - @web.require_admin - @expose_api - def delete(self, trans, id, **kwd): + @router.delete("/api/dynamic_tools/{dynamic_tool_id}") + def delete(self, trans, dynamic_tool_id: DecodedDatabaseIdField, **kwd): """ DELETE /api/dynamic_tools/{encoded_dynamic_tool_id|tool_uuid} Deactivate the specified dynamic tool. Deactivated tools will not be loaded into the toolbox. """ - manager = self.app.dynamic_tool_manager - dynamic_tool = self._get_dynamic_tool(trans, id) - updated_dynamic_tool = manager.deactivate(dynamic_tool) + dynamic_tool = dynamic_tool = self.dynamic_tools_manager.get_tool_by_id(dynamic_tool_id) + updated_dynamic_tool = self.dynamic_tools_manager.deactivate(dynamic_tool) return updated_dynamic_tool.to_dict() - - def _get_dynamic_tool(self, trans, request_id): - manager = self.app.dynamic_tool_manager - if util.is_uuid(request_id): - dynamic_tool = manager.get_tool_by_uuid(request_id) - else: - dynamic_tool = manager.get_tool_by_id(trans.security.decode_id(request_id)) - if dynamic_tool is None: - raise ObjectNotFound() - return dynamic_tool diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 9c1f9de277e6..aceea3465a5a 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -430,7 +430,6 @@ def populate_api_routes(webapp, app): ) webapp.mapper.connect("/api/tools/{id:.+?}", action="show", controller="tools") webapp.mapper.resource("tool", "tools", path_prefix="/api") - webapp.mapper.resource("dynamic_tools", "dynamic_tools", path_prefix="/api") webapp.mapper.connect( "/api/sanitize_allow", action="index", controller="sanitize_allow", conditions=dict(method=["GET"]) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 0e8255e75c2a..4e2148b11f7c 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -79,22 +79,24 @@ } TOOL_WITH_SHELL_COMMAND = { "name": "Base command tool", - "class": "GalaxyTool", + "class": "GalaxyUserTool", + "container": "busybox", "version": "1.0.0", "shell_command": "cat '$(inputs.input.path)' > output.fastq", "inputs": [ { "type": "data", "name": "input", + "format": "txt", } ], - "outputs": { - "output": { + "outputs": [ + { "type": "data", "from_work_dir": "output.fastq", "name": "output", } - }, + ], } @@ -1387,7 +1389,7 @@ def test_dynamic_list_output_datasets_in_failed_state(self, history_id): def test_nonadmin_users_cannot_create_tools(self): payload = dict( - representation=json.dumps(MINIMAL_TOOL), + representation=MINIMAL_TOOL, ) create_response = self._post("dynamic_tools", data=payload, admin=False) self._assert_status_code_is(create_response, 403) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index ee32c27b822c..4065bcd488a5 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -838,8 +838,6 @@ def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]: return self._create_tool_raw(payload) def create_tool(self, representation, tool_directory: Optional[str] = None) -> Dict[str, Any]: - if isinstance(representation, dict): - representation = json.dumps(representation) payload = dict( representation=representation, tool_directory=tool_directory, @@ -849,9 +847,9 @@ def create_tool(self, representation, tool_directory: Optional[str] = None) -> D def _create_tool_raw(self, payload) -> Dict[str, Any]: using_requirement("admin") try: - create_response = self._post("dynamic_tools", data=payload, admin=True) + create_response = self._post("dynamic_tools", data=payload, admin=True, json=True) except TypeError: - create_response = self._post("dynamic_tools", data=payload) + create_response = self._post("dynamic_tools", data=payload, json=True) assert create_response.status_code == 200, create_response.text return create_response.json() From 702fc945ad816726c3cb0a066891101e5c133dab Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 5 Dec 2024 19:29:18 +0100 Subject: [PATCH 03/67] Minor fix to restore admin-style dynamic tools --- lib/galaxy/managers/tools.py | 28 ++++--- lib/galaxy/schema/tools.py | 30 +++++++ lib/galaxy/tool_util/models.py | 46 ++++++++-- .../webapps/galaxy/api/dynamic_tools.py | 35 +++----- lib/galaxy/workflow/modules.py | 5 +- .../api/embed_test_1_tool.gxtool.yml | 1 + lib/galaxy_test/api/test_tools.py | 83 +++++++++---------- lib/galaxy_test/api/test_workflows.py | 2 + .../api/test_workflows_from_yaml.py | 1 + .../base/data/minimal_tool_no_id.json | 3 +- lib/galaxy_test/base/populators.py | 14 ++-- 11 files changed, 151 insertions(+), 97 deletions(-) create mode 100644 lib/galaxy/schema/tools.py diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index ab770043e981..76410af2fd14 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -18,7 +18,9 @@ model, ) from galaxy.exceptions import DuplicatedIdentifierException +from galaxy.managers.context import ProvidesUserContext from galaxy.model import DynamicTool +from galaxy.schema.tools import DynamicToolPayload from galaxy.tool_util.cwl import tool_proxy from .base import ( ModelManager, @@ -38,6 +40,12 @@ class DynamicToolManager(ModelManager[model.DynamicTool]): model_class = model.DynamicTool + def get_tool_by_id_or_uuid(self, id_or_uuid: Union[int, str]): + if isinstance(id_or_uuid, int): + return self.get_tool_by_id(id_or_uuid) + else: + return self.get_tool_by_uuid(id_or_uuid) + def get_tool_by_uuid(self, uuid: Optional[Union[UUID, str]]): stmt = select(DynamicTool).where(DynamicTool.uuid == uuid) return self.session().scalars(stmt).one_or_none() @@ -50,7 +58,7 @@ def get_tool_by_id(self, object_id): stmt = select(DynamicTool).where(DynamicTool.id == object_id) return self.session().scalars(stmt).one_or_none() - def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any], allow_load: bool = True): + def create_tool(self, trans: ProvidesUserContext, tool_payload: DynamicToolPayload, allow_load=True): if not getattr(self.app.config, "enable_beta_tool_formats", False): raise exceptions.ConfigDoesNotAllowException( "Set 'enable_beta_tool_formats' in Galaxy config to create dynamic tools." @@ -66,17 +74,15 @@ def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any] dynamic_tool = self.get_tool_by_uuid(uuid_str) if dynamic_tool: if not allow_load: - raise DuplicatedIdentifierException(dynamic_tool.id) + raise DuplicatedIdentifierException( + f"Attempted to create dynamic tool with duplicate UUID '{uuid_str}'" + ) assert dynamic_tool.uuid == uuid if not dynamic_tool: - src = tool_payload.src - is_path = src == "from_path" - - if is_path: - tool_format, representation, _ = artifact_class(None, tool_payload) + if tool_payload.src == "from_path": + tool_format, representation, _ = artifact_class(None, tool_payload.model_dump()) else: - assert src == "representation" - representation = tool_payload.representation.dict(by_alias=True, exclude_unset=True) + representation = tool_payload.representation.model_dump(by_alias=True, exclude_unset=True) if not representation: raise exceptions.ObjectAttributeMissingException("A tool 'representation' is required.") @@ -84,15 +90,15 @@ def create_tool(self, trans: "ProvidesUserContext", tool_payload: Dict[str, Any] if not tool_format: raise exceptions.ObjectAttributeMissingException("Current tool representations require 'class'.") - tool_path = tool_payload.path tool_directory = tool_payload.tool_directory + tool_path = tool_payload.path if tool_payload.src == "from_path" else None if tool_format in ("GalaxyTool", "GalaxyUserTool"): tool_id = representation.get("id") if not tool_id: tool_id = str(uuid) elif tool_format in ("CommandLineTool", "ExpressionTool"): # CWL tools - if is_path: + if tool_path: proxy = tool_proxy(tool_path=tool_path, uuid=uuid) else: # Build a tool proxy so that we can convert to the persistable diff --git a/lib/galaxy/schema/tools.py b/lib/galaxy/schema/tools.py new file mode 100644 index 000000000000..8d551a292775 --- /dev/null +++ b/lib/galaxy/schema/tools.py @@ -0,0 +1,30 @@ +from typing import ( + Literal, + Optional, + Union, +) + +from pydantic import BaseModel + +from galaxy.tool_util.models import DynamicToolSources + + +class BaseDynamicToolCreatePayload(BaseModel): + allow_load: bool = True + uuid: Optional[str] = None + active: Optional[bool] = None + hidden: Optional[bool] = None + tool_directory: Optional[str] = None + + +class DynamicToolCreatePayload(BaseDynamicToolCreatePayload): + src: Literal["representation"] = "representation" + representation: DynamicToolSources + + +class PathBasedDynamicToolCreatePayload(BaseDynamicToolCreatePayload): + src: Literal["from_path"] + path: str + + +DynamicToolPayload = Union[DynamicToolCreatePayload, PathBasedDynamicToolCreatePayload] diff --git a/lib/galaxy/tool_util/models.py b/lib/galaxy/tool_util/models.py index f6bb9058f068..09229cc2b930 100644 --- a/lib/galaxy/tool_util/models.py +++ b/lib/galaxy/tool_util/models.py @@ -18,6 +18,7 @@ BaseModel, ConfigDict, Field, + model_validator, RootModel, ) from typing_extensions import ( @@ -47,15 +48,22 @@ from .verify.assertion_models import assertions -class UserToolSource(BaseModel): - class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")] - name: str - container: str - version: str +def normalize_dict(values, keys: List[str]): + for key in keys: + items = values.get(key) + if isinstance(items, dict): # dict-of-dicts format + # Transform dict-of-dicts to list-of-dicts + values[key] = [{"name": k, **v} for k, v in items.items()] + + +class ToolSourceBase(BaseModel): + id: Optional[str] = None + name: Optional[str] = None + version: Optional[str] = "1.0" description: Optional[str] = None - shell_command: str - inputs: List[GalaxyParameterT] - outputs: List[IncomingToolOutput] + container: Optional[str] = None + inputs: List[GalaxyParameterT] = [] + outputs: List[IncomingToolOutput] = [] citations: Optional[List[Citation]] = None license: Optional[str] = None profile: Optional[str] = None @@ -64,6 +72,28 @@ class UserToolSource(BaseModel): xrefs: Optional[List[XrefDict]] = None help: Optional[HelpContent] = None + @model_validator(mode="before") + @classmethod + def normalize_items(cls, values): + if isinstance(values, dict): + normalize_dict(values, ["inputs", "outputs"]) + return values + + +class UserToolSource(ToolSourceBase): + class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")] + name: str + shell_command: str + container: str + + +class AdminToolSource(ToolSourceBase): + class_: Annotated[Literal["GalaxyTool"], Field(alias="class")] + command: str + + +DynamicToolSources = Annotated[Union[UserToolSource, AdminToolSource], Field(discriminator="class_")] + class ParsedTool(BaseModel): id: str diff --git a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py index e26a56607ab0..52a2f29e62f2 100644 --- a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py +++ b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py @@ -1,38 +1,23 @@ import logging -from typing import ( - Literal, - Optional, -) - -from pydantic.main import BaseModel +from typing import Union from galaxy.exceptions import ObjectNotFound from galaxy.managers.context import ProvidesUserContext from galaxy.managers.tools import DynamicToolManager from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.tool_util.models import UserToolSource +from galaxy.schema.tools import DynamicToolPayload from . import ( depends, DependsOnTrans, Router, ) - -class DynamicToolCreatePayload(BaseModel): - allow_load: bool = True - uuid: Optional[str] = None - src: Literal["representation", "path"] = "representation" - representation: UserToolSource - active: Optional[bool] = None - hidden: Optional[bool] = None - path: Optional[str] = None - tool_directory: Optional[str] = None - - log = logging.getLogger(__name__) router = Router(tags=["dynamic_tools"]) +DatabaseIdOrUUID = Union[DecodedDatabaseIdField, str] + @router.cbv class DynamicToolApi: @@ -43,25 +28,25 @@ def index(self): return [t.to_dict() for t in self.dynamic_tools_manager.list_tools()] @router.get("/api/dynamic_tools/{dynamic_tool_id}") - def show(self, dynamic_tool_id: DecodedDatabaseIdField): - dynamic_tool = self.dynamic_tools_manager.get_tool_by_id(dynamic_tool_id) + def show(self, dynamic_tool_id: Union[DatabaseIdOrUUID, str]): + dynamic_tool = self.dynamic_tools_manager.get_tool_by_id_or_uuid(dynamic_tool_id) if dynamic_tool is None: raise ObjectNotFound() return dynamic_tool.to_dict() @router.post("/api/dynamic_tools", require_admin=True) - def create(self, payload: DynamicToolCreatePayload, trans: ProvidesUserContext = DependsOnTrans): + def create(self, payload: DynamicToolPayload, trans: ProvidesUserContext = DependsOnTrans): dynamic_tool = self.dynamic_tools_manager.create_tool(trans, payload, allow_load=payload.allow_load) return dynamic_tool.to_dict() - @router.delete("/api/dynamic_tools/{dynamic_tool_id}") - def delete(self, trans, dynamic_tool_id: DecodedDatabaseIdField, **kwd): + @router.delete("/api/dynamic_tools/{dynamic_tool_id}", require_admin=True) + def delete(self, dynamic_tool_id: DatabaseIdOrUUID): """ DELETE /api/dynamic_tools/{encoded_dynamic_tool_id|tool_uuid} Deactivate the specified dynamic tool. Deactivated tools will not be loaded into the toolbox. """ - dynamic_tool = dynamic_tool = self.dynamic_tools_manager.get_tool_by_id(dynamic_tool_id) + dynamic_tool = dynamic_tool = self.dynamic_tools_manager.get_tool_by_id_or_uuid(dynamic_tool_id) updated_dynamic_tool = self.dynamic_tools_manager.deactivate(dynamic_tool) return updated_dynamic_tool.to_dict() diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index b174a8d2f57f..521bbc99e675 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -52,6 +52,7 @@ InvocationFailureWhenNotBoolean, InvocationFailureWorkflowParameterInvalid, ) +from galaxy.schema.tools import DynamicToolCreatePayload from galaxy.tool_util.cwl.util import set_basename_and_derived_properties from galaxy.tool_util.parser import get_input_source from galaxy.tool_util.parser.output_objects import ToolExpressionOutput @@ -1886,9 +1887,7 @@ def from_dict(Class, trans, d, **kwds): if tool_id is None and tool_uuid is None: tool_representation = d.get("tool_representation") if tool_representation: - create_request = { - "representation": tool_representation, - } + create_request = DynamicToolCreatePayload(src="representation", representation=tool_representation) if not trans.user_is_admin: raise exceptions.AdminRequiredException("Only admin users can create tools dynamically.") dynamic_tool = trans.app.dynamic_tool_manager.create_tool(trans, create_request, allow_load=False) diff --git a/lib/galaxy_test/api/embed_test_1_tool.gxtool.yml b/lib/galaxy_test/api/embed_test_1_tool.gxtool.yml index ff99796a65cf..d99049c9bd22 100644 --- a/lib/galaxy_test/api/embed_test_1_tool.gxtool.yml +++ b/lib/galaxy_test/api/embed_test_1_tool.gxtool.yml @@ -3,3 +3,4 @@ command: echo 'hello world 2' > $output1 outputs: output1: format: txt + type: data diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 4e2148b11f7c..965fb1f4a731 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -43,9 +43,7 @@ "version": "1.0.0", "command": "echo 'Hello World' > $output1", "inputs": [], - "outputs": dict( - output1=dict(format="txt"), - ), + "outputs": {"output1": {"format": "txt", "type": "data"}}, } MINIMAL_TOOL_NO_ID = { "name": "Minimal Tool", @@ -53,30 +51,9 @@ "version": "1.0.0", "command": "echo 'Hello World 2' > $output1", "inputs": [], - "outputs": dict( - output1=dict(format="txt"), - ), -} -TOOL_WITH_BASE_COMMAND = { - "name": "Base command tool", - "class": "GalaxyTool", - "version": "1.0.0", - "base_command": "cat", - "arguments": ["$(inputs.input.path)", ">", "output.fastq"], - "inputs": [ - { - "type": "data", - "name": "input", - } - ], - "outputs": { - "output": { - "type": "data", - "from_work_dir": "output.fastq", - "name": "output", - } - }, + "outputs": {"output1": {"format": "txt", "type": "data"}}, } + TOOL_WITH_SHELL_COMMAND = { "name": "Base command tool", "class": "GalaxyUserTool", @@ -1435,22 +1412,44 @@ def test_dynamic_tool_no_id(self): output_content = self.dataset_populator.get_history_dataset_content(history_id) assert output_content == "Hello World 2\n" - def test_dynamic_tool_base_command(self): - tool_response = self.dataset_populator.create_tool(TOOL_WITH_BASE_COMMAND) - self._assert_has_keys(tool_response, "uuid") - - # Run tool. - history_id = self.dataset_populator.new_history() - dataset = self.dataset_populator.new_dataset(history_id=history_id, content="abc") - self._run( - history_id=history_id, - tool_uuid=tool_response["uuid"], - inputs={"input": {"src": "hda", "id": dataset["id"]}}, - ) - - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - output_content = self.dataset_populator.get_history_dataset_content(history_id) - assert output_content == "abc\n" + # This works except I don't want to add it to the schema right now, + # since I think the shell_command is what we'lll go with (at least initially) + # def test_dynamic_tool_base_command(self): + # TOOL_WITH_BASE_COMMAND = { + # "name": "Base command tool", + # "class": "GalaxyTool", + # "version": "1.0.0", + # "base_command": "cat", + # "arguments": ["$(inputs.input.path)", ">", "output.fastq"], + # "inputs": [ + # { + # "type": "data", + # "name": "input", + # } + # ], + # "outputs": { + # "output": { + # "type": "data", + # "from_work_dir": "output.fastq", + # "name": "output", + # } + # }, + # } + # tool_response = self.dataset_populator.create_tool(TOOL_WITH_BASE_COMMAND) + # self._assert_has_keys(tool_response, "uuid") + + # # Run tool. + # history_id = self.dataset_populator.new_history() + # dataset = self.dataset_populator.new_dataset(history_id=history_id, content="abc") + # self._run( + # history_id=history_id, + # tool_uuid=tool_response["uuid"], + # inputs={"input": {"src": "hda", "id": dataset["id"]}}, + # ) + + # self.dataset_populator.wait_for_history(history_id, assert_ok=True) + # output_content = self.dataset_populator.get_history_dataset_content(history_id) + # assert output_content == "abc\n" def test_dynamic_tool_shell_command(self): tool_response = self.dataset_populator.create_tool(TOOL_WITH_SHELL_COMMAND) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 48f623a19487..69948eb18c67 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1097,6 +1097,7 @@ def test_import_export_dynamic(self): outputs: output1: format: txt + type: data - tool_id: cat1 state: input1: @@ -7826,6 +7827,7 @@ def test_import_export_dynamic_tools(self, history_id): outputs: output1: format: txt + type: data - tool_id: cat1 state: input1: diff --git a/lib/galaxy_test/api/test_workflows_from_yaml.py b/lib/galaxy_test/api/test_workflows_from_yaml.py index 770e2c6625b1..383ff9cb449e 100644 --- a/lib/galaxy_test/api/test_workflows_from_yaml.py +++ b/lib/galaxy_test/api/test_workflows_from_yaml.py @@ -323,6 +323,7 @@ def test_workflow_embed_tool(self): outputs: output1: format: txt + type: data - tool_id: cat1 state: input1: diff --git a/lib/galaxy_test/base/data/minimal_tool_no_id.json b/lib/galaxy_test/base/data/minimal_tool_no_id.json index 1ebd230c86a4..c519d4762015 100644 --- a/lib/galaxy_test/base/data/minimal_tool_no_id.json +++ b/lib/galaxy_test/base/data/minimal_tool_no_id.json @@ -6,7 +6,8 @@ "inputs": [], "outputs": { "output1": { - "format": "txt" + "format": "txt", + "type": "data" } } } diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 4065bcd488a5..47694530064b 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -862,12 +862,13 @@ def list_dynamic_tools(self) -> list: def show_dynamic_tool(self, uuid) -> dict: using_requirement("admin") show_response = self._get(f"dynamic_tools/{uuid}", admin=True) - assert show_response.status_code == 200, show_response + assert show_response.status_code == 200, show_response.text return show_response.json() def deactivate_dynamic_tool(self, uuid) -> dict: using_requirement("admin") - delete_response = self._delete(f"dynamic_tools/{uuid}", admin=True) + delete_response = self._delete(f"dynamic_tools/{uuid}", admin=True, json=True) + assert delete_response.status_code == 200, delete_response.text return delete_response.json() @abstractmethod @@ -2453,7 +2454,7 @@ def import_workflow(self, workflow, **kwds) -> Dict[str, Any]: } data.update(**kwds) upload_response = self._post("workflows", data=data) - assert upload_response.status_code == 200, upload_response.content + assert upload_response.status_code == 200, upload_response.text return upload_response.json() def import_tool(self, tool) -> Dict[str, Any]: @@ -2461,7 +2462,7 @@ def import_tool(self, tool) -> Dict[str, Any]: comparable interface into Galaxy. """ upload_response = self._import_tool_response(tool) - assert upload_response.status_code == 200, upload_response + assert upload_response.status_code == 200, upload_response.text return upload_response.json() def build_module(self, step_type: str, content_id: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None): @@ -2472,9 +2473,8 @@ def build_module(self, step_type: str, content_id: Optional[str] = None, inputs: def _import_tool_response(self, tool) -> Response: using_requirement("admin") - tool_str = json.dumps(tool, indent=4) - data = {"representation": tool_str} - upload_response = self._post("dynamic_tools", data=data, admin=True) + data = {"representation": tool} + upload_response = self._post("dynamic_tools", data=data, admin=True, json=True) return upload_response def scaling_workflow_yaml(self, **kwd): From 293e329ea8796dd255e7e8a485b3f95e63e65d77 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 6 Dec 2024 12:12:58 +0100 Subject: [PATCH 04/67] Add user_tool role types --- lib/galaxy/model/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9c897f2eb634..855530955eb6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3811,6 +3811,8 @@ class types(str, Enum): USER = "user" ADMIN = "admin" SHARING = "sharing" + USER_TOOL_CREATION = "user_tool_create" + USER_TOOL_EXECUTION = "user_tool_execution" @staticmethod def default_name(role_type): From aa5da13d9bf227cde873ddb063623a35613aebbf Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 6 Dec 2024 17:11:07 +0100 Subject: [PATCH 05/67] Add unprivileged_tools endpoint and models --- lib/galaxy/managers/tools.py | 76 +++++++++++++++++-- lib/galaxy/model/__init__.py | 12 +++ ...1_create_user_dynamic_tool_association_.py | 41 ++++++++++ lib/galaxy/schema/tools.py | 19 ++++- lib/galaxy/tool_util/models.py | 1 + .../webapps/galaxy/api/dynamic_tools.py | 49 +++++++++++- lib/galaxy_test/api/test_tools.py | 1 + 7 files changed, 188 insertions(+), 11 deletions(-) create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/f070559879f1_create_user_dynamic_tool_association_.py diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index 76410af2fd14..9b8a7e6d18a7 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -11,6 +11,8 @@ from sqlalchemy import ( select, sql, + true, + update, ) from galaxy import ( @@ -19,8 +21,14 @@ ) from galaxy.exceptions import DuplicatedIdentifierException from galaxy.managers.context import ProvidesUserContext -from galaxy.model import DynamicTool -from galaxy.schema.tools import DynamicToolPayload +from galaxy.model import ( + DynamicTool, + UserDynamicToolAssociation, +) +from galaxy.schema.tools import ( + DynamicToolPayload, + DynamicUnprivilegedToolCreatePayload, +) from galaxy.tool_util.cwl import tool_proxy from .base import ( ModelManager, @@ -51,11 +59,15 @@ def get_tool_by_uuid(self, uuid: Optional[Union[UUID, str]]): return self.session().scalars(stmt).one_or_none() def get_tool_by_tool_id(self, tool_id): - stmt = select(DynamicTool).where(DynamicTool.tool_id == tool_id) + stmt = select(DynamicTool).where(DynamicTool.tool_id == tool_id, DynamicTool.public == true()) + return self.session().scalars(stmt).one_or_none() + + def get_unprivileged_tool_by_tool_id(self, user: model.User, tool_id: str): + stmt = self.owned_unprivileged_statement(user).where(DynamicTool.tool_id == tool_id) return self.session().scalars(stmt).one_or_none() def get_tool_by_id(self, object_id): - stmt = select(DynamicTool).where(DynamicTool.id == object_id) + stmt = select(DynamicTool).where(DynamicTool.id == object_id, DynamicTool.public == true()) return self.session().scalars(stmt).one_or_none() def create_tool(self, trans: ProvidesUserContext, tool_payload: DynamicToolPayload, allow_load=True): @@ -90,8 +102,12 @@ def create_tool(self, trans: ProvidesUserContext, tool_payload: DynamicToolPaylo if not tool_format: raise exceptions.ObjectAttributeMissingException("Current tool representations require 'class'.") - tool_directory = tool_payload.tool_directory - tool_path = tool_payload.path if tool_payload.src == "from_path" else None + tool_directory: Optional[str] = None + tool_path: Optional[str] = None + if tool_payload.src == "from_path": + tool_directory = tool_payload.tool_directory + tool_path = tool_payload.path + if tool_format in ("GalaxyTool", "GalaxyUserTool"): tool_id = representation.get("id") if not tool_id: @@ -122,14 +138,62 @@ def create_tool(self, trans: ProvidesUserContext, tool_payload: DynamicToolPaylo active=tool_payload.active, hidden=tool_payload.hidden, value=representation, + public=True, ) self.app.toolbox.load_dynamic_tool(dynamic_tool) return dynamic_tool + def create_unprivileged_tool(self, user: model.User, tool_payload: DynamicUnprivilegedToolCreatePayload): + if not getattr(self.app.config, "enable_beta_tool_formats", False): + raise exceptions.ConfigDoesNotAllowException( + "Set 'enable_beta_tool_formats' in Galaxy config to create dynamic tools." + ) + if self.get_unprivileged_tool_by_tool_id(user, tool_payload.representation.id): + raise exceptions.Conflict("Tool with id already exists for your user") + + dynamic_tool = self.create( + tool_format=tool_payload.representation.class_, + tool_id=tool_payload.representation.id, + tool_version=tool_payload.representation.version, + active=tool_payload.active, + hidden=tool_payload.hidden, + value=tool_payload.representation, + public=False, + ) + return dynamic_tool + def list_tools(self, active=True): stmt = select(DynamicTool).where(DynamicTool.active == active) return self.session().scalars(stmt) + def list_unprivileged_tools(self, user: model.User, active=True): + owned_statement = self.owned_unprivileged_statement(user=user) + stmt = owned_statement.where( + DynamicTool.active == active, + UserDynamicToolAssociation.active == active, + ) + return self.session().scalars(stmt) + + def owned_unprivileged_statement(self, user: model.User): + return ( + select(DynamicTool) + .join(UserDynamicToolAssociation, DynamicTool.id == UserDynamicToolAssociation.dynamic_tool_id) + .where( + UserDynamicToolAssociation.user_id == user.id, + ) + ) + + def deactivate_unprivileged_tool(self, user: model.User, dynamic_tool: DynamicTool): + update_stmt = ( + update(UserDynamicToolAssociation) + .where( + UserDynamicToolAssociation.user_id == user.id, + UserDynamicToolAssociation.dynamic_tool_id == dynamic_tool.id, + ) + .values(active=False) + ) + self.session().execute(update_stmt) + def deactivate(self, dynamic_tool): self.update(dynamic_tool, {"active": False}) return dynamic_tool diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 855530955eb6..33f21dc8f0fb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1357,6 +1357,17 @@ class ToolRequest(Base, Dictifiable, RepresentById): history: Mapped[Optional["History"]] = relationship(back_populates="tool_requests") +class UserDynamicToolAssociation(Base, Dictifiable, RepresentById): + __tablename__ = "user_dynamic_tool_association" + + id: Mapped[int] = mapped_column(primary_key=True) + dynamic_tool_id: Mapped[int] = mapped_column(ForeignKey("dynamic_tool.id"), index=True) + user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id"), index=True) + create_time: Mapped[datetime] = mapped_column(default=now, nullable=True) + hidden: Mapped[Optional[bool]] = mapped_column(default=False) + active: Mapped[Optional[bool]] = mapped_column(default=True) + + class DynamicTool(Base, Dictifiable, RepresentById): __tablename__ = "dynamic_tool" @@ -1372,6 +1383,7 @@ class DynamicTool(Base, Dictifiable, RepresentById): hidden: Mapped[Optional[bool]] = mapped_column(default=True) active: Mapped[Optional[bool]] = mapped_column(default=True) value: Mapped[Optional[Dict[str, Any]]] = mapped_column(MutableJSONType) + public: Mapped[bool] = mapped_column(default=False, server_default=false()) dict_collection_visible_keys = ("id", "tool_id", "tool_format", "tool_version", "uuid", "active", "hidden") dict_element_visible_keys = ("id", "tool_id", "tool_format", "tool_version", "uuid", "active", "hidden") diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/f070559879f1_create_user_dynamic_tool_association_.py b/lib/galaxy/model/migrations/alembic/versions_gxy/f070559879f1_create_user_dynamic_tool_association_.py new file mode 100644 index 000000000000..2afc5e610f2c --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/f070559879f1_create_user_dynamic_tool_association_.py @@ -0,0 +1,41 @@ +"""Create user_dynamic_tool_association table + +Revision ID: f070559879f1 +Revises: 75348cfb3715 +Create Date: 2024-12-06 14:56:18.494243 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import func + +# revision identifiers, used by Alembic. +revision = "f070559879f1" +down_revision = "75348cfb3715" +branch_labels = None +depends_on = None + + +def upgrade(): + # Create user_dynamic_tool_association table + op.create_table( + "user_dynamic_tool_association", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("dynamic_tool_id", sa.Integer, sa.ForeignKey("dynamic_tool.id"), index=True, nullable=False), + sa.Column("user_id", sa.Integer, sa.ForeignKey("galaxy_user.id"), index=True, nullable=False), + sa.Column("create_time", sa.DateTime, nullable=True, server_default=func.now()), + sa.Column("hidden", sa.Boolean, nullable=True, server_default=sa.text("false")), + sa.Column("active", sa.Boolean, nullable=True, server_default=sa.text("true")), + ) + + # Add the public column with a default value of False + op.add_column("dynamic_tool", sa.Column("public", sa.Boolean, nullable=False, server_default=sa.text("false"))) + # Update existing rows (these have been created by admins) to make them public + op.execute("UPDATE dynamic_tool SET public = TRUE") + + +def downgrade(): + # Drop user_dynamic_tool_association table + op.drop_table("user_dynamic_tool_association") + op.drop_column("dynamic_tool", "public") diff --git a/lib/galaxy/schema/tools.py b/lib/galaxy/schema/tools.py index 8d551a292775..8e88740e2ccf 100644 --- a/lib/galaxy/schema/tools.py +++ b/lib/galaxy/schema/tools.py @@ -6,25 +6,36 @@ from pydantic import BaseModel -from galaxy.tool_util.models import DynamicToolSources +from galaxy.tool_util.models import ( + DynamicToolSources, + UserToolSource, +) class BaseDynamicToolCreatePayload(BaseModel): - allow_load: bool = True - uuid: Optional[str] = None active: Optional[bool] = None hidden: Optional[bool] = None - tool_directory: Optional[str] = None + uuid: Optional[str] = None class DynamicToolCreatePayload(BaseDynamicToolCreatePayload): src: Literal["representation"] = "representation" representation: DynamicToolSources + active: Optional[bool] = True + hidden: Optional[bool] = False + # TODO: split out, doesn't mean anything for unprivileged tools + allow_load: Optional[bool] = True + + +class DynamicUnprivilegedToolCreatePayload(DynamicToolCreatePayload): + representation: UserToolSource class PathBasedDynamicToolCreatePayload(BaseDynamicToolCreatePayload): src: Literal["from_path"] path: str + tool_directory: Optional[str] = None + allow_load: bool = True DynamicToolPayload = Union[DynamicToolCreatePayload, PathBasedDynamicToolCreatePayload] diff --git a/lib/galaxy/tool_util/models.py b/lib/galaxy/tool_util/models.py index 09229cc2b930..8255fe408607 100644 --- a/lib/galaxy/tool_util/models.py +++ b/lib/galaxy/tool_util/models.py @@ -82,6 +82,7 @@ def normalize_items(cls, values): class UserToolSource(ToolSourceBase): class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")] + id: str name: str shell_command: str container: str diff --git a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py index 52a2f29e62f2..ee73cc3e0012 100644 --- a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py +++ b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py @@ -4,11 +4,16 @@ from galaxy.exceptions import ObjectNotFound from galaxy.managers.context import ProvidesUserContext from galaxy.managers.tools import DynamicToolManager +from galaxy.model import User from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.tools import DynamicToolPayload +from galaxy.schema.tools import ( + DynamicToolPayload, + DynamicUnprivilegedToolCreatePayload, +) from . import ( depends, DependsOnTrans, + DependsOnUser, Router, ) @@ -19,6 +24,48 @@ DatabaseIdOrUUID = Union[DecodedDatabaseIdField, str] +@router.cbv +class UnprivilegedToolsApi: + # Almost identical to dynamic tools api, but operates with tool ids + # and is scoped to to individual user and never adds to global toolbox + dynamic_tools_manager: DynamicToolManager = depends(DynamicToolManager) + + @router.get("/api/unprivileged_tools") + def index(self, active: bool = True, trans: ProvidesUserContext = DependsOnTrans): + if not trans.user: + return [] + return [t.to_dict() for t in self.dynamic_tools_manager.list_unprivileged_tools(trans.user, active=active)] + + @router.get("/api/unprivileged_tools/{tool_id}") + def show(self, tool_id: str, user: User = DependsOnUser): + dynamic_tool = self.dynamic_tools_manager.get_unprivileged_tool_by_tool_id(user, tool_id) + if dynamic_tool is None: + raise ObjectNotFound() + return dynamic_tool.to_dict() + + @router.post("/api/unprivileged_tools") + def create(self, payload: DynamicUnprivilegedToolCreatePayload, user: User = DependsOnUser): + dynamic_tool = self.dynamic_tools_manager.create_unprivileged_tool( + user, + payload, + ) + return dynamic_tool.to_dict() + + @router.delete("/api/dynamic_tools/{tool_id}") + def delete(self, tool_id: str, user: User = DependsOnUser): + """ + DELETE /api/dynamic_tools/{encoded_dynamic_tool_id|tool_uuid} + + Deactivate the specified dynamic tool. Deactivated tools will not + be loaded into the toolbox. + """ + dynamic_tool = self.dynamic_tools_manager.get_unprivileged_tool_by_tool_id(user, tool_id) + if not dynamic_tool: + raise ObjectNotFound() + updated_dynamic_tool = self.dynamic_tools_manager.deactivate_unprivileged_tool(user, dynamic_tool) + return updated_dynamic_tool.to_dict() + + @router.cbv class DynamicToolApi: dynamic_tools_manager: DynamicToolManager = depends(DynamicToolManager) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 965fb1f4a731..465aa8136bb0 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -55,6 +55,7 @@ } TOOL_WITH_SHELL_COMMAND = { + "id": "basecommand", "name": "Base command tool", "class": "GalaxyUserTool", "container": "busybox", From 1bf182136e08a8deb19a6bd69246d39eedefc9d7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 9 Dec 2024 10:07:35 +0100 Subject: [PATCH 06/67] Add separate unprivileged tools endpoint --- lib/galaxy/managers/tools.py | 6 +- .../api/test_unprivileged_tools.py | 74 +++++++++++++++++++ lib/galaxy_test/base/populators.py | 16 ++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy_test/api/test_unprivileged_tools.py diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index 9b8a7e6d18a7..dc18b4e34cef 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -157,9 +157,13 @@ def create_unprivileged_tool(self, user: model.User, tool_payload: DynamicUnpriv tool_version=tool_payload.representation.version, active=tool_payload.active, hidden=tool_payload.hidden, - value=tool_payload.representation, + value=tool_payload.representation.model_dump(), public=False, + flush=True, ) + session = self.session() + session.add(UserDynamicToolAssociation(user_id=user.id, dynamic_tool_id=dynamic_tool.id)) + session.commit() return dynamic_tool def list_tools(self, active=True): diff --git a/lib/galaxy_test/api/test_unprivileged_tools.py b/lib/galaxy_test/api/test_unprivileged_tools.py new file mode 100644 index 000000000000..75a2cc6f3897 --- /dev/null +++ b/lib/galaxy_test/api/test_unprivileged_tools.py @@ -0,0 +1,74 @@ +# Test tools API. +import contextlib +import json +import os +import zipfile +from io import BytesIO +from typing import ( + Any, + Dict, + List, + Optional, +) +from uuid import uuid4 + +import pytest +from requests import ( + get, + put, +) + +from galaxy.tool_util.verify.interactor import ValidToolTestDict +from galaxy.util import galaxy_root_path +from galaxy.util.unittest_utils import skip_if_github_down +from galaxy.schema.tools import UserToolSource +from galaxy_test.base import rules_test_data +from galaxy_test.base.api_asserts import ( + assert_has_keys, + assert_status_code_is, +) +from galaxy_test.base.decorators import requires_new_history +from galaxy_test.base.populators import ( + BaseDatasetCollectionPopulator, + DatasetCollectionPopulator, + DatasetPopulator, + skip_without_tool, + stage_rules_example, +) +from ._framework import ApiTestCase + +from .test_tools import TOOL_WITH_SHELL_COMMAND + + +class TestUnprivilegedToolsApi(ApiTestCase): + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) + + def test_create_unprivileged(self): + response = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) + assert response.status_code == 200, response.text + dynamic_tool = response.json() + assert dynamic_tool["uuid"] + + def test_list_unprivileged(self): + response = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) + assert response.status_code == 200, response.text + response = self.dataset_populator.get_unprivileged_tools() + assert response.status_code == 200, response.text + assert response.json() + + def test_show(self): + response = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) + assert response.status_code == 200, response.text + response = self.dataset_populator.show_unprivileged_tool(TOOL_WITH_SHELL_COMMAND["id"]) + assert response.status_code == 200, response.text + assert response.json() + + def test_deactivate(self): + pass + + def test_run(self): + pass diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 47694530064b..ebbea910f629 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -93,6 +93,10 @@ ToolLandingRequest, WorkflowLandingRequest, ) +from galaxy.schema.tools import ( + DynamicUnprivilegedToolCreatePayload, + UserToolSource, +) from galaxy.tool_util.client.staging import InteractorStaging from galaxy.tool_util.cwl.util import ( download_output, @@ -837,6 +841,18 @@ def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]: ) return self._create_tool_raw(payload) + def create_unprivileged_tool(self, representation: UserToolSource, active=True, hidden=False, uuid=None): + data = DynamicUnprivilegedToolCreatePayload( + active=active, hidden=hidden, uuid=uuid, src="representation", representation=representation + ).model_dump(by_alias=True, exclude_unset=True) + return self._post("unprivileged_tools", data=data, json=True) + + def get_unprivileged_tools(self, active=True): + return self._get("unprivileged_tools", data={"active": active}) + + def show_unprivileged_tool(self, tool_id: str): + return self._get(f"unprivileged_tools/{tool_id}") + def create_tool(self, representation, tool_directory: Optional[str] = None) -> Dict[str, Any]: payload = dict( representation=representation, From fde110b18dbc3e02ec88a98c68d78d890139655b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 11 Dec 2024 16:38:50 +0100 Subject: [PATCH 07/67] Test tool run --- lib/galaxy/managers/tools.py | 6 +- lib/galaxy/tool_util/models.py | 2 +- .../tool_util/parser/output_collection_def.py | 2 +- lib/galaxy/tool_util/parser/output_objects.py | 12 +-- lib/galaxy/tool_util/parser/yaml.py | 10 +-- .../webapps/galaxy/api/dynamic_tools.py | 6 +- .../api/test_unprivileged_tools.py | 78 +++++++------------ lib/galaxy_test/base/populators.py | 23 ++++-- 8 files changed, 66 insertions(+), 73 deletions(-) diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index dc18b4e34cef..1ea67a29a667 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -62,6 +62,10 @@ def get_tool_by_tool_id(self, tool_id): stmt = select(DynamicTool).where(DynamicTool.tool_id == tool_id, DynamicTool.public == true()) return self.session().scalars(stmt).one_or_none() + def get_unprivileged_tool_by_uuid(self, user: model.User, uuid: str): + stmt = self.owned_unprivileged_statement(user).where(DynamicTool.uuid == uuid) + return self.session().scalars(stmt).one_or_none() + def get_unprivileged_tool_by_tool_id(self, user: model.User, tool_id: str): stmt = self.owned_unprivileged_statement(user).where(DynamicTool.tool_id == tool_id) return self.session().scalars(stmt).one_or_none() @@ -148,8 +152,6 @@ def create_unprivileged_tool(self, user: model.User, tool_payload: DynamicUnpriv raise exceptions.ConfigDoesNotAllowException( "Set 'enable_beta_tool_formats' in Galaxy config to create dynamic tools." ) - if self.get_unprivileged_tool_by_tool_id(user, tool_payload.representation.id): - raise exceptions.Conflict("Tool with id already exists for your user") dynamic_tool = self.create( tool_format=tool_payload.representation.class_, diff --git a/lib/galaxy/tool_util/models.py b/lib/galaxy/tool_util/models.py index 8255fe408607..1c558ca4eedf 100644 --- a/lib/galaxy/tool_util/models.py +++ b/lib/galaxy/tool_util/models.py @@ -82,7 +82,7 @@ def normalize_items(cls, values): class UserToolSource(ToolSourceBase): class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")] - id: str + id: Optional[str] name: str shell_command: str container: str diff --git a/lib/galaxy/tool_util/parser/output_collection_def.py b/lib/galaxy/tool_util/parser/output_collection_def.py index 81675de7159f..eb6e6cc9d3ac 100644 --- a/lib/galaxy/tool_util/parser/output_collection_def.py +++ b/lib/galaxy/tool_util/parser/output_collection_def.py @@ -58,7 +58,7 @@ def dataset_collector_descriptions_from_elem(elem, legacy=True): def dataset_collector_descriptions_from_output_dict(as_dict): - discover_datasets_dicts = as_dict.get("discover_datasets", []) + discover_datasets_dicts = as_dict.get("discover_datasets") or [] if is_dict(discover_datasets_dicts): discover_datasets_dicts = [discover_datasets_dicts] dataset_collector_descriptions = dataset_collector_descriptions_from_list(discover_datasets_dicts) diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index 047b25d8480e..8b7bf23e46e7 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -166,18 +166,18 @@ def to_model(self) -> ToolOutputDataModel: @staticmethod def from_dict(name: str, output_dict: Dict[str, Any], tool: Optional[object] = None) -> "ToolOutput": output = ToolOutput(name) - output.format = output_dict.get("format", "data") + output.format = output_dict.get("format") or "data" output.change_format = [] - output.format_source = output_dict.get("format_source", None) - output.default_identifier_source = output_dict.get("default_identifier_source", None) - output.metadata_source = output_dict.get("metadata_source", "") + output.format_source = output_dict.get("format_source") + output.default_identifier_source = output_dict.get("default_identifier_source") + output.metadata_source = output_dict.get("metadata_source") or "" output.parent = output_dict.get("parent", None) output.label = output_dict.get("label", None) output.count = output_dict.get("count", 1) output.filters = [] output.tool = tool - output.from_work_dir = output_dict.get("from_work_dir", None) - output.hidden = output_dict.get("hidden", False) + output.from_work_dir = output_dict.get("from_work_dir") + output.hidden = output_dict.get("hidden") or False # TODO: implement tool output action group fixes output.actions = ToolOutputActionGroup(output, None) output.dataset_collector_descriptions = dataset_collector_descriptions_from_output_dict(output_dict) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index 192b0f5b0603..b83301a3ebba 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -72,16 +72,16 @@ def parse_name(self) -> str: return str(rval) def parse_description(self) -> str: - return self.root_dict.get("description", "") + return self.root_dict.get("description") or "" def parse_edam_operations(self) -> List[str]: - return self.root_dict.get("edam_operations", []) + return self.root_dict.get("edam_operations") or [] def parse_edam_topics(self) -> List[str]: - return self.root_dict.get("edam_topics", []) + return self.root_dict.get("edam_topics") or [] def parse_xrefs(self) -> List[XrefDict]: - xrefs = self.root_dict.get("xrefs", []) + xrefs = self.root_dict.get("xrefs") or [] return [XrefDict(value=xref["value"], reftype=xref["type"]) for xref in xrefs if xref["type"]] def parse_sanitize(self): @@ -226,7 +226,7 @@ def parse_tests_to_dict(self) -> ToolSourceTests: return rval def parse_profile(self) -> str: - return self.root_dict.get("profile", "16.04") + return self.root_dict.get("profile") or "16.04" def parse_license(self) -> Optional[str]: return self.root_dict.get("license") diff --git a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py index ee73cc3e0012..18909c90f722 100644 --- a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py +++ b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py @@ -36,9 +36,9 @@ def index(self, active: bool = True, trans: ProvidesUserContext = DependsOnTrans return [] return [t.to_dict() for t in self.dynamic_tools_manager.list_unprivileged_tools(trans.user, active=active)] - @router.get("/api/unprivileged_tools/{tool_id}") - def show(self, tool_id: str, user: User = DependsOnUser): - dynamic_tool = self.dynamic_tools_manager.get_unprivileged_tool_by_tool_id(user, tool_id) + @router.get("/api/unprivileged_tools/{uuid}") + def show(self, uuid: str, user: User = DependsOnUser): + dynamic_tool = self.dynamic_tools_manager.get_unprivileged_tool_by_uuid(user, uuid) if dynamic_tool is None: raise ObjectNotFound() return dynamic_tool.to_dict() diff --git a/lib/galaxy_test/api/test_unprivileged_tools.py b/lib/galaxy_test/api/test_unprivileged_tools.py index 75a2cc6f3897..e98c64415606 100644 --- a/lib/galaxy_test/api/test_unprivileged_tools.py +++ b/lib/galaxy_test/api/test_unprivileged_tools.py @@ -1,46 +1,18 @@ # Test tools API. -import contextlib -import json -import os -import zipfile -from io import BytesIO -from typing import ( - Any, - Dict, - List, - Optional, -) -from uuid import uuid4 - -import pytest -from requests import ( - get, - put, -) - -from galaxy.tool_util.verify.interactor import ValidToolTestDict -from galaxy.util import galaxy_root_path -from galaxy.util.unittest_utils import skip_if_github_down from galaxy.schema.tools import UserToolSource -from galaxy_test.base import rules_test_data -from galaxy_test.base.api_asserts import ( - assert_has_keys, - assert_status_code_is, -) -from galaxy_test.base.decorators import requires_new_history from galaxy_test.base.populators import ( - BaseDatasetCollectionPopulator, DatasetCollectionPopulator, DatasetPopulator, - skip_without_tool, - stage_rules_example, ) from ._framework import ApiTestCase -from .test_tools import TOOL_WITH_SHELL_COMMAND +from .test_tools import ( + TestsTools, + TOOL_WITH_SHELL_COMMAND, +) -class TestUnprivilegedToolsApi(ApiTestCase): +class TestUnprivilegedToolsApi(ApiTestCase, TestsTools): def setUp(self): super().setUp() @@ -48,27 +20,35 @@ def setUp(self): self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) def test_create_unprivileged(self): - response = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) - assert response.status_code == 200, response.text - dynamic_tool = response.json() + dynamic_tool = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) assert dynamic_tool["uuid"] def test_list_unprivileged(self): - response = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) - assert response.status_code == 200, response.text - response = self.dataset_populator.get_unprivileged_tools() - assert response.status_code == 200, response.text - assert response.json() + dynamic_tool = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) + dynamic_tools = self.dataset_populator.get_unprivileged_tools() + assert any( + dynamic_tool["uuid"] == t["uuid"] for t in dynamic_tools + ), f"Newly created dynamic tool {dynamic_tool['uuid']} not in dynamic tools list {dynamic_tools}" def test_show(self): - response = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) - assert response.status_code == 200, response.text - response = self.dataset_populator.show_unprivileged_tool(TOOL_WITH_SHELL_COMMAND["id"]) - assert response.status_code == 200, response.text - assert response.json() - - def test_deactivate(self): - pass + dynamic_tool = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) + show_response = self.dataset_populator.show_unprivileged_tool(dynamic_tool["uuid"]) + assert show_response["name"] def test_run(self): + dynamic_tool = self.dataset_populator.create_unprivileged_tool(UserToolSource(**TOOL_WITH_SHELL_COMMAND)) + # Run tool. + with self.dataset_populator.test_history() as history_id: + dataset = self.dataset_populator.new_dataset(history_id=history_id, content="abc") + self._run( + history_id=history_id, + tool_uuid=dynamic_tool["uuid"], + inputs={"input": {"src": "hda", "id": dataset["id"]}}, + ) + + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + assert output_content == "abc\n" + + def test_deactivate(self): pass diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index ebbea910f629..f5aaf6d9637b 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -841,17 +841,28 @@ def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]: ) return self._create_tool_raw(payload) - def create_unprivileged_tool(self, representation: UserToolSource, active=True, hidden=False, uuid=None): + def create_unprivileged_tool( + self, representation: UserToolSource, active=True, hidden=False, uuid=None, assert_ok=True + ): data = DynamicUnprivilegedToolCreatePayload( active=active, hidden=hidden, uuid=uuid, src="representation", representation=representation ).model_dump(by_alias=True, exclude_unset=True) - return self._post("unprivileged_tools", data=data, json=True) + response = self._post("unprivileged_tools", data=data, json=True) + if assert_ok: + assert response.status_code == 200, response.text + return response.json() - def get_unprivileged_tools(self, active=True): - return self._get("unprivileged_tools", data={"active": active}) + def get_unprivileged_tools(self, active=True, assert_ok=True): + response = self._get("unprivileged_tools", data={"active": active}) + if assert_ok: + assert response.status_code == 200 + return response.json() - def show_unprivileged_tool(self, tool_id: str): - return self._get(f"unprivileged_tools/{tool_id}") + def show_unprivileged_tool(self, uuid: str, assert_ok=True): + response = self._get(f"unprivileged_tools/{uuid}") + if assert_ok: + assert response.status_code == 200, response.text + return response.json() def create_tool(self, representation, tool_directory: Optional[str] = None) -> Dict[str, Any]: payload = dict( From 7de50eb13103508325ec7c7c64ebe45aecd52962 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 11 Dec 2024 18:35:37 +0100 Subject: [PATCH 08/67] Bypass global toolbox for unprivileged tools --- lib/galaxy/jobs/__init__.py | 4 ++-- lib/galaxy/model/__init__.py | 1 + lib/galaxy/tools/__init__.py | 20 ++++++++++++++++++++ lib/galaxy/webapps/galaxy/services/tools.py | 5 ++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c77a07f7da3f..43b3beea9037 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -2567,7 +2567,7 @@ def requires_setting_metadata(self): def _report_error(self): job = self.get_job() - tool = self.app.toolbox.get_tool(job.tool_id, tool_version=job.tool_version) or None + tool = self.app.toolbox.tool_for_job(job) for dataset in job.output_datasets: self.app.error_reports.default_error_plugin.submit_report(dataset, job, tool, user_submission=False) @@ -2590,7 +2590,7 @@ def __init__(self, job, queue: "JobHandlerQueue", use_persisted_destination=Fals job, app=app, use_persisted_destination=use_persisted_destination, - tool=app.toolbox.get_tool(job.tool_id, job.tool_version, exact=True), + tool=app.toolbox.tool_for_job(job, exact=True), ) self.queue = queue self.job_runner_mapper = JobRunnerMapper(self, queue.dispatcher.url_to_destination, self.app.job_config) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 33f21dc8f0fb..ac2756b0be25 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1527,6 +1527,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): state_history: Mapped[List["JobStateHistory"]] = relationship() text_metrics: Mapped[List["JobMetricText"]] = relationship() numeric_metrics: Mapped[List["JobMetricNumeric"]] = relationship() + dynamic_tool: Mapped[Optional["DynamicTool"]] = relationship() interactivetool_entry_points: Mapped[List["InteractiveToolEntryPoint"]] = relationship( back_populates="job", uselist=True ) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 1dde1bc75091..e8b6ea6c4ef1 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -94,6 +94,7 @@ XmlPageSource, XmlToolSource, ) +from galaxy.tool_util.parser.yaml import YamlToolSource from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata from galaxy.tool_util.toolbox import ( AbstractToolBox, @@ -458,6 +459,8 @@ class ToolBox(AbstractToolBox): dependency management, etc. """ + app: "UniverseApplication" + def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel: bool = True): self._reload_count = 0 self.tool_location_fetcher = ToolLocationFetcher() @@ -600,6 +603,23 @@ def get_expanded_tool_source(self, config_file, **kwargs): def _create_tool_from_source(self, tool_source: ToolSource, **kwds): return create_tool_from_source(self.app, tool_source, **kwds) + def get_unprivileged_tool(self, user: model.User, tool_uuid: str) -> Optional["Tool"]: + dynamic_tool = self.app.dynamic_tool_manager.get_unprivileged_tool_by_uuid(user, tool_uuid) + return self.dynamic_tool_to_tool(dynamic_tool) + + def dynamic_tool_to_tool(self, dynamic_tool: Optional[model.DynamicTool]) -> Optional["Tool"]: + if dynamic_tool and dynamic_tool.active and dynamic_tool.value: + tool_source = YamlToolSource(dynamic_tool.value) + tool = create_tool_from_source(self.app, tool_source=tool_source, tool_dir=None) + tool.dynamic_tool = dynamic_tool + return tool + + def tool_for_job(self, job: model.Job, exact=True) -> Optional["Tool"]: + if job.dynamic_tool: + return self.dynamic_tool_to_tool(job.dynamic_tool) + else: + return self.get_tool(job.tool_id, tool_version=job.tool_version, exact=exact) + def create_dynamic_tool(self, dynamic_tool, **kwds): tool_format = dynamic_tool.tool_format tool_representation = dynamic_tool.value diff --git a/lib/galaxy/webapps/galaxy/services/tools.py b/lib/galaxy/webapps/galaxy/services/tools.py index 103b4ec82464..1697ede5bbcd 100644 --- a/lib/galaxy/webapps/galaxy/services/tools.py +++ b/lib/galaxy/webapps/galaxy/services/tools.py @@ -118,7 +118,10 @@ def _create(self, trans: ProvidesHistoryContext, payload, **kwd): if tool_id is None and tool_uuid is None: raise exceptions.RequestParameterMissingException("Must specify either a tool_id or a tool_uuid.") - tool = trans.app.toolbox.get_tool(**get_kwds) + if tool_uuid: + tool = trans.app.toolbox.get_unprivileged_tool(trans.user, tool_uuid=tool_uuid) + else: + tool = trans.app.toolbox.get_tool(**get_kwds) if not tool: log.debug(f"Not found tool with kwds [{get_kwds}]") raise exceptions.ToolMissingException("Tool not found.") From 3c7dd0d4f54eb58bddc392cd88ed3c63e9f128a1 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 12 Dec 2024 12:22:16 +0100 Subject: [PATCH 09/67] Refactor InvocationScrollList to generic ScrollList + Invocation specific component --- .../src/components/ScrollList/ScrollList.vue | 176 ++++++++++++ .../Invocation/InvocationScrollList.vue | 265 +++++------------- 2 files changed, 244 insertions(+), 197 deletions(-) create mode 100644 client/src/components/ScrollList/ScrollList.vue diff --git a/client/src/components/ScrollList/ScrollList.vue b/client/src/components/ScrollList/ScrollList.vue new file mode 100644 index 000000000000..2c8177003ac5 --- /dev/null +++ b/client/src/components/ScrollList/ScrollList.vue @@ -0,0 +1,176 @@ + + + diff --git a/client/src/components/Workflow/Invocation/InvocationScrollList.vue b/client/src/components/Workflow/Invocation/InvocationScrollList.vue index 2240a3d8c315..2d329ebc6fdd 100644 --- a/client/src/components/Workflow/Invocation/InvocationScrollList.vue +++ b/client/src/components/Workflow/Invocation/InvocationScrollList.vue @@ -3,25 +3,21 @@ import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; import { faArrowDown, faInfoCircle } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; -import { useInfiniteScroll } from "@vueuse/core"; -import { BAlert, BBadge, BButton, BListGroup, BListGroupItem } from "bootstrap-vue"; -import { storeToRefs } from "pinia"; -import { computed, onMounted, onUnmounted, ref, watch } from "vue"; +import { computed } from "vue"; import { useRoute, useRouter } from "vue-router/composables"; import type { WorkflowInvocation } from "@/api/invocations"; import { getData } from "@/components/Grid/configs/invocations"; -import { useAnimationFrameResizeObserver } from "@/composables/sensors/animationFrameResizeObserver"; -import { useAnimationFrameScroll } from "@/composables/sensors/animationFrameScroll"; import { useHistoryStore } from "@/stores/historyStore"; import { useUserStore } from "@/stores/userStore"; import { useWorkflowStore } from "@/stores/workflowStore"; import Heading from "@/components/Common/Heading.vue"; -import LoadingSpan from "@/components/LoadingSpan.vue"; -import ScrollToTopButton from "@/components/ToolsList/ScrollToTopButton.vue"; +import ScrollList from "@/components/ScrollList/ScrollList.vue"; import UtcDate from "@/components/UtcDate.vue"; +const currentUser = computed(() => useUserStore().currentUser); + interface Props { inPanel?: boolean; limit?: number; @@ -42,99 +38,13 @@ const stateClasses: Record = { failed: "error", }; -const busy = ref(false); -const errorMessage = ref(""); -const initDataLoading = ref(true); -const invocations = ref([]); -const totalInvocationCount = ref(undefined); -const currentPage = ref(0); -const scrollableDiv = ref(null); - -const { currentUser } = storeToRefs(useUserStore()); - -const allLoaded = computed( - () => totalInvocationCount.value !== undefined && totalInvocationCount.value <= invocations.value.length -); - -// check if we have scrolled to the top or bottom of the scrollable div -const { arrived, scrollTop } = useAnimationFrameScroll(scrollableDiv); -const isScrollable = ref(false); -useAnimationFrameResizeObserver(scrollableDiv, ({ clientSize, scrollSize }) => { - isScrollable.value = scrollSize.height >= clientSize.height + 1; -}); -const scrolledTop = computed(() => !isScrollable.value || arrived.top); -const scrolledBottom = computed(() => !isScrollable.value || arrived.bottom); - -onMounted(async () => { - useInfiniteScroll(scrollableDiv.value, () => loadInvocations()); -}); - -onUnmounted(() => { - // Remove the infinite scrolling behavior - useInfiniteScroll(scrollableDiv.value, () => {}); -}); - -/** if screen size is as such that a scroller is not rendered, - * we load enough invocations so that a scroller is rendered - */ -watch( - () => isScrollable.value, - (scrollable: boolean) => { - if (!scrollable && !allLoaded.value) { - loadInvocations(); - } - } -); - -const currentItemId = computed(() => { - const path = route.path; - const match = path.match(/\/workflows\/invocations\/([a-zA-Z0-9]+)/); - return match ? match[1] : undefined; -}); - -const route = useRoute(); -const router = useRouter(); - -function cardClicked(invocation: WorkflowInvocation) { - if (props.inPanel) { - emit("invocation-clicked"); - } - router.push(`/workflows/invocations/${invocation.id}`); -} - -function scrollToTop() { - scrollableDiv.value?.scrollTo({ top: 0, behavior: "smooth" }); -} - -/** - * Request invocations for the current user - */ -async function loadInvocations() { - if (currentUser.value && !currentUser.value.isAnonymous && !busy.value && !allLoaded.value) { - busy.value = true; - try { - const offset = props.limit * currentPage.value; - const extraProps = currentUser.value ? { user_id: currentUser.value.id } : {}; - - const [responseData, responseTotal] = await getData( - offset, - props.limit, - "", - "create_time", - true, - extraProps - ); - invocations.value = invocations.value.concat(responseData as WorkflowInvocation[]); - totalInvocationCount.value = responseTotal as number; - currentPage.value += 1; - errorMessage.value = ""; - } catch (e) { - errorMessage.value = `Failed to obtain invocations: ${e}`; - } finally { - initDataLoading.value = false; - busy.value = false; - } +async function loadInvocations(offset: number, limit: number) { + if (!currentUser.value || currentUser.value.isAnonymous) { + return { items: [], total: 0 }; } + const extraProps = { user_id: currentUser.value.id }; + const [responseData, responseTotal] = await getData(offset, limit, "", "create_time", true, extraProps); + return { items: responseData, total: responseTotal }; } function historyName(historyId: string) { @@ -153,109 +63,70 @@ function workflowName(workflowId: string) { const workflowStore = useWorkflowStore(); return workflowStore.getStoredWorkflowNameByInstanceId(workflowId); } + +const route = useRoute(); +const router = useRouter(); + +const currentItemId = computed(() => { + const path = route.path; + const match = path.match(/\/workflows\/invocations\/([a-zA-Z0-9]+)/); + return match ? match[1] : undefined; +}); + +function cardClicked(invocation: WorkflowInvocation) { + if (props.inPanel) { + emit("invocation-clicked"); + } + router.push(`/workflows/invocations/${invocation.id}`); +} diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue index f2e9aae616f4..b7b142948a73 100644 --- a/client/src/components/Workflow/Editor/Index.vue +++ b/client/src/components/Workflow/Editor/Index.vue @@ -97,6 +97,10 @@ @creator="onCreator" @update:nameCurrent="setName" @update:annotationCurrent="setAnnotation" /> +