From 9a0a58b1e40315bf90b323c150826d898e7cca1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 5 Feb 2025 14:52:40 +0100 Subject: [PATCH 1/3] Fix `test_mulled_build.py::test_mulled_build_files_cli` with `use_mamba=True` Test `tests/tool_util/mulled/test_mulled_build.py::test_mulled_build_files_cli[True]`, where `[True]` refers to the parameter `use_mamba`, fails because if `conda install --quiet --yes 'mamba='` (the preinstall command) runs before `mamba install -p /usr/local`, then the latter expects either `/usr/local` not to exist or to be an existing environment. To work this around, create an environment in `/usr/local/env`, but still put it on the expected location `/usr/local` later. --- lib/galaxy/tool_util/deps/mulled/invfile.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/invfile.lua b/lib/galaxy/tool_util/deps/mulled/invfile.lua index f7bef84d18b5..fdacf4740ae2 100644 --- a/lib/galaxy/tool_util/deps/mulled/invfile.lua +++ b/lib/galaxy/tool_util/deps/mulled/invfile.lua @@ -88,13 +88,14 @@ inv.task('build') .using(conda_image) .withHostConfig({binds = bind_args}) .run('/bin/sh', '-c', preinstall + .. conda_bin .. ' create --quiet --yes -p /usr/local/env --copy && ' .. conda_bin .. ' install ' .. channel_args .. ' ' .. target_args - .. ' --strict-channel-priority -p /usr/local --copy --yes ' + .. ' --strict-channel-priority -p /usr/local/env --copy --yes ' .. verbose .. postinstall) - .wrap('build/dist') + .wrap('build/dist/env') .at('/usr/local') .inImage(destination_base_image) .as(repo) From bd24d0cafc14ba347e9a50713ebece6b630e7dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 6 Feb 2025 11:23:26 +0100 Subject: [PATCH 2/3] Split installation shell command into four With the aim of easing debugging, write one `run()` statement for the preinstall command, one to create the conda environment, another to install packages to the environment and one for the postinstall command. --- lib/galaxy/tool_util/deps/mulled/invfile.lua | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/invfile.lua b/lib/galaxy/tool_util/deps/mulled/invfile.lua index fdacf4740ae2..25b2c7b07592 100644 --- a/lib/galaxy/tool_util/deps/mulled/invfile.lua +++ b/lib/galaxy/tool_util/deps/mulled/invfile.lua @@ -72,14 +72,8 @@ else end local preinstall = VAR.PREINSTALL -if preinstall ~= '' then - preinstall = preinstall .. ' && ' -end local postinstall = VAR.POSTINSTALL -if postinstall ~= '' then - postinstall = '&&' .. postinstall -end inv.task('build') .using(conda_image) @@ -87,14 +81,11 @@ inv.task('build') .run('rm', '-rf', '/data/dist') .using(conda_image) .withHostConfig({binds = bind_args}) - .run('/bin/sh', '-c', preinstall - .. conda_bin .. ' create --quiet --yes -p /usr/local/env --copy && ' - .. conda_bin .. ' install ' - .. channel_args .. ' ' - .. target_args - .. ' --strict-channel-priority -p /usr/local/env --copy --yes ' - .. verbose - .. postinstall) + .run('/bin/sh', '-c', preinstall) + .run('/bin/sh', '-c', conda_bin .. ' create --quiet --yes -p /usr/local/env --copy') + .run('/bin/sh', '-c', conda_bin .. ' install ' .. channel_args .. ' ' .. target_args + .. ' --strict-channel-priority -p /usr/local/env --copy --yes ' .. verbose) + .run('/bin/sh', '-c', postinstall) .wrap('build/dist/env') .at('/usr/local') .inImage(destination_base_image) From 7c1b644e3b9b627f8f7a5f476d381a92ba3f53ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 6 Feb 2025 14:23:49 +0100 Subject: [PATCH 3/3] Revert "Split installation shell command into four" This reverts commit bd24d0cafc14ba347e9a50713ebece6b630e7dc1. It makes sense to revert the changes because every `run()` statement runs in its own container, thus it cannot be ruled out that other code is building the preinstall or postinstall commands under the assumption that they will run in the same container (or even in the same container as the conda commands). An example is available on https://github.com/galaxyproject/galaxy/pull/19545#discussion_r1944557524. --- lib/galaxy/tool_util/deps/mulled/invfile.lua | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/invfile.lua b/lib/galaxy/tool_util/deps/mulled/invfile.lua index 25b2c7b07592..fdacf4740ae2 100644 --- a/lib/galaxy/tool_util/deps/mulled/invfile.lua +++ b/lib/galaxy/tool_util/deps/mulled/invfile.lua @@ -72,8 +72,14 @@ else end local preinstall = VAR.PREINSTALL +if preinstall ~= '' then + preinstall = preinstall .. ' && ' +end local postinstall = VAR.POSTINSTALL +if postinstall ~= '' then + postinstall = '&&' .. postinstall +end inv.task('build') .using(conda_image) @@ -81,11 +87,14 @@ inv.task('build') .run('rm', '-rf', '/data/dist') .using(conda_image) .withHostConfig({binds = bind_args}) - .run('/bin/sh', '-c', preinstall) - .run('/bin/sh', '-c', conda_bin .. ' create --quiet --yes -p /usr/local/env --copy') - .run('/bin/sh', '-c', conda_bin .. ' install ' .. channel_args .. ' ' .. target_args - .. ' --strict-channel-priority -p /usr/local/env --copy --yes ' .. verbose) - .run('/bin/sh', '-c', postinstall) + .run('/bin/sh', '-c', preinstall + .. conda_bin .. ' create --quiet --yes -p /usr/local/env --copy && ' + .. conda_bin .. ' install ' + .. channel_args .. ' ' + .. target_args + .. ' --strict-channel-priority -p /usr/local/env --copy --yes ' + .. verbose + .. postinstall) .wrap('build/dist/env') .at('/usr/local') .inImage(destination_base_image)