diff --git a/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/api/v1/dto/EngineStatus.java b/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/api/v1/dto/EngineStatus.java index c0e79ed5..266b0e68 100644 --- a/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/api/v1/dto/EngineStatus.java +++ b/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/api/v1/dto/EngineStatus.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.DockerEnvironmentDTO; +import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.LocalEnvironmentDTO; import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.TarballEnvironmentDTO; import lombok.AllArgsConstructor; import lombok.Getter; @@ -36,5 +37,6 @@ public class EngineStatus { public static class Environments { private List tarball; private List docker; + private List local; } } diff --git a/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/descriptor/dto/LocalEnvironmentDTO.java b/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/descriptor/dto/LocalEnvironmentDTO.java new file mode 100644 index 00000000..485d8847 --- /dev/null +++ b/commons/src/main/java/com/odysseusinc/arachne/execution_engine_common/descriptor/dto/LocalEnvironmentDTO.java @@ -0,0 +1,14 @@ +package com.odysseusinc.arachne.execution_engine_common.descriptor.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class LocalEnvironmentDTO { + // TODO add data describing the local environment +} diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/AnalysisService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/AnalysisService.java index 43ce7dc7..d1483b2a 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/AnalysisService.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/AnalysisService.java @@ -36,11 +36,13 @@ import com.odysseusinc.arachne.execution_engine_common.api.v1.dto.ExecutionOutcome; import com.odysseusinc.arachne.execution_engine_common.api.v1.dto.Stage; import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.DockerEnvironmentDTO; +import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.LocalEnvironmentDTO; import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.TarballEnvironmentDTO; import com.odysseusinc.arachne.executionengine.aspect.FileDescriptorCount; import com.odysseusinc.arachne.executionengine.auth.AuthEffects; import com.odysseusinc.arachne.executionengine.auth.CredentialsProvider; import com.odysseusinc.arachne.executionengine.execution.r.DockerEnvironmentService; +import com.odysseusinc.arachne.executionengine.execution.r.LocalEnvironmentService; import com.odysseusinc.arachne.executionengine.model.descriptor.converter.DescriptorConverter; import com.odysseusinc.arachne.executionengine.service.CdmMetadataService; import com.odysseusinc.arachne.executionengine.service.impl.DescriptorServiceImpl; @@ -99,6 +101,8 @@ public class AnalysisService { private DockerEnvironmentService dockerDescriptorService; @Autowired private DescriptorServiceImpl tarballDescriptorService; + @Autowired + private LocalEnvironmentService localEnvironmentService; @Value("${submission.update.interval}") @@ -197,16 +201,15 @@ private void attachMetadata(AnalysisSyncRequestDTO analysis, File analysisDir) { public EngineStatus getStatus(List idsMaybe) { Map statuses = Optional.ofNullable(idsMaybe).map(ids -> ids.stream().flatMap(id -> - Optional.ofNullable(overseers.get(id)).map(overseer -> - Stream.of(Pair.of(id, getStatus(overseer))) - ).orElseGet(Stream::of) + Stream.ofNullable(overseers.get(id)).map(overseer -> Pair.of(id, getStatus(overseer))) ).collect(Collectors.toMap(Pair::getKey, Pair::getValue)) ).orElseGet(Collections::emptyMap); List tarballs = tarballDescriptorService.getDescriptors().stream().map(DescriptorConverter::toDto).collect(Collectors.toList()); List dockers = dockerDescriptorService.getEnvironments(); + List locals = localEnvironmentService.getEnvironments(); - return new EngineStatus(Instant.now(), statuses, new Environments(tarballs, dockers)); + return new EngineStatus(Instant.now(), statuses, new Environments(tarballs, dockers, locals)); } private static ExecutionOutcome getStatus(Overseer overseer) { diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/LocalEnvironmentService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/LocalEnvironmentService.java new file mode 100644 index 00000000..2def7d71 --- /dev/null +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/LocalEnvironmentService.java @@ -0,0 +1,20 @@ +package com.odysseusinc.arachne.executionengine.execution.r; + +import com.odysseusinc.arachne.execution_engine_common.descriptor.dto.LocalEnvironmentDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class LocalEnvironmentService { + @Value("${runtime.local:false}") + private boolean useLocalREnv; + + public List getEnvironments() { + return useLocalREnv ? List.of(new LocalEnvironmentDTO()) : List.of(); + } + +} diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/LocalRService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/LocalRService.java new file mode 100644 index 00000000..ff262345 --- /dev/null +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/LocalRService.java @@ -0,0 +1,43 @@ +package com.odysseusinc.arachne.executionengine.execution.r; + +import com.odysseusinc.arachne.execution_engine_common.api.v1.dto.AnalysisSyncRequestDTO; +import com.odysseusinc.arachne.executionengine.execution.Overseer; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import java.util.function.BiConsumer; + +@Service +@Slf4j +@ConditionalOnProperty(name = "runtime.local") +public class LocalRService extends RService { + + @Override + protected Overseer analyze(AnalysisSyncRequestDTO analysis, File file, Integer updateInterval, Map envp, BiConsumer callback) { + Long id = analysis.getId(); + String executableFileName = analysis.getExecutableFileName(); + + try { + Instant started = Instant.now(); + log.info("Execution [{}] using local R environment (runtime.local=true)", id); + ProcessBuilder pb = new ProcessBuilder(EXECUTION_COMMAND, executableFileName).directory(file).redirectErrorStream(true); + pb.environment().putAll(envp); + log.info("Execution [{}] start local R process: {}", id, String.join(" ", new String[]{EXECUTION_COMMAND, executableFileName})); + Process process = pb.start(); + + return new TarballROverseer( + id, process, runtimeTimeOutSec, callback, updateInterval, started, "local", killTimeoutSec + ); + + } catch (IOException ex) { + log.error("Execution [{}] error building runtime command", id, ex); + throw new RuntimeException(ex.getMessage(), ex); + } + } + +} diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RExecutionService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RExecutionService.java index ca509937..8b8bd466 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RExecutionService.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RExecutionService.java @@ -5,8 +5,10 @@ import com.odysseusinc.arachne.executionengine.execution.ExecutionService; import com.odysseusinc.arachne.executionengine.execution.Overseer; import com.odysseusinc.arachne.executionengine.service.DescriptorService; +import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -22,6 +24,11 @@ public class RExecutionService implements ExecutionService { private DescriptorService descriptorService; @Autowired private DockerService dockerService; + @Autowired(required = false) + private LocalRService localService; + + @Value("${runtime.local:false}") + private boolean useLocalREnv; public String getExtension() { return "r"; @@ -34,7 +41,10 @@ public Overseer analyze(AnalysisSyncRequestDTO analysis, File dir, BiConsumer envp, BiConsumer callback) { DescriptorBundle descriptorBundle = descriptorService.getDescriptorBundle( @@ -215,21 +200,9 @@ private String[] buildRuntimeCommand(File runFile, File workingDir, String fileN throw new FileNotFoundException("file '" + fileName + "' is not exists in directory '" + workingDir.getAbsolutePath() + "'"); } - String[] command; - if (RuntimeServiceMode.ISOLATED.equals(getRuntimeServiceMode())) { - command = ArrayUtils.addAll(rIsolatedRuntimeProps.getRunCmd(), - runFile.getAbsolutePath(), workingDir.getAbsolutePath(), fileName, bundlePath); - } else { - command = new String[]{EXECUTION_COMMAND, fileName}; - } - return command; - } - - private RuntimeServiceMode getRuntimeServiceMode() { - return useLocalREnv ? RuntimeServiceMode.SINGLE : RuntimeServiceMode.ISOLATED; + return ArrayUtils.addAll(rIsolatedRuntimeProps.getRunCmd(), runFile.getAbsolutePath(), workingDir.getAbsolutePath(), fileName, bundlePath); } - @SuppressWarnings("SameParameterValue") private static File extractToTempFile(ResourceLoader loader, String resourceName, String prefix, String suffix) throws IOException { File runFile = Files.createTempFile(prefix, suffix).toFile();