diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/Command.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/Command.java index 6eed8637..3ca6f2b9 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/Command.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/Command.java @@ -29,7 +29,7 @@ @Getter public enum Command { // Available for: Cluster, Host, Service, Component - // Remove is not a command because it won't create job, please refer to the related controller for remove action. + // Remove is not a command because it won't create job, please refer to the related API for remove action. ADD("add", "Add"), START("start", "Start"), STOP("stop", "Stop"), @@ -40,7 +40,9 @@ public enum Command { CONFIGURE("configure", "Configure"), CUSTOM("custom", "Custom"), - // Internal use only, not available for job creation + // Internal use only, not available for API call + INIT("init", "Init"), + PREPARE("prepare", "Prepare"), STATUS("status", "Status"), ; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java new file mode 100644 index 00000000..05ab40d2 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.helper; + +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.stage.ComponentAddStage; +import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage; +import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage; +import org.apache.bigtop.manager.server.command.stage.ComponentInitStage; +import org.apache.bigtop.manager.server.command.stage.ComponentPrepareStage; +import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; +import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; +import org.apache.bigtop.manager.server.command.stage.Stage; +import org.apache.bigtop.manager.server.command.stage.StageContext; +import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.exception.ServerException; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.dto.ComponentDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceDTO; +import org.apache.bigtop.manager.server.utils.StackUtils; + +import org.apache.commons.collections4.CollectionUtils; + +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Slf4j +public class ComponentStageHelper { + + public static List createComponentStages(Map> componentHosts, CommandDTO commandDTO) { + return createComponentStages(componentHosts, commandDTO.getCommand(), commandDTO); + } + + public static List createComponentStages( + Map> componentHosts, Command command, CommandDTO commandDTO) { + return createComponentStages(componentHosts, List.of(command), commandDTO); + } + + public static List createComponentStages( + Map> componentHosts, List commands, CommandDTO commandDTO) { + List stages = new ArrayList<>(); + List componentNames = new ArrayList<>(componentHosts.keySet()); + List todoList = getTodoList(componentNames, commands); + + for (String componentCommand : todoList) { + String[] split = componentCommand.split("-"); + String componentName = split[0].toLowerCase(); + Command command = Command.valueOf(split[1].toUpperCase()); + + List hostnames = componentHosts.get(componentName); + if (CollectionUtils.isEmpty(hostnames)) { + continue; + } + + StageContext stageContext; + switch (command) { + case ADD: + stageContext = createStageContext(componentName, hostnames, commandDTO); + stages.add(new ComponentAddStage(stageContext)); + break; + case START: + if (!StackUtils.isClientComponent(componentName)) { + stageContext = createStageContext(componentName, hostnames, commandDTO); + stages.add(new ComponentStartStage(stageContext)); + } + break; + case STOP: + if (!StackUtils.isClientComponent(componentName)) { + stageContext = createStageContext(componentName, hostnames, commandDTO); + stages.add(new ComponentStopStage(stageContext)); + } + break; + case CHECK: + if (!StackUtils.isClientComponent(componentName)) { + stageContext = createStageContext(componentName, List.of(hostnames.get(0)), commandDTO); + stages.add(new ComponentCheckStage(stageContext)); + } + break; + case CONFIGURE: + stageContext = createStageContext(componentName, hostnames, commandDTO); + stages.add(new ComponentConfigureStage(stageContext)); + break; + case INIT: + stageContext = createStageContext(componentName, hostnames, commandDTO); + stages.add(new ComponentInitStage(stageContext)); + break; + case PREPARE: + // Prepare phase runs after component started, client component shouldn't create this. + if (!StackUtils.isClientComponent(componentName)) { + stageContext = createStageContext(componentName, hostnames, commandDTO); + stages.add(new ComponentPrepareStage(stageContext)); + } + break; + } + } + + return stages; + } + + private static StageContext createStageContext( + String componentName, List hostnames, CommandDTO commandDTO) { + StageContext stageContext = StageContext.fromCommandDTO(commandDTO); + + ServiceDTO serviceDTO = StackUtils.getServiceDTOByComponentName(componentName); + ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); + + stageContext.setHostnames(hostnames); + stageContext.setServiceDTO(serviceDTO); + stageContext.setComponentDTO(componentDTO); + + return stageContext; + } + + private static List getTodoList(List componentNames, Command command) { + return getTodoList(componentNames, List.of(command)); + } + + private static List getTodoList(List componentNames, List commands) { + try { + List orderedList = + StackUtils.DAG.getAllNodesList().isEmpty() ? new ArrayList<>() : StackUtils.DAG.topologicalSort(); + orderedList.replaceAll(String::toUpperCase); + List componentCommandNames = new ArrayList<>(); + for (String componentName : componentNames) { + for (Command command : commands) { + String name = + componentName.toUpperCase() + "-" + command.name().toUpperCase(); + componentCommandNames.add(name); + } + } + + // Re-order the commands, since order.json does not contain all commands, + // only contains which has dependencies, we need to add the rest to the end. + if (commands.size() == 1) { + orderedList.retainAll(componentCommandNames); + componentCommandNames.removeAll(orderedList); + orderedList.addAll(componentCommandNames); + return orderedList; + } else { + // TODO, order.json currently only contains start/stop dependencies of major components, the situation + // of commands size greater than 1 is only with init/start/prepare commands, we see this as a special + // situation and use special logic for it, should use a better solution in the future. + orderedList.retainAll(componentCommandNames); + componentCommandNames.removeAll(orderedList); + + List res = new ArrayList<>(); + List commandOrder = List.of("ADD", "CONFIGURE", "INIT", "START", "PREPARE", "CHECK"); + for (String command : commandOrder) { + List filtered = componentCommandNames.stream() + .filter(s -> s.endsWith(command)) + .toList(); + res.addAll(filtered); + if (command.equals("START")) { + res.addAll(orderedList); + } + } + + return res; + } + + } catch (Exception e) { + throw new ServerException(e); + } + } + + public static void printStageDump(List stages) { + log.info("Dumping stages..."); + for (Stage stage : stages) { + List taskNames = + stage.getTasks().stream().map(Task::getName).toList(); + log.info("[{}] : [{}]", stage.getName(), String.join(",", taskNames)); + } + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java index 635d5590..1f72ba5d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java @@ -28,7 +28,9 @@ import org.apache.bigtop.manager.dao.repository.JobDao; import org.apache.bigtop.manager.dao.repository.StageDao; import org.apache.bigtop.manager.dao.repository.TaskDao; +import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; import org.apache.bigtop.manager.server.command.stage.Stage; +import org.apache.bigtop.manager.server.command.stage.StageContext; import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.holder.SpringContextHolder; @@ -82,6 +84,11 @@ protected void beforeCreateStages() { protected abstract void createStages(); + protected void createCacheStage() { + StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); + stages.add(new CacheFileUpdateStage(stageContext)); + } + @Override public void beforeRun() { jobPO.setState(JobState.PROCESSING.getName()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/AbstractClusterJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/AbstractClusterJob.java index 211788e7..d6bfbd71 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/AbstractClusterJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/AbstractClusterJob.java @@ -18,25 +18,17 @@ */ package org.apache.bigtop.manager.server.command.job.cluster; -import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.query.ComponentQuery; import org.apache.bigtop.manager.dao.repository.ComponentDao; import org.apache.bigtop.manager.server.command.job.AbstractJob; import org.apache.bigtop.manager.server.command.job.JobContext; -import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; -import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; -import org.apache.bigtop.manager.server.command.stage.StageContext; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.model.dto.ComponentDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.utils.StackDAGUtils; -import org.apache.bigtop.manager.server.utils.StackUtils; - -import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class AbstractClusterJob extends AbstractJob { @@ -58,84 +50,16 @@ protected void beforeCreateStages() { super.beforeCreateStages(); } - protected StageContext createStageContext(String componentName, List hostnames) { - StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - - ServiceDTO serviceDTO = StackUtils.getServiceDTOByComponentName(componentName); - ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); - - stageContext.setHostnames(hostnames); - stageContext.setServiceDTO(serviceDTO); - stageContext.setComponentDTO(componentDTO); - - return stageContext; - } - - protected void createStartStages() { - List componentPOList = getComponentPOList(); - List todoList = StackDAGUtils.getTodoList(getComponentNames(componentPOList), Command.START); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = findHostnamesByComponentName(componentPOList, componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentStartStage(stageContext)); - } - } - - protected void createStopStages() { - List componentPOList = getComponentPOList(); - List todoList = StackDAGUtils.getTodoList(getComponentNames(componentPOList), Command.STOP); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = findHostnamesByComponentName(componentPOList, componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentStopStage(stageContext)); - } - } - - private List getComponentPOList() { - return componentDao.findByQuery( - ComponentQuery.builder().clusterId(clusterPO.getId()).build()); - } - - private List getComponentNames(List componentPOList) { - if (componentPOList == null) { - return new ArrayList<>(); - } else { - return componentPOList.stream().map(ComponentPO::getName).distinct().toList(); + protected Map> getComponentHostsMap() { + ComponentQuery componentQuery = + ComponentQuery.builder().clusterId(clusterPO.getId()).build(); + List componentPOList = componentDao.findByQuery(componentQuery); + Map> componentHostsMap = new HashMap<>(); + for (ComponentPO componentPO : componentPOList) { + List hosts = componentHostsMap.computeIfAbsent(componentPO.getName(), k -> new ArrayList<>()); + hosts.add(componentPO.getHostname()); } - } - private List findHostnamesByComponentName(List componentPOList, String componentName) { - if (componentPOList == null) { - return new ArrayList<>(); - } else { - return componentPOList.stream() - .filter(componentPO -> componentPO.getName().equals(componentName)) - .map(ComponentPO::getHostname) - .toList(); - } + return componentHostsMap; } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterRestartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterRestartJob.java index 6ac4c5d1..7813693f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterRestartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterRestartJob.java @@ -18,7 +18,13 @@ */ package org.apache.bigtop.manager.server.command.job.cluster; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ClusterRestartJob extends AbstractClusterJob { @@ -28,9 +34,12 @@ public ClusterRestartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); - super.createStartStages(); + // Restart services + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.STOP, commandDTO)); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.START, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStartJob.java index 9f8ff7a8..d1f6dba1 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStartJob.java @@ -18,7 +18,12 @@ */ package org.apache.bigtop.manager.server.command.job.cluster; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ClusterStartJob extends AbstractClusterJob { @@ -28,7 +33,10 @@ public ClusterStartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStartStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStopJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStopJob.java index ee4cc770..ff509d4c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStopJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/cluster/ClusterStopJob.java @@ -18,7 +18,12 @@ */ package org.apache.bigtop.manager.server.command.job.cluster; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ClusterStopJob extends AbstractClusterJob { @@ -28,7 +33,10 @@ public ClusterStopJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/AbstractComponentJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/AbstractComponentJob.java index 7fa057f5..3374cf22 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/AbstractComponentJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/AbstractComponentJob.java @@ -18,29 +18,16 @@ */ package org.apache.bigtop.manager.server.command.job.component; -import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.dao.repository.ComponentDao; import org.apache.bigtop.manager.dao.repository.HostDao; import org.apache.bigtop.manager.dao.repository.ServiceDao; import org.apache.bigtop.manager.server.command.job.AbstractJob; import org.apache.bigtop.manager.server.command.job.JobContext; -import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; -import org.apache.bigtop.manager.server.command.stage.ComponentAddStage; -import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage; -import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; -import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; -import org.apache.bigtop.manager.server.command.stage.StageContext; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.model.dto.ComponentDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; -import org.apache.bigtop.manager.server.utils.StackDAGUtils; -import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class AbstractComponentJob extends AbstractJob { @@ -66,105 +53,15 @@ protected void beforeCreateStages() { super.beforeCreateStages(); } - protected StageContext createStageContext(String componentName, List hostnames) { - StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - - ServiceDTO serviceDTO = StackUtils.getServiceDTOByComponentName(componentName); - ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); - - stageContext.setHostnames(hostnames); - stageContext.setServiceDTO(serviceDTO); - stageContext.setComponentDTO(componentDTO); - - return stageContext; - } - - protected void createCacheStage() { - StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - stages.add(new CacheFileUpdateStage(stageContext)); - } - - protected void createAddStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.ADD); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - List hostnames = getHostnames(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentAddStage(stageContext)); - } - } + protected Map> getComponentHostsMap() { + Map> componentHostsMap = new HashMap<>(); - protected void createConfigureStages() { - for (ComponentCommandDTO componentCommand : jobContext.getCommandDTO().getComponentCommands()) { + jobContext.getCommandDTO().getComponentCommands().forEach(componentCommand -> { String componentName = componentCommand.getComponentName(); List hostnames = componentCommand.getHostnames(); + componentHostsMap.put(componentName, hostnames); + }); - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentConfigureStage(stageContext)); - } - } - - protected void createStartStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.START); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = getHostnames(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentStartStage(stageContext)); - } - } - - protected void createStopStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.STOP); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = getHostnames(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentStopStage(stageContext)); - } - } - - private List getComponentNames() { - return jobContext.getCommandDTO().getComponentCommands().stream() - .map(ComponentCommandDTO::getComponentName) - .toList(); - } - - private List getHostnames(String componentName) { - for (ComponentCommandDTO componentCommand : jobContext.getCommandDTO().getComponentCommands()) { - if (componentCommand.getComponentName().equals(componentName)) { - return componentCommand.getHostnames(); - } - } - - return new ArrayList<>(); + return componentHostsMap; } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentAddJob.java index 762b7b08..c95fd60b 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentAddJob.java @@ -18,9 +18,11 @@ */ package org.apache.bigtop.manager.server.command.job.component; +import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.po.HostPO; import org.apache.bigtop.manager.dao.po.ServicePO; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.model.converter.ComponentConverter; @@ -32,6 +34,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; public class ComponentAddJob extends AbstractComponentJob { @@ -49,14 +52,18 @@ protected void createStages() { // Update cache files super.createCacheStage(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + // Install components - super.createAddStages(); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.ADD, commandDTO)); - // Configure services - super.createConfigureStages(); + // Configure components + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.CONFIGURE, commandDTO)); - // Start all master components - super.createStartStages(); + // Init/Start/Prepare components + List commands = List.of(Command.INIT, Command.START, Command.PREPARE); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commands, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentRestartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentRestartJob.java index 820ff6fb..769f7b48 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentRestartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentRestartJob.java @@ -18,7 +18,13 @@ */ package org.apache.bigtop.manager.server.command.job.component; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ComponentRestartJob extends AbstractComponentJob { @@ -28,9 +34,12 @@ public ComponentRestartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); - super.createStartStages(); + // Restart services + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.STOP, commandDTO)); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.START, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStartJob.java index 94d2c6c3..13212034 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStartJob.java @@ -18,7 +18,12 @@ */ package org.apache.bigtop.manager.server.command.job.component; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ComponentStartJob extends AbstractComponentJob { @@ -28,7 +33,10 @@ public ComponentStartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStartStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStopJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStopJob.java index cef74f65..20c66b9b 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStopJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/component/ComponentStopJob.java @@ -18,7 +18,12 @@ */ package org.apache.bigtop.manager.server.command.job.component; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ComponentStopJob extends AbstractComponentJob { @@ -28,7 +33,10 @@ public ComponentStopJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/AbstractHostJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/AbstractHostJob.java index d1af00f0..7135efc0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/AbstractHostJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/AbstractHostJob.java @@ -18,25 +18,17 @@ */ package org.apache.bigtop.manager.server.command.job.host; -import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.query.ComponentQuery; import org.apache.bigtop.manager.dao.repository.ComponentDao; import org.apache.bigtop.manager.server.command.job.AbstractJob; import org.apache.bigtop.manager.server.command.job.JobContext; -import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; -import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; -import org.apache.bigtop.manager.server.command.stage.StageContext; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.model.dto.ComponentDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.utils.StackDAGUtils; -import org.apache.bigtop.manager.server.utils.StackUtils; - -import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class AbstractHostJob extends AbstractJob { @@ -58,77 +50,20 @@ protected void beforeCreateStages() { super.beforeCreateStages(); } - protected StageContext createStageContext(String componentName, List hostnames) { - StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - - ServiceDTO serviceDTO = StackUtils.getServiceDTOByComponentName(componentName); - ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); - - stageContext.setHostnames(hostnames); - stageContext.setServiceDTO(serviceDTO); - stageContext.setComponentDTO(componentDTO); - - return stageContext; - } - - protected void createStartStages() { - List componentPOList = getComponentPOList(); - List todoList = StackDAGUtils.getTodoList(getComponentNames(componentPOList), Command.START); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = getHostnames(); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentStartStage(stageContext)); - } - } - - protected void createStopStages() { - List componentPOList = getComponentPOList(); - List todoList = StackDAGUtils.getTodoList(getComponentNames(componentPOList), Command.STOP); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = getHostnames(); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(componentName, hostnames); - stages.add(new ComponentStopStage(stageContext)); - } - } - - private List getComponentPOList() { - ComponentQuery query = ComponentQuery.builder() + protected Map> getComponentHostsMap() { + List hostnames = getHostnames(); + ComponentQuery componentQuery = ComponentQuery.builder() .clusterId(clusterPO.getId()) - .hostnames(getHostnames()) + .hostnames(hostnames) .build(); - return componentDao.findByQuery(query); - } - - private List getComponentNames(List componentPOList) { - if (componentPOList == null) { - return new ArrayList<>(); - } else { - return componentPOList.stream().map(ComponentPO::getName).distinct().toList(); + List componentPOList = componentDao.findByQuery(componentQuery); + Map> componentHostsMap = new HashMap<>(); + for (ComponentPO componentPO : componentPOList) { + List hosts = componentHostsMap.computeIfAbsent(componentPO.getName(), k -> new ArrayList<>()); + hosts.add(componentPO.getHostname()); } + + return componentHostsMap; } private List getHostnames() { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostRestartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostRestartJob.java index 9bc0b337..c8453748 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostRestartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostRestartJob.java @@ -18,7 +18,13 @@ */ package org.apache.bigtop.manager.server.command.job.host; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class HostRestartJob extends AbstractHostJob { @@ -28,9 +34,12 @@ public HostRestartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); - super.createStartStages(); + // Restart services + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.STOP, commandDTO)); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.START, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStartJob.java index 80c2c85a..8a4563e7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStartJob.java @@ -18,7 +18,12 @@ */ package org.apache.bigtop.manager.server.command.job.host; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class HostStartJob extends AbstractHostJob { @@ -28,7 +33,10 @@ public HostStartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStartStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStopJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStopJob.java index 2ab781ba..198a8e52 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStopJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/host/HostStopJob.java @@ -18,7 +18,12 @@ */ package org.apache.bigtop.manager.server.command.job.host; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class HostStopJob extends AbstractHostJob { @@ -28,7 +33,10 @@ public HostStopJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/AbstractServiceJob.java index b47292ac..d32400b9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/AbstractServiceJob.java @@ -18,7 +18,6 @@ */ package org.apache.bigtop.manager.server.command.job.service; -import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.query.ComponentQuery; import org.apache.bigtop.manager.dao.repository.ComponentDao; @@ -28,25 +27,13 @@ import org.apache.bigtop.manager.dao.repository.ServiceDao; import org.apache.bigtop.manager.server.command.job.AbstractJob; import org.apache.bigtop.manager.server.command.job.JobContext; -import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; -import org.apache.bigtop.manager.server.command.stage.ComponentAddStage; -import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage; -import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage; -import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; -import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; -import org.apache.bigtop.manager.server.command.stage.StageContext; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.model.dto.ComponentDTO; -import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; -import org.apache.bigtop.manager.server.utils.StackDAGUtils; -import org.apache.bigtop.manager.server.utils.StackUtils; - -import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class AbstractServiceJob extends AbstractJob { @@ -76,146 +63,20 @@ protected void beforeCreateStages() { super.beforeCreateStages(); } - protected StageContext createStageContext(String serviceName, String componentName, List hostnames) { - StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - - ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName); - ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); - - stageContext.setHostnames(hostnames); - stageContext.setServiceDTO(serviceDTO); - stageContext.setComponentDTO(componentDTO); - - return stageContext; - } - - protected List getComponentNames() { + protected Map> getComponentHostsMap() { List serviceNames = getServiceNames(); ComponentQuery componentQuery = ComponentQuery.builder() .clusterId(clusterPO.getId()) .serviceNames(serviceNames) .build(); List componentPOList = componentDao.findByQuery(componentQuery); - - return componentPOList.stream().map(ComponentPO::getName).distinct().toList(); - } - - protected String findServiceNameByComponentName(String componentName) { - return StackUtils.getServiceDTOByComponentName(componentName).getName(); - } - - protected List findHostnamesByComponentName(String componentName) { - ComponentQuery componentQuery = ComponentQuery.builder() - .clusterId(clusterPO.getId()) - .name(componentName) - .build(); - List componentPOList = componentDao.findByQuery(componentQuery); - if (componentPOList == null) { - return new ArrayList<>(); - } else { - return componentPOList.stream().map(ComponentPO::getHostname).toList(); - } - } - - protected void createCacheStage() { - StageContext stageContext = StageContext.fromCommandDTO(jobContext.getCommandDTO()); - stages.add(new CacheFileUpdateStage(stageContext)); - } - - protected void createAddStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.ADD); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - String serviceName = findServiceNameByComponentName(componentName); - List hostnames = findHostnamesByComponentName(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(serviceName, componentName, hostnames); - stages.add(new ComponentAddStage(stageContext)); - } - } - - protected void createConfigureStages() { - for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { - for (ComponentHostDTO componentHost : serviceCommand.getComponentHosts()) { - String serviceName = serviceCommand.getServiceName(); - String componentName = componentHost.getComponentName(); - List hostnames = componentHost.getHostnames(); - - StageContext stageContext = createStageContext(serviceName, componentName, hostnames); - stages.add(new ComponentConfigureStage(stageContext)); - } - } - } - - protected void createStartStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.START); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - String serviceName = findServiceNameByComponentName(componentName); - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = findHostnamesByComponentName(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(serviceName, componentName, hostnames); - stages.add(new ComponentStartStage(stageContext)); + Map> componentHostsMap = new HashMap<>(); + for (ComponentPO componentPO : componentPOList) { + List hosts = componentHostsMap.computeIfAbsent(componentPO.getName(), k -> new ArrayList<>()); + hosts.add(componentPO.getHostname()); } - } - - protected void createStopStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.STOP); - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - String serviceName = findServiceNameByComponentName(componentName); - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = findHostnamesByComponentName(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(serviceName, componentName, hostnames); - stages.add(new ComponentStopStage(stageContext)); - } - } - - protected void createCheckStages() { - List todoList = StackDAGUtils.getTodoList(getComponentNames(), Command.CHECK); - - for (String componentCommand : todoList) { - String[] split = componentCommand.split("-"); - String componentName = split[0]; - String serviceName = findServiceNameByComponentName(componentName); - - if (StackUtils.isClientComponent(componentName)) { - continue; - } - - List hostnames = findHostnamesByComponentName(componentName); - if (CollectionUtils.isEmpty(hostnames)) { - continue; - } - - StageContext stageContext = createStageContext(serviceName, componentName, List.of(hostnames.get(0))); - stages.add(new ComponentCheckStage(stageContext)); - } + return componentHostsMap; } private List getServiceNames() { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java index 35553ebd..690f3542 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java @@ -18,12 +18,14 @@ */ package org.apache.bigtop.manager.server.command.job.service; +import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.dao.po.ComponentPO; import org.apache.bigtop.manager.dao.po.HostPO; import org.apache.bigtop.manager.dao.po.ServiceConfigPO; import org.apache.bigtop.manager.dao.po.ServiceConfigSnapshotPO; import org.apache.bigtop.manager.dao.po.ServicePO; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.model.converter.ComponentConverter; @@ -59,45 +61,41 @@ protected void createStages() { // Update cache files super.createCacheStage(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + // Install components - super.createAddStages(); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.ADD, commandDTO)); - // Configure services - super.createConfigureStages(); + // Configure components + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.CONFIGURE, commandDTO)); - // Start all master components - super.createStartStages(); + // Init/Start/Prepare components + // Since the order of these stages might be mixed up, we need to sort and add them together. + // For example, the order usually is init -> start -> prepare, but Hive Metastore init requires MySQL Server to + // be prepared. + List commands = List.of(Command.INIT, Command.START, Command.PREPARE); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commands, commandDTO)); // Check all master components after started - super.createCheckStages(); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.CHECK, commandDTO)); } @Override - protected List getComponentNames() { - List componentNames = new ArrayList<>(); - for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { - List componentHosts = serviceCommand.getComponentHosts(); - for (ComponentHostDTO componentHost : componentHosts) { - String componentName = componentHost.getComponentName(); - componentNames.add(componentName); - } - } - - return componentNames; - } - - @Override - protected List findHostnamesByComponentName(String componentName) { - for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { - List componentHosts = serviceCommand.getComponentHosts(); - for (ComponentHostDTO componentHost : componentHosts) { - if (componentHost.getComponentName().equals(componentName)) { - return componentHost.getHostnames(); - } - } - } - - return new ArrayList<>(); + protected Map> getComponentHostsMap() { + Map> componentHostsMap = new HashMap<>(); + + jobContext.getCommandDTO().getServiceCommands().stream() + .map(ServiceCommandDTO::getComponentHosts) + .forEach(componentHosts -> { + for (ComponentHostDTO componentHost : componentHosts) { + String componentName = componentHost.getComponentName(); + List hostnames = componentHost.getHostnames(); + componentHostsMap.put(componentName, hostnames); + } + }); + + return componentHostsMap; } @Override @@ -144,7 +142,7 @@ private void saveService(ServiceCommandDTO serviceCommand) { Long clusterId = commandDTO.getClusterId(); String serviceName = serviceCommand.getServiceName(); - // 1. Persist service + // Persist services StackDTO stackDTO = StackUtils.getServiceStack(serviceName); ServiceDTO serviceDTO = StackUtils.getServiceDTO(serviceName); ServicePO servicePO = ServiceConverter.INSTANCE.fromDTO2PO(serviceDTO); @@ -153,7 +151,7 @@ private void saveService(ServiceCommandDTO serviceCommand) { servicePO.setStatus(HealthyStatusEnum.UNHEALTHY.getCode()); serviceDao.save(servicePO); - // 2. Persist components + // Persist components List componentPOList = new ArrayList<>(); for (ComponentHostDTO componentHostDTO : serviceCommand.getComponentHosts()) { String componentName = componentHostDTO.getComponentName(); @@ -172,7 +170,7 @@ private void saveService(ServiceCommandDTO serviceCommand) { componentDao.saveAll(componentPOList); - // 3. Persist current configs + // Persist current configs Map confMap = new HashMap<>(); List configs = serviceCommand.getConfigs(); List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(configs); @@ -184,7 +182,7 @@ private void saveService(ServiceCommandDTO serviceCommand) { serviceConfigDao.saveAll(serviceConfigPOList); - // 4. Create initial config snapshot + // Create initial config snapshot ServiceConfigSnapshotPO serviceConfigSnapshotPO = new ServiceConfigSnapshotPO(); serviceConfigSnapshotPO.setName("initial"); serviceConfigSnapshotPO.setDesc("Initial config snapshot"); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceCheckJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceCheckJob.java index 7f3bc192..b0bc2885 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceCheckJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceCheckJob.java @@ -19,12 +19,14 @@ package org.apache.bigtop.manager.server.command.job.service; import org.apache.bigtop.manager.dao.po.ServicePO; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import java.util.List; +import java.util.Map; public class ServiceCheckJob extends AbstractServiceJob { @@ -34,7 +36,10 @@ public ServiceCheckJob(JobContext jobContext) { @Override protected void createStages() { - super.createCheckStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceConfigureJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceConfigureJob.java index 03fed56e..f6325685 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceConfigureJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceConfigureJob.java @@ -18,7 +18,13 @@ */ package org.apache.bigtop.manager.server.command.job.service; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; + +import java.util.List; +import java.util.Map; public class ServiceConfigureJob extends AbstractServiceJob { @@ -31,12 +37,14 @@ protected void createStages() { // Update cache files super.createCacheStage(); - // Configure services - super.createConfigureStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.CONFIGURE, commandDTO)); // Restart services - super.createStopStages(); - super.createStartStages(); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.STOP, commandDTO)); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.START, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceRestartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceRestartJob.java index 1124e865..e32e36d8 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceRestartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceRestartJob.java @@ -18,13 +18,16 @@ */ package org.apache.bigtop.manager.server.command.job.service; +import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.dao.po.ServicePO; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import java.util.List; +import java.util.Map; public class ServiceRestartJob extends AbstractServiceJob { @@ -34,9 +37,12 @@ public ServiceRestartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); - super.createStartStages(); + // Restart services + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.STOP, commandDTO)); + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, Command.START, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStartJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStartJob.java index 06d8f9bc..b4bb236a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStartJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStartJob.java @@ -19,12 +19,14 @@ package org.apache.bigtop.manager.server.command.job.service; import org.apache.bigtop.manager.dao.po.ServicePO; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import java.util.List; +import java.util.Map; public class ServiceStartJob extends AbstractServiceJob { @@ -34,7 +36,10 @@ public ServiceStartJob(JobContext jobContext) { @Override protected void createStages() { - super.createStartStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStopJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStopJob.java index bb1296b9..ff45c5d3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStopJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceStopJob.java @@ -19,12 +19,14 @@ package org.apache.bigtop.manager.server.command.job.service; import org.apache.bigtop.manager.dao.po.ServicePO; +import org.apache.bigtop.manager.server.command.helper.ComponentStageHelper; import org.apache.bigtop.manager.server.command.job.JobContext; import org.apache.bigtop.manager.server.enums.HealthyStatusEnum; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import java.util.List; +import java.util.Map; public class ServiceStopJob extends AbstractServiceJob { @@ -34,7 +36,10 @@ public ServiceStopJob(JobContext jobContext) { @Override protected void createStages() { - super.createStopStages(); + CommandDTO commandDTO = jobContext.getCommandDTO(); + Map> componentHostsMap = getComponentHostsMap(); + + stages.addAll(ComponentStageHelper.createComponentStages(componentHostsMap, commandDTO)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java new file mode 100644 index 00000000..2cc48652 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.stage; + +import org.apache.bigtop.manager.server.command.task.ComponentInitTask; +import org.apache.bigtop.manager.server.command.task.Task; + +public class ComponentInitStage extends AbstractComponentStage { + + public ComponentInitStage(StageContext stageContext) { + super(stageContext); + } + + @Override + protected Task createTask(String hostname) { + return new ComponentInitTask(createTaskContext(hostname)); + } + + @Override + public String getName() { + return "Init " + stageContext.getComponentDTO().getDisplayName(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java new file mode 100644 index 00000000..3b91dc8b --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.stage; + +import org.apache.bigtop.manager.server.command.task.ComponentPrepareTask; +import org.apache.bigtop.manager.server.command.task.Task; + +public class ComponentPrepareStage extends AbstractComponentStage { + + public ComponentPrepareStage(StageContext stageContext) { + super(stageContext); + } + + @Override + protected Task createTask(String hostname) { + return new ComponentPrepareTask(createTaskContext(hostname)); + } + + @Override + public String getName() { + return "Prepare " + stageContext.getComponentDTO().getDisplayName(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInitTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInitTask.java new file mode 100644 index 00000000..72d8c686 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInitTask.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.task; + +import org.apache.bigtop.manager.common.enums.Command; + +public class ComponentInitTask extends AbstractComponentTask { + + public ComponentInitTask(TaskContext taskContext) { + super(taskContext); + } + + @Override + protected Command getCommand() { + return Command.INIT; + } + + @Override + public String getName() { + return "Init " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentPrepareTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentPrepareTask.java new file mode 100644 index 00000000..418da2c1 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentPrepareTask.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.task; + +import org.apache.bigtop.manager.common.enums.Command; + +public class ComponentPrepareTask extends AbstractComponentTask { + + public ComponentPrepareTask(TaskContext taskContext) { + super(taskContext); + } + + @Override + protected Command getCommand() { + return Command.PREPARE; + } + + @Override + public String getName() { + return "Prepare " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackDAGUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackDAGUtils.java deleted file mode 100644 index 994170f2..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackDAGUtils.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.server.utils; - -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.exception.ServerException; - -import java.util.ArrayList; -import java.util.List; - -public class StackDAGUtils { - - public static List orderTodoList(List componentCommandNames) { - try { - List orderedList = - StackUtils.DAG.getAllNodesList().isEmpty() ? new ArrayList<>() : StackUtils.DAG.topologicalSort(); - - // Re-order the commands, since order.json does not contain all commands, - // only contains which has dependencies, we need to add the rest to the end. - orderedList.retainAll(componentCommandNames); - componentCommandNames.removeAll(orderedList); - orderedList.addAll(componentCommandNames); - - return orderedList; - } catch (Exception e) { - throw new ServerException(e); - } - } - - public static List getTodoList(List componentNames, Command command) { - try { - List orderedList = - StackUtils.DAG.getAllNodesList().isEmpty() ? new ArrayList<>() : StackUtils.DAG.topologicalSort(); - List componentCommandNames = new ArrayList<>(componentNames.stream() - .map(x -> x + "-" + command.name().toUpperCase()) - .toList()); - - // Re-order the commands, since order.json does not contain all commands, - // only contains which has dependencies, we need to add the rest to the end. - orderedList.retainAll(componentCommandNames); - componentCommandNames.removeAll(orderedList); - orderedList.addAll(componentCommandNames); - - return orderedList; - } catch (Exception e) { - throw new ServerException(e); - } - } -} diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json index e5b220ce..e15e27e8 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json @@ -1,6 +1,8 @@ { + "HIVE_METASTORE-INIT": [ + "MYSQL_SERVER-PREPARE" + ], "HIVE_METASTORE-START": [ - "MYSQL_SERVER-START", "NAMENODE-START", "NODEMANAGER-START" ], diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json index 6591b238..2f55e86d 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json @@ -1,7 +1,4 @@ { - "KAFKA_BROKER-ADD": [ - "ZOOKEEPER_SERVER-ADD" - ], "KAFKA_BROKER-START": [ "ZOOKEEPER_SERVER-START" ], diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java index 181ab29a..7b19663a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java @@ -48,21 +48,7 @@ public ShellResult add(Params params) { Properties properties = new Properties(); properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); - ShellResult shellResult = super.add(params, properties); - - // Download mysql jdbc driver - RepoInfo repoInfo = LocalSettings.repos().stream() - .filter(r -> OSDetection.getArch().equals(r.getArch()) && r.getType() == 2) - .findFirst() - .orElseThrow(() -> new RuntimeException("Cannot find repo for os: [" + OSDetection.getOS() - + "] and arch: [" + OSDetection.getArch() + "]")); - String mysqlDriver = repoInfo.getBaseUrl() + "/mysql-connector-j-8.0.33.jar"; - TarballDownloader.download(mysqlDriver, params.stackHome()); - LinuxFileUtils.moveFile(params.stackHome() + "/mysql-connector-j-8.0.33.jar", params.serviceHome() + "/lib/"); - LinuxFileUtils.updateOwner(params.serviceHome() + "/lib", params.user(), params.group(), true); - LinuxFileUtils.updatePermissions(params.serviceHome() + "/lib", Constants.PERMISSION_755, true); - - return shellResult; + return super.add(params, properties); } @Override @@ -70,6 +56,13 @@ public ShellResult configure(Params params) { return HiveSetup.configure(params); } + @Override + public ShellResult init(Params params) { + downloadMySQLJdbcDriver(params); + initSchema(params); + return ShellResult.success(); + } + @Override public ShellResult start(Params params) { configure(params); @@ -119,19 +112,36 @@ public ShellResult status(Params params) { return LinuxOSUtils.checkProcess(hiveParams.getHiveMetastorePidFile()); } - private void initSchema(Params params) throws Exception { - HiveParams hiveParams = (HiveParams) params; - String cmd = hiveParams.serviceHome() + "/bin/schematool -validate -dbType mysql"; - ShellResult shellResult = LinuxOSUtils.sudoExecCmd(cmd, hiveParams.user()); - String clusterName = LocalSettings.cluster().getName(); - if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE - && shellResult.getErrMsg().contains("Table '" + clusterName + "_hive.VERSION' doesn't exist")) { - // init schema - cmd = hiveParams.serviceHome() + "/bin/schematool -initSchema -dbType mysql"; - shellResult = LinuxOSUtils.sudoExecCmd(cmd, hiveParams.user()); - if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) { - throw new StackException(shellResult.getErrMsg()); + private void downloadMySQLJdbcDriver(Params params) { + RepoInfo repoInfo = LocalSettings.repos().stream() + .filter(r -> OSDetection.getArch().equals(r.getArch()) && r.getType() == 2) + .findFirst() + .orElseThrow(() -> new RuntimeException("Cannot find repo for os: [" + OSDetection.getOS() + + "] and arch: [" + OSDetection.getArch() + "]")); + String mysqlDriver = repoInfo.getBaseUrl() + "/mysql-connector-j-8.0.33.jar"; + TarballDownloader.download(mysqlDriver, params.stackHome()); + LinuxFileUtils.moveFile(params.stackHome() + "/mysql-connector-j-8.0.33.jar", params.serviceHome() + "/lib/"); + LinuxFileUtils.updateOwner(params.serviceHome() + "/lib", params.user(), params.group(), true); + LinuxFileUtils.updatePermissions(params.serviceHome() + "/lib", Constants.PERMISSION_755, true); + } + + private void initSchema(Params params) { + try { + HiveParams hiveParams = (HiveParams) params; + String cmd = hiveParams.serviceHome() + "/bin/schematool -validate -dbType mysql"; + ShellResult shellResult = LinuxOSUtils.sudoExecCmd(cmd, hiveParams.user()); + String clusterName = LocalSettings.cluster().getName(); + if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE + && shellResult.getErrMsg().contains("Table '" + clusterName + "_hive.VERSION' doesn't exist")) { + // init schema + cmd = hiveParams.serviceHome() + "/bin/schematool -initSchema -dbType mysql"; + shellResult = LinuxOSUtils.sudoExecCmd(cmd, hiveParams.user()); + if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) { + throw new StackException(shellResult.getErrMsg()); + } } + } catch (IOException e) { + throw new StackException(e); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java index 2bd67de1..8131fa00 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java @@ -68,6 +68,17 @@ public ShellResult add(Params params, Properties properties) { return ShellResult.success(); } + @Override + public ShellResult init(Params params) { + return ShellResult.success(); + } + + @Override + public ShellResult prepare(Params params) { + return ShellResult.success(); + } + + @Override public ShellResult restart(Params params) { ShellResult shellResult = stop(params); if (shellResult.getExitCode() != 0) { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java index 3c95dc82..337814aa 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java @@ -28,7 +28,7 @@ public interface Script extends PrioritySPI { /** - * Install the component. + * Install the component, download and extract tarball files. * * @param params the parameters required for installation * @return a ShellResult object containing the result of the installation process @@ -36,13 +36,25 @@ public interface Script extends PrioritySPI { ShellResult add(Params params); /** - * Configure the component. + * Configure the component, usually only for config files creation. * * @param params the parameters required for configuration * @return a ShellResult object containing the result of the configuration process */ ShellResult configure(Params params); + /** + * Initialize the component, run necessary tasks which are required by the component before it can be started. + * Except for create config files, which is done in {@link #configure(Params)}. + * This method is used to run tasks like initializing databases, setting up initial data, etc. + * Like Hive Metastore database initialization. + * Should only be run in service/component add job. + * + * @param params the parameters required for initialize + * @return a ShellResult object containing the result of the initialize process + */ + ShellResult init(Params params); + /** * Start the component. * @@ -51,6 +63,17 @@ public interface Script extends PrioritySPI { */ ShellResult start(Params params); + /** + * Prepare the component, Unlike {@link #configure(Params)} and {@link #init(Params)}. + * This method is to prepare the environment which are required by cluster after component starts. + * Like change default root password for MySQL or upload files to HDFS. + * Should only be run in service/component add job. + * + * @param params the parameters required for prepare + * @return a ShellResult object containing the result of the prepare process + */ + ShellResult prepare(Params params); + /** * Stop the component. * @@ -69,7 +92,7 @@ public interface Script extends PrioritySPI { /** * Check the healthy status of the component. - * A simple status check which will only check if the process is running + * A simple status check which will only check if the process is running. * use {@link #check(Params)} to check the real healthy status of the component with smoke tests. * * @param params the parameters required to check the status diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java index 7cfcdbd7..bff5e436 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java @@ -42,35 +42,7 @@ public ShellResult add(Params params) { Properties properties = new Properties(); properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); - super.add(params, properties); - - // Initialize server after added - log.info("Initializing MySQL root user"); - MySQLParams mysqlParams = (MySQLParams) params; - String user = params.user(); - String binDir = params.serviceHome() + "/bin"; - String password = mysqlParams.getRootPassword(); - configure(params); - runCommand(binDir + "/mysqld --initialize-insecure", user); - start(params); - runCommand( - MessageFormat.format( - "{0}/mysql -u root -e \"ALTER USER 'root'@'localhost' IDENTIFIED BY ''{1}'';\"", - binDir, password), - user); - runCommand( - MessageFormat.format( - "{0}/mysql -u root -p''{1}'' -e \"CREATE USER ''root''@''%'' IDENTIFIED BY ''{1}'';\"", - binDir, password), - user); - runCommand( - MessageFormat.format( - "{0}/mysql -u root -p''{1}'' -e \"GRANT ALL PRIVILEGES ON *.* TO ''root''@''%'' WITH GRANT OPTION;\"", - binDir, password), - user); - stop(params); - - return ShellResult.success(); + return super.add(params, properties); } @Override @@ -78,6 +50,14 @@ public ShellResult configure(Params params) { return MySQLSetup.configure(params); } + @Override + public ShellResult init(Params params) { + String user = params.user(); + String binDir = params.serviceHome() + "/bin"; + runCommand(binDir + "/mysqld --initialize-insecure", user); + return ShellResult.success(); + } + @Override public ShellResult start(Params params) { configure(params); @@ -106,6 +86,32 @@ public ShellResult start(Params params) { } } + @Override + public ShellResult prepare(Params params) { + MySQLParams mysqlParams = (MySQLParams) params; + String user = params.user(); + String binDir = params.serviceHome() + "/bin"; + String password = mysqlParams.getRootPassword(); + runCommand( + MessageFormat.format( + "{0}/mysql -u root -e \"ALTER USER 'root'@'localhost' IDENTIFIED BY ''{1}'';\"", + binDir, password), + user); + runCommand( + MessageFormat.format( + "{0}/mysql -u root -p''{1}'' -e \"CREATE USER ''root''@''%'' IDENTIFIED BY ''{1}'';\"", + binDir, password), + user); + runCommand( + MessageFormat.format( + "{0}/mysql -u root -p''{1}'' -e \"GRANT ALL PRIVILEGES ON *.* TO ''root''@''%'' WITH GRANT OPTION;\"", + binDir, password), + user); + runCommand( + MessageFormat.format("{0}/mysql -u root -p''{1}'' -e \"FLUSH PRIVILEGES;\"", binDir, password), user); + return ShellResult.success(); + } + @Override public ShellResult stop(Params params) { MySQLParams mysqlParams = (MySQLParams) params;