Search in sources :

Example 6 with GatewayConfig

use of com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig in project cloudbreak by hortonworks.

the class SaltOrchestrator method bootstrap.

@Override
public void bootstrap(List<GatewayConfig> allGatewayConfigs, Set<Node> targets, ExitCriteriaModel exitModel) throws CloudbreakOrchestratorException {
    LOGGER.info("Start SaltBootstrap on nodes: {}", targets);
    GatewayConfig primaryGateway = getPrimaryGatewayConfig(allGatewayConfigs);
    Set<String> gatewayTargets = getGatewayPrivateIps(allGatewayConfigs);
    try (SaltConnector sc = new SaltConnector(primaryGateway, restDebug)) {
        uploadSaltConfig(sc, gatewayTargets, exitModel);
        uploadSignKey(sc, primaryGateway, gatewayTargets, targets.stream().map(Node::getPrivateIp).collect(Collectors.toSet()), exitModel);
        OrchestratorBootstrap saltBootstrap = new SaltBootstrap(sc, allGatewayConfigs, targets);
        Callable<Boolean> saltBootstrapRunner = runner(saltBootstrap, exitCriteria, exitModel);
        Future<Boolean> saltBootstrapRunnerFuture = parallelOrchestratorComponentRunner.submit(saltBootstrapRunner);
        saltBootstrapRunnerFuture.get();
    } catch (Exception e) {
        LOGGER.error("Error occurred during the salt bootstrap", e);
        throw new CloudbreakOrchestratorFailedException(e);
    }
    LOGGER.info("SaltBootstrap finished");
}
Also used : CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) OrchestratorBootstrap(com.sequenceiq.cloudbreak.orchestrator.OrchestratorBootstrap) Node(com.sequenceiq.cloudbreak.orchestrator.model.Node) SaltBootstrap(com.sequenceiq.cloudbreak.orchestrator.salt.poller.SaltBootstrap) SaltConnector(com.sequenceiq.cloudbreak.orchestrator.salt.client.SaltConnector) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig)

Example 7 with GatewayConfig

use of com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig in project cloudbreak by hortonworks.

the class AmbariDecommissioner method removeHostsFromOrchestrator.

private PollingResult removeHostsFromOrchestrator(Stack stack, AmbariClient ambariClient, List<String> hostNames) throws CloudbreakException {
    Orchestrator orchestrator = stack.getOrchestrator();
    Map<String, Object> map = new HashMap<>(orchestrator.getAttributes().getMap());
    OrchestratorType orchestratorType = orchestratorTypeResolver.resolveType(orchestrator.getType());
    try {
        if (orchestratorType.containerOrchestrator()) {
            OrchestrationCredential credential = new OrchestrationCredential(orchestrator.getApiEndpoint(), map);
            ContainerOrchestrator containerOrchestrator = containerOrchestratorResolver.get(orchestrator.getType());
            Set<Container> containers = containerRepository.findContainersInCluster(stack.getCluster().getId());
            List<ContainerInfo> containersToDelete = containers.stream().filter(input -> hostNames.contains(input.getHost()) && input.getImage().contains(AMBARI_AGENT.getName())).map(input -> new ContainerInfo(input.getContainerId(), input.getName(), input.getHost(), input.getImage())).collect(Collectors.toList());
            containerOrchestrator.deleteContainer(containersToDelete, credential);
            containerRepository.delete(containers);
            return waitForHostsToLeave(stack, ambariClient, hostNames);
        } else if (orchestratorType.hostOrchestrator()) {
            HostOrchestrator hostOrchestrator = hostOrchestratorResolver.get(stack.getOrchestrator().getType());
            Map<String, String> privateIpsByFQDN = new HashMap<>();
            stack.getInstanceMetaDataAsList().stream().filter(instanceMetaData -> hostNames.stream().anyMatch(hn -> hn.contains(instanceMetaData.getDiscoveryFQDN().split("\\.")[0]))).forEach(instanceMetaData -> privateIpsByFQDN.put(instanceMetaData.getDiscoveryFQDN(), instanceMetaData.getPrivateIp()));
            List<GatewayConfig> allGatewayConfigs = gatewayConfigService.getAllGatewayConfigs(stack);
            hostOrchestrator.tearDown(allGatewayConfigs, privateIpsByFQDN);
        }
    } catch (CloudbreakOrchestratorException e) {
        LOGGER.error("Failed to delete orchestrator components while decommissioning: ", e);
        throw new CloudbreakException("Failed to delete orchestrator components while decommissioning: ", e);
    }
    return SUCCESS;
}
Also used : CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) HttpResponseException(groovyx.net.http.HttpResponseException) LoggerFactory(org.slf4j.LoggerFactory) RSDecommissionStatusCheckerTask(com.sequenceiq.cloudbreak.service.cluster.flow.RSDecommissionStatusCheckerTask) DECOMMISSION_AMBARI_PROGRESS_STATE(com.sequenceiq.cloudbreak.service.cluster.ambari.AmbariOperationType.DECOMMISSION_AMBARI_PROGRESS_STATE) Msg(com.sequenceiq.cloudbreak.core.flow2.stack.Msg) Orchestrator(com.sequenceiq.cloudbreak.domain.Orchestrator) START_SERVICES_AMBARI_PROGRESS_STATE(com.sequenceiq.cloudbreak.service.cluster.ambari.AmbariOperationType.START_SERVICES_AMBARI_PROGRESS_STATE) Collections.singletonList(java.util.Collections.singletonList) PollingResult.isTimeout(com.sequenceiq.cloudbreak.service.PollingResult.isTimeout) BadRequestException(com.sequenceiq.cloudbreak.controller.BadRequestException) Map(java.util.Map) ContainerRepository(com.sequenceiq.cloudbreak.repository.ContainerRepository) HostMetadataState(com.sequenceiq.cloudbreak.common.type.HostMetadataState) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) HostMetadataRepository(com.sequenceiq.cloudbreak.repository.HostMetadataRepository) TlsSecurityService(com.sequenceiq.cloudbreak.service.TlsSecurityService) Collection(java.util.Collection) MAX_ATTEMPTS_FOR_HOSTS(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariOperationService.MAX_ATTEMPTS_FOR_HOSTS) Set(java.util.Set) HttpClientConfig(com.sequenceiq.cloudbreak.client.HttpClientConfig) FlowMessageService(com.sequenceiq.cloudbreak.core.flow2.stack.FlowMessageService) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) NotEnoughNodeException(com.sequenceiq.cloudbreak.service.cluster.NotEnoughNodeException) Collectors(java.util.stream.Collectors) AMBARI_POLLING_INTERVAL(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariOperationService.AMBARI_POLLING_INTERVAL) Sets(com.google.common.collect.Sets) PollingResult.isSuccess(com.sequenceiq.cloudbreak.service.PollingResult.isSuccess) GatewayConfigService(com.sequenceiq.cloudbreak.service.GatewayConfigService) List(java.util.List) AmbariOperationService(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariOperationService) HostOrchestratorResolver(com.sequenceiq.cloudbreak.core.bootstrap.service.host.HostOrchestratorResolver) Entry(java.util.Map.Entry) PostConstruct(javax.annotation.PostConstruct) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig) AmbariClientPollerObject(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariClientPollerObject) DataNodeUtils.sortByUsedSpace(com.sequenceiq.cloudbreak.service.cluster.ambari.DataNodeUtils.sortByUsedSpace) PollingService(com.sequenceiq.cloudbreak.service.PollingService) ContainerOrchestratorResolver(com.sequenceiq.cloudbreak.core.bootstrap.service.container.ContainerOrchestratorResolver) HashMap(java.util.HashMap) HostGroupService(com.sequenceiq.cloudbreak.service.hostgroup.HostGroupService) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) HostMetadata(com.sequenceiq.cloudbreak.domain.HostMetadata) ServiceAndHostService(com.sequenceiq.ambari.client.services.ServiceAndHostService) ConfigParam(com.sequenceiq.cloudbreak.service.cluster.filter.ConfigParam) OrchestratorTypeResolver(com.sequenceiq.cloudbreak.core.bootstrap.service.OrchestratorTypeResolver) OrchestrationCredential(com.sequenceiq.cloudbreak.orchestrator.model.OrchestrationCredential) Objects.requireNonNull(java.util.Objects.requireNonNull) Collections.singletonMap(java.util.Collections.singletonMap) PollingResult(com.sequenceiq.cloudbreak.service.PollingResult) Stack(com.sequenceiq.cloudbreak.domain.Stack) Nonnull(javax.annotation.Nonnull) ContainerInfo(com.sequenceiq.cloudbreak.orchestrator.model.ContainerInfo) AMBARI_AGENT(com.sequenceiq.cloudbreak.orchestrator.container.DockerContainer.AMBARI_AGENT) AmbariDFSSpaceRetrievalTask(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariDFSSpaceRetrievalTask) AmbariHostsLeaveStatusCheckerTask(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariHostsLeaveStatusCheckerTask) HostGroup(com.sequenceiq.cloudbreak.domain.HostGroup) Logger(org.slf4j.Logger) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) Iterator(java.util.Iterator) AVAILABLE(com.sequenceiq.cloudbreak.api.model.Status.AVAILABLE) DECOMMISSION_SERVICES_AMBARI_PROGRESS_STATE(com.sequenceiq.cloudbreak.service.cluster.ambari.AmbariOperationType.DECOMMISSION_SERVICES_AMBARI_PROGRESS_STATE) DNDecommissionStatusCheckerTask(com.sequenceiq.cloudbreak.service.cluster.flow.DNDecommissionStatusCheckerTask) AmbariClientExceptionUtil(com.sequenceiq.cloudbreak.util.AmbariClientExceptionUtil) SUCCESS(com.sequenceiq.cloudbreak.service.PollingResult.SUCCESS) Cluster(com.sequenceiq.cloudbreak.domain.Cluster) Container(com.sequenceiq.cloudbreak.domain.Container) Component(org.springframework.stereotype.Component) OrchestratorType(com.sequenceiq.cloudbreak.common.model.OrchestratorType) STOP_SERVICES_AMBARI_PROGRESS_STATE(com.sequenceiq.cloudbreak.service.cluster.ambari.AmbariOperationType.STOP_SERVICES_AMBARI_PROGRESS_STATE) HostFilterService(com.sequenceiq.cloudbreak.service.cluster.filter.HostFilterService) Collections(java.util.Collections) CloudbreakServiceException(com.sequenceiq.cloudbreak.service.CloudbreakServiceException) AmbariClient(com.sequenceiq.ambari.client.AmbariClient) AmbariClientProvider(com.sequenceiq.cloudbreak.service.cluster.AmbariClientProvider) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) HashMap(java.util.HashMap) OrchestrationCredential(com.sequenceiq.cloudbreak.orchestrator.model.OrchestrationCredential) OrchestratorType(com.sequenceiq.cloudbreak.common.model.OrchestratorType) Orchestrator(com.sequenceiq.cloudbreak.domain.Orchestrator) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) Container(com.sequenceiq.cloudbreak.domain.Container) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) ContainerInfo(com.sequenceiq.cloudbreak.orchestrator.model.ContainerInfo) CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) AmbariClientPollerObject(com.sequenceiq.cloudbreak.service.cluster.flow.AmbariClientPollerObject) Collections.singletonList(java.util.Collections.singletonList) List(java.util.List) ArrayList(java.util.ArrayList) Map(java.util.Map) HashMap(java.util.HashMap) Collections.singletonMap(java.util.Collections.singletonMap)

Example 8 with GatewayConfig

use of com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig in project cloudbreak by hortonworks.

the class OrchestratorRecipeExecutor method postAmbariStartRecipes.

public void postAmbariStartRecipes(Stack stack) throws CloudbreakException {
    HostOrchestrator hostOrchestrator = hostOrchestratorResolver.get(stack.getOrchestrator().getType());
    GatewayConfig gatewayConfig = gatewayConfigService.getPrimaryGatewayConfig(stack);
    try {
        hostOrchestrator.postAmbariStartRecipes(gatewayConfig, collectNodes(stack), clusterDeletionBasedModel(stack.getId(), stack.getCluster().getId()));
    } catch (CloudbreakOrchestratorFailedException e) {
        throw new CloudbreakException(e);
    }
}
Also used : CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig)

Example 9 with GatewayConfig

use of com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig in project cloudbreak by hortonworks.

the class OrchestratorRecipeExecutor method uploadRecipes.

public void uploadRecipes(Stack stack, Collection<HostGroup> hostGroups) throws CloudbreakException {
    HostOrchestrator hostOrchestrator = hostOrchestratorResolver.get(stack.getOrchestrator().getType());
    Map<String, List<RecipeModel>> recipeMap = hostGroups.stream().filter(hg -> !hg.getRecipes().isEmpty()).collect(Collectors.toMap(HostGroup::getName, h -> convert(h.getRecipes())));
    List<GatewayConfig> allGatewayConfigs = gatewayConfigService.getAllGatewayConfigs(stack);
    recipesEvent(stack.getId(), stack.getStatus(), recipeMap);
    try {
        hostOrchestrator.uploadRecipes(allGatewayConfigs, recipeMap, clusterDeletionBasedModel(stack.getId(), stack.getCluster().getId()));
    } catch (CloudbreakOrchestratorFailedException e) {
        throw new CloudbreakException(e);
    }
}
Also used : CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) Joiner(com.google.api.client.util.Joiner) Base64(org.apache.commons.codec.binary.Base64) ArrayList(java.util.ArrayList) InstanceGroup(com.sequenceiq.cloudbreak.domain.InstanceGroup) RecipeModel(com.sequenceiq.cloudbreak.orchestrator.model.RecipeModel) CloudbreakMessagesService(com.sequenceiq.cloudbreak.service.messages.CloudbreakMessagesService) HashSet(java.util.HashSet) Inject(javax.inject.Inject) Status(com.sequenceiq.cloudbreak.api.model.Status) Map(java.util.Map) Stack(com.sequenceiq.cloudbreak.domain.Stack) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) HostGroup(com.sequenceiq.cloudbreak.domain.HostGroup) ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel(com.sequenceiq.cloudbreak.core.bootstrap.service.ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel) InstanceMetaData(com.sequenceiq.cloudbreak.domain.InstanceMetaData) Collection(java.util.Collection) Set(java.util.Set) DEFAULT_RECIPES(com.sequenceiq.cloudbreak.service.cluster.flow.RecipeEngine.DEFAULT_RECIPES) NotFoundException(com.sequenceiq.cloudbreak.controller.NotFoundException) Collectors(java.util.stream.Collectors) Recipe(com.sequenceiq.cloudbreak.domain.Recipe) GatewayConfigService(com.sequenceiq.cloudbreak.service.GatewayConfigService) List(java.util.List) Component(org.springframework.stereotype.Component) HostOrchestratorResolver(com.sequenceiq.cloudbreak.core.bootstrap.service.host.HostOrchestratorResolver) CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) Node(com.sequenceiq.cloudbreak.orchestrator.model.Node) Entry(java.util.Map.Entry) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig) Collections(java.util.Collections) CloudbreakEventService(com.sequenceiq.cloudbreak.service.events.CloudbreakEventService) CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) ArrayList(java.util.ArrayList) List(java.util.List) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig)

Example 10 with GatewayConfig

use of com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig in project cloudbreak by hortonworks.

the class SaltBootstrap method createBootstrap.

private SaltAction createBootstrap() {
    SaltAction saltAction = new SaltAction(SaltActionType.RUN);
    SaltAuth auth = new SaltAuth();
    auth.setPassword(sc.getSaltPassword());
    List<String> targetIps = targets.stream().map(Node::getPrivateIp).collect(Collectors.toList());
    for (GatewayConfig gatewayConfig : allGatewayConfigs) {
        String gatewayAddress = gatewayConfig.getPrivateAddress();
        if (targetIps.contains(gatewayAddress)) {
            Node saltMaster = targets.stream().filter(n -> n.getPrivateIp().equals(gatewayAddress)).findFirst().get();
            SaltMaster master = new SaltMaster();
            master.setAddress(gatewayAddress);
            master.setAuth(auth);
            master.setDomain(saltMaster.getDomain());
            master.setHostName(saltMaster.getHostname());
            // set due to compatibility reasons
            saltAction.setServer(gatewayAddress);
            saltAction.setMaster(master);
            saltAction.addMinion(createMinion(saltMaster));
            saltAction.addMaster(master);
        }
    }
    for (Node minion : targets.stream().filter(node -> !getGatewayPrivateIps().contains(node.getPrivateIp())).collect(Collectors.toList())) {
        saltAction.addMinion(createMinion(minion));
    }
    return saltAction;
}
Also used : GenericResponse(com.sequenceiq.cloudbreak.orchestrator.model.GenericResponse) Logger(org.slf4j.Logger) LoggerFactory(org.slf4j.LoggerFactory) SaltAuth(com.sequenceiq.cloudbreak.orchestrator.salt.domain.SaltAuth) Set(java.util.Set) SaltMaster(com.sequenceiq.cloudbreak.orchestrator.salt.domain.SaltMaster) Collectors(java.util.stream.Collectors) HashSet(java.util.HashSet) HttpStatus(org.springframework.http.HttpStatus) GenericResponses(com.sequenceiq.cloudbreak.orchestrator.model.GenericResponses) List(java.util.List) Minion(com.sequenceiq.cloudbreak.orchestrator.salt.domain.Minion) SaltStates(com.sequenceiq.cloudbreak.orchestrator.salt.states.SaltStates) CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) Node(com.sequenceiq.cloudbreak.orchestrator.model.Node) Map(java.util.Map) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig) OrchestratorBootstrap(com.sequenceiq.cloudbreak.orchestrator.OrchestratorBootstrap) Glob(com.sequenceiq.cloudbreak.orchestrator.salt.client.target.Glob) Collections(java.util.Collections) SaltAction(com.sequenceiq.cloudbreak.orchestrator.salt.domain.SaltAction) SaltConnector(com.sequenceiq.cloudbreak.orchestrator.salt.client.SaltConnector) SaltActionType(com.sequenceiq.cloudbreak.orchestrator.salt.client.SaltActionType) SaltAuth(com.sequenceiq.cloudbreak.orchestrator.salt.domain.SaltAuth) Node(com.sequenceiq.cloudbreak.orchestrator.model.Node) SaltMaster(com.sequenceiq.cloudbreak.orchestrator.salt.domain.SaltMaster) SaltAction(com.sequenceiq.cloudbreak.orchestrator.salt.domain.SaltAction) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig)

Aggregations

GatewayConfig (com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig)29 HostOrchestrator (com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator)17 CloudbreakOrchestratorFailedException (com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException)15 Node (com.sequenceiq.cloudbreak.orchestrator.model.Node)13 CloudbreakException (com.sequenceiq.cloudbreak.service.CloudbreakException)13 InstanceMetaData (com.sequenceiq.cloudbreak.domain.InstanceMetaData)12 CloudbreakOrchestratorException (com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException)12 Stack (com.sequenceiq.cloudbreak.domain.Stack)11 ArrayList (java.util.ArrayList)10 SaltConnector (com.sequenceiq.cloudbreak.orchestrator.salt.client.SaltConnector)9 IOException (java.io.IOException)9 HashSet (java.util.HashSet)9 Map (java.util.Map)9 Cluster (com.sequenceiq.cloudbreak.domain.Cluster)8 List (java.util.List)8 SaltConfig (com.sequenceiq.cloudbreak.orchestrator.model.SaltConfig)7 Set (java.util.Set)7 OrchestratorBootstrap (com.sequenceiq.cloudbreak.orchestrator.OrchestratorBootstrap)6 ExitCriteriaModel (com.sequenceiq.cloudbreak.orchestrator.state.ExitCriteriaModel)6 Collection (java.util.Collection)6