Examples with InstanceMetaData com.sequenceiq.cloudbreak.domain.InstanceMetaData used on opensource projects

Example 46 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.InstanceMetaData in project cloudbreak by hortonworks.

the class UnhealthyInstancesFinalizerTest method shouldFinalizeInstancesMarkedAsTerminated.

@Test
public void shouldFinalizeInstancesMarkedAsTerminated() {
    Stack stack = TestUtil.stack(Status.AVAILABLE, TestUtil.awsCredential());
    CloudCredential cloudCredential = mock(CloudCredential.class);
    when(credentialConverter.convert(stack.getCredential())).thenReturn(cloudCredential);
    String instanceId1 = "i-0f1e0605506aaaaaa";
    String instanceId2 = "i-0f1e0605506bbbbbb";
    Set<InstanceMetaData> candidateUnhealthyInstances = new HashSet<>();
    setupInstanceMetaData(instanceId1, candidateUnhealthyInstances);
    setupInstanceMetaData(instanceId2, candidateUnhealthyInstances);
    List<CloudInstance> cloudInstances = new ArrayList<>();
    CloudInstance cloudInstance1 = setupCloudInstance(instanceId1, cloudInstances);
    CloudInstance cloudInstance2 = setupCloudInstance(instanceId2, cloudInstances);
    when(cloudInstanceConverter.convert(candidateUnhealthyInstances)).thenReturn(cloudInstances);
    List<CloudVmInstanceStatus> cloudVmInstanceStatusList = new ArrayList<>();
    setupCloudVmInstanceStatus(cloudInstance1, InstanceStatus.STOPPED, cloudVmInstanceStatusList);
    setupCloudVmInstanceStatus(cloudInstance2, InstanceStatus.TERMINATED, cloudVmInstanceStatusList);
    when(instanceStateQuery.getCloudVmInstanceStatuses(eq(cloudCredential), any(CloudContext.class), eq(cloudInstances))).thenReturn(cloudVmInstanceStatusList);
    Set<String> unhealthyInstances = underTest.finalizeUnhealthyInstances(stack, candidateUnhealthyInstances);
    assertEquals(1, unhealthyInstances.size());
    assertTrue(unhealthyInstances.contains(instanceId2));
}

Example 47 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterBootstrapperErrorHandlerTest method clusterBootstrapErrorHandlerWhenNodeCountHigherThanZeroAfterTheRollbackThenClusterProvisionFailed.

@Test
public void clusterBootstrapErrorHandlerWhenNodeCountHigherThanZeroAfterTheRollbackThenClusterProvisionFailed() throws CloudbreakOrchestratorFailedException {
    Stack stack = TestUtil.stack();
    doNothing().when(eventService).fireCloudbreakEvent(anyLong(), anyString(), anyString());
    when(orchestrator.getAvailableNodes(any(GatewayConfig.class), anySet())).thenReturn(new ArrayList<>());
    when(instanceGroupRepository.save(any(InstanceGroup.class))).then(returnsFirstArg());
    when(instanceMetaDataRepository.save(any(InstanceMetaData.class))).then(returnsFirstArg());
    doNothing().when(resourceRepository).delete(anyLong());
    when(resourceRepository.findByStackIdAndNameAndType(anyLong(), anyString(), any(ResourceType.class))).thenReturn(new Resource());
    when(connector.removeInstances(any(Stack.class), anySet(), anyString())).thenReturn(new HashSet<>());
    when(instanceMetaDataRepository.findNotTerminatedByPrivateAddress(anyLong(), anyString())).thenAnswer(new Answer<InstanceMetaData>() {

        @Override
        public InstanceMetaData answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            String ip = (String) args[1];
            for (InstanceMetaData instanceMetaData : stack.getRunningInstanceMetaData()) {
                if (instanceMetaData.getPrivateIp().equals(ip)) {
                    return instanceMetaData;
                }
            }
            return null;
        }
    });
    when(instanceGroupRepository.findOneByGroupNameInStack(anyLong(), anyString())).thenAnswer(new Answer<InstanceGroup>() {

        @Override
        public InstanceGroup answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            String name = (String) args[1];
            for (InstanceMetaData instanceMetaData : stack.getRunningInstanceMetaData()) {
                if (instanceMetaData.getInstanceGroup().getGroupName().equals(name)) {
                    InstanceGroup instanceGroup = instanceMetaData.getInstanceGroup();
                    instanceGroup.setNodeCount(2);
                    return instanceGroup;
                }
            }
            return null;
        }
    });
    underTest.terminateFailedNodes(null, orchestrator, TestUtil.stack(), new GatewayConfig("10.0.0.1", "198.0.0.1", "10.0.0.1", 8443, false), prepareNodes(stack));
    verify(eventService, times(4)).fireCloudbreakEvent(anyLong(), anyString(), anyString());
    verify(instanceGroupRepository, times(3)).save(any(InstanceGroup.class));
    verify(instanceMetaDataRepository, times(3)).save(any(InstanceMetaData.class));
    verify(connector, times(3)).removeInstances(any(Stack.class), anySet(), anyString());
    verify(resourceRepository, times(3)).findByStackIdAndNameAndType(anyLong(), anyString(), any(ResourceType.class));
    verify(resourceRepository, times(3)).delete(anyLong());
    verify(instanceGroupRepository, times(3)).findOneByGroupNameInStack(anyLong(), anyString());
}

Example 48 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterBootstrapper method bootstrapNewNodes.

public void bootstrapNewNodes(Long stackId, Set<String> upscaleCandidateAddresses, Collection<String> recoveryHostNames) throws CloudbreakException {
    Stack stack = stackRepository.findOneWithLists(stackId);
    Set<Node> nodes = new HashSet<>();
    Set<Node> allNodes = new HashSet<>();
    boolean recoveredNodes = Integer.valueOf(recoveryHostNames.size()).equals(upscaleCandidateAddresses.size());
    Set<InstanceMetaData> metaDataSet = stack.getRunningInstanceMetaData().stream().filter(im -> im.getPrivateIp() != null && im.getPublicIpWrapper() != null).collect(Collectors.toSet());
    String clusterDomain = metaDataSet.stream().filter(im -> isNoneBlank(im.getDiscoveryFQDN())).findAny().get().getDomain();
    for (InstanceMetaData im : metaDataSet) {
        Node node = createNode(stack.getCustomHostname(), im, clusterDomain, stack.isHostgroupNameAsHostname());
        if (upscaleCandidateAddresses.contains(im.getPrivateIp())) {
            // but only when we would have generated a hostname, otherwise use the cloud provider's default mechanism
            if (recoveredNodes && isNoneBlank(node.getHostname())) {
                Iterator<String> iterator = recoveryHostNames.iterator();
                node.setHostname(iterator.next().split("\\.")[0]);
                iterator.remove();
                LOGGER.info("Set the hostname to {} for address: {}", node.getHostname(), im.getPrivateIp());
            }
            nodes.add(node);
        }
        allNodes.add(node);
    }
    try {
        String stackOrchestratorType = stack.getOrchestrator().getType();
        OrchestratorType orchestratorType = orchestratorTypeResolver.resolveType(stack.getOrchestrator().getType());
        if (orchestratorType.hostOrchestrator()) {
            List<GatewayConfig> allGatewayConfigs = gatewayConfigService.getAllGatewayConfigs(stack);
            bootstrapNewNodesOnHost(stack, allGatewayConfigs, nodes, allNodes);
        } else if (orchestratorType.containerOrchestrator()) {
            LOGGER.info("Skipping bootstrap of the new machines because the stack's orchestrator type is '{}'.", stackOrchestratorType);
        } else {
            LOGGER.error("Orchestrator not found: {}", stackOrchestratorType);
            throw new CloudbreakException("HostOrchestrator not found: " + stackOrchestratorType);
        }
    } catch (CloudbreakOrchestratorCancelledException e) {
        throw new CancellationException(e.getMessage());
    } catch (CloudbreakOrchestratorException e) {
        throw new CloudbreakException(e);
    }
}

Example 49 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterBootstrapper method bootstrapOnHost.

@SuppressFBWarnings("REC_CATCH_EXCEPTION")
@SuppressWarnings("unchecked")
public void bootstrapOnHost(Stack stack) throws CloudbreakException {
    Set<Node> nodes = new HashSet<>();
    String domain = hostDiscoveryService.determineDomain(stack.getCustomDomain(), stack.getName(), stack.isClusterNameAsSubdomain());
    for (InstanceMetaData im : stack.getRunningInstanceMetaData()) {
        if (im.getPrivateIp() == null && im.getPublicIpWrapper() == null) {
            LOGGER.warn("Skipping instance metadata because the public ip and private ips are null '{}'.", im);
        } else {
            String generatedHostName = hostDiscoveryService.generateHostname(stack.getCustomHostname(), im.getInstanceGroupName(), im.getPrivateId(), stack.isHostgroupNameAsHostname());
            nodes.add(new Node(im.getPrivateIp(), im.getPublicIpWrapper(), generatedHostName, domain, im.getInstanceGroupName()));
        }
    }
    try {
        HostOrchestrator hostOrchestrator = hostOrchestratorResolver.get(stack.getOrchestrator().getType());
        List<GatewayConfig> allGatewayConfig = new ArrayList<>();
        Boolean enableKnox = stack.getCluster().getGateway().getEnableGateway();
        for (InstanceMetaData gateway : stack.getGatewayInstanceMetadata()) {
            GatewayConfig gatewayConfig = gatewayConfigService.getGatewayConfig(stack, gateway, enableKnox);
            allGatewayConfig.add(gatewayConfig);
            PollingResult bootstrapApiPolling = hostBootstrapApiPollingService.pollWithTimeoutSingleFailure(hostBootstrapApiCheckerTask, new HostBootstrapApiContext(stack, gatewayConfig, hostOrchestrator), POLL_INTERVAL, MAX_POLLING_ATTEMPTS);
            validatePollingResultForCancellation(bootstrapApiPolling, "Polling of bootstrap API was cancelled.");
        }
        ClusterComponent saltComponent = clusterComponentProvider.getComponent(stack.getCluster().getId(), ComponentType.SALT_STATE);
        if (saltComponent == null) {
            byte[] stateConfigZip = hostOrchestrator.getStateConfigZip();
            saltComponent = new ClusterComponent(ComponentType.SALT_STATE, new Json(singletonMap(ComponentType.SALT_STATE.name(), Base64.encodeBase64String(stateConfigZip))), stack.getCluster());
            clusterComponentProvider.store(saltComponent);
        }
        hostOrchestrator.bootstrap(allGatewayConfig, nodes, clusterDeletionBasedModel(stack.getId(), null));
        InstanceMetaData primaryGateway = stack.getPrimaryGatewayInstance();
        GatewayConfig gatewayConfig = gatewayConfigService.getGatewayConfig(stack, primaryGateway, enableKnox);
        String gatewayIp = gatewayConfigService.getGatewayIp(stack, primaryGateway);
        PollingResult allNodesAvailabilityPolling = hostClusterAvailabilityPollingService.pollWithTimeoutSingleFailure(hostClusterAvailabilityCheckerTask, new HostOrchestratorClusterContext(stack, hostOrchestrator, gatewayConfig, nodes), POLL_INTERVAL, MAX_POLLING_ATTEMPTS);
        validatePollingResultForCancellation(allNodesAvailabilityPolling, "Polling of all nodes availability was cancelled.");
        Orchestrator orchestrator = stack.getOrchestrator();
        orchestrator.setApiEndpoint(gatewayIp + ':' + stack.getGatewayPort());
        orchestrator.setType(hostOrchestrator.name());
        orchestratorRepository.save(orchestrator);
        if (TIMEOUT.equals(allNodesAvailabilityPolling)) {
            clusterBootstrapperErrorHandler.terminateFailedNodes(hostOrchestrator, null, stack, gatewayConfig, nodes);
        }
    } catch (Exception e) {
        throw new CloudbreakException(e);
    }
}

Example 50 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterServiceRunner method runAmbariServices.

public void runAmbariServices(Long stackId) throws CloudbreakException {
    Stack stack = stackService.getByIdWithLists(stackId);
    Cluster cluster = stack.getCluster();
    Orchestrator orchestrator = stack.getOrchestrator();
    MDCBuilder.buildMdcContext(cluster);
    OrchestratorType orchestratorType = orchestratorTypeResolver.resolveType(orchestrator.getType());
    if (orchestratorType.containerOrchestrator()) {
        Map<String, List<Container>> containers = containerRunner.runClusterContainers(stack);
        Container ambariServerContainer = containers.get(DockerContainer.AMBARI_SERVER.name()).stream().findFirst().get();
        String ambariServerIp = ambariServerContainer.getHost();
        HttpClientConfig ambariClientConfig = buildAmbariClientConfig(stack, ambariServerIp);
        clusterService.updateAmbariClientConfig(cluster.getId(), ambariClientConfig);
        Map<String, List<String>> hostsPerHostGroup = new HashMap<>();
        for (Entry<String, List<Container>> containersEntry : containers.entrySet()) {
            List<String> hostNames = new ArrayList<>();
            for (Container container : containersEntry.getValue()) {
                hostNames.add(container.getHost());
            }
            hostsPerHostGroup.put(containersEntry.getKey(), hostNames);
        }
        clusterService.updateHostMetadata(cluster.getId(), hostsPerHostGroup, HostMetadataState.CONTAINER_RUNNING);
    } else if (orchestratorType.hostOrchestrator()) {
        hostRunner.runAmbariServices(stack, cluster);
        String gatewayIp = gatewayConfigService.getPrimaryGatewayIp(stack);
        HttpClientConfig ambariClientConfig = buildAmbariClientConfig(stack, gatewayIp);
        clusterService.updateAmbariClientConfig(cluster.getId(), ambariClientConfig);
        Map<String, List<String>> hostsPerHostGroup = new HashMap<>();
        for (InstanceMetaData instanceMetaData : stack.getRunningInstanceMetaData()) {
            String groupName = instanceMetaData.getInstanceGroup().getGroupName();
            if (!hostsPerHostGroup.keySet().contains(groupName)) {
                hostsPerHostGroup.put(groupName, new ArrayList<>());
            }
            hostsPerHostGroup.get(groupName).add(instanceMetaData.getDiscoveryFQDN());
        }
        clusterService.updateHostMetadata(cluster.getId(), hostsPerHostGroup, HostMetadataState.SERVICES_RUNNING);
    } else {
        LOGGER.info(String.format("Please implement %s orchestrator because it is not on classpath.", orchestrator.getType()));
        throw new CloudbreakException(String.format("Please implement %s orchestrator because it is not on classpath.", orchestrator.getType()));
    }
}