Search in sources :

Example 46 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method decommissionNodes.

public Set<String> decommissionNodes(Stack stack, Map<String, InstanceMetaData> hostsToRemove, ApiClient client) {
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        List<String> stillAvailableRemovableHosts = hostRefList.getItems().stream().filter(apiHostRef -> hostsToRemove.containsKey(apiHostRef.getHostname())).parallel().map(ApiHost::getHostname).collect(Collectors.toList());
        LOGGER.debug("Decommissioning nodes: [{}]", stillAvailableRemovableHosts);
        boolean onlyLostNodesAffected = hostsToRemove.values().stream().allMatch(InstanceMetaData::isDeletedOnProvider);
        String hostGroupName = hostsToRemove.values().stream().map(instanceMetaData -> instanceMetaData.getInstanceGroup().getGroupName()).findFirst().get();
        ClouderaManagerResourceApi apiInstance = clouderaManagerApiFactory.getClouderaManagerResourceApi(client);
        ApiHostNameList body = new ApiHostNameList().items(stillAvailableRemovableHosts);
        ApiCommand apiCommand = apiInstance.hostsDecommissionCommand(body);
        ExtendedPollingResult pollingResult = clouderaManagerPollingServiceProvider.startPollingCmHostDecommissioning(stack, client, apiCommand.getId(), onlyLostNodesAffected, stillAvailableRemovableHosts.size());
        if (pollingResult.isExited()) {
            throw new CancellationException("Cluster was terminated while waiting for host decommission");
        } else if (pollingResult.isTimeout()) {
            if (onlyLostNodesAffected) {
                String warningMessage = "Cloudera Manager decommission host command {} polling timed out, " + "thus we are aborting the decommission and we are retrying it for lost nodes once again.";
                abortDecommissionWithWarningMessage(apiCommand, client, warningMessage);
                retryDecommissionNodes(apiInstance, body, stack, client, stillAvailableRemovableHosts, hostGroupName);
            } else {
                throw new CloudbreakServiceException("Timeout while Cloudera Manager decommissioned host.");
            }
        }
        return stillAvailableRemovableHosts.stream().map(hostsToRemove::get).filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.toSet());
    } catch (ApiException e) {
        LOGGER.error("Failed to decommission hosts: {}", hostsToRemove.keySet(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}
Also used : LoggerFactory(org.slf4j.LoggerFactory) ApiService(com.cloudera.api.swagger.model.ApiService) ApiRole(com.cloudera.api.swagger.model.ApiRole) ApiException(com.cloudera.api.swagger.client.ApiException) ClustersResourceApi(com.cloudera.api.swagger.ClustersResourceApi) ApiRoleState(com.cloudera.api.swagger.model.ApiRoleState) FlowMessageService(com.sequenceiq.cloudbreak.message.FlowMessageService) Map(java.util.Map) ApiHostTemplate(com.cloudera.api.swagger.model.ApiHostTemplate) ClouderaManagerResourceApi(com.cloudera.api.swagger.ClouderaManagerResourceApi) HostTemplatesResourceApi(com.cloudera.api.swagger.HostTemplatesResourceApi) ApiHostTemplateList(com.cloudera.api.swagger.model.ApiHostTemplateList) NotEnoughNodeException(com.sequenceiq.cloudbreak.cluster.service.NotEnoughNodeException) ApiConfig(com.cloudera.api.swagger.model.ApiConfig) ApiHostsToRemoveArgs(com.cloudera.api.swagger.model.ApiHostsToRemoveArgs) ResourceAttributeUtil(com.sequenceiq.cloudbreak.cluster.util.ResourceAttributeUtil) Collection(java.util.Collection) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) Set(java.util.Set) Status(com.sequenceiq.cloudbreak.api.endpoint.v4.common.Status) ExtendedPollingResult(com.sequenceiq.cloudbreak.polling.ExtendedPollingResult) ApiHostNameList(com.cloudera.api.swagger.model.ApiHostNameList) VolumeSetAttributes(com.sequenceiq.cloudbreak.cloud.model.VolumeSetAttributes) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) ApiServiceConfig(com.cloudera.api.swagger.model.ApiServiceConfig) Stream(java.util.stream.Stream) ApiHealthSummary(com.cloudera.api.swagger.model.ApiHealthSummary) Optional(java.util.Optional) CancellationException(com.sequenceiq.cloudbreak.cloud.scheduler.CancellationException) Joiner(com.google.common.base.Joiner) Stack(com.sequenceiq.cloudbreak.domain.stack.Stack) ApiCommand(com.cloudera.api.swagger.model.ApiCommand) HostServiceStatuses(com.sequenceiq.cloudbreak.cluster.status.HostServiceStatuses) HostServiceStatus(com.sequenceiq.cloudbreak.cluster.status.HostServiceStatus) ApiClient(com.cloudera.api.swagger.client.ApiClient) ApiHostList(com.cloudera.api.swagger.model.ApiHostList) Function(java.util.function.Function) ServicesResourceApi(com.cloudera.api.swagger.ServicesResourceApi) ClouderaManagerPollingServiceProvider(com.sequenceiq.cloudbreak.cm.polling.ClouderaManagerPollingServiceProvider) HashSet(java.util.HashSet) Inject(javax.inject.Inject) ApiHost(com.cloudera.api.swagger.model.ApiHost) RolesResourceApi(com.cloudera.api.swagger.RolesResourceApi) NodeIsBusyException(com.sequenceiq.cloudbreak.cluster.service.NodeIsBusyException) Logger(org.slf4j.Logger) ResourceEvent(com.sequenceiq.cloudbreak.event.ResourceEvent) Consumer(java.util.function.Consumer) HostName.hostName(com.sequenceiq.cloudbreak.cloud.model.HostName.hostName) Component(org.springframework.stereotype.Component) HostGroup(com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup) InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) CommandsResourceApi(com.cloudera.api.swagger.CommandsResourceApi) HostName(com.sequenceiq.cloudbreak.cloud.model.HostName) CloudbreakServiceException(com.sequenceiq.cloudbreak.common.exception.CloudbreakServiceException) MgmtServiceResourceApi(com.cloudera.api.swagger.MgmtServiceResourceApi) Comparator(java.util.Comparator) ApiRoleRef(com.cloudera.api.swagger.model.ApiRoleRef) ApiServiceList(com.cloudera.api.swagger.model.ApiServiceList) ClouderaManagerApiFactory(com.sequenceiq.cloudbreak.cm.client.retry.ClouderaManagerApiFactory) StringUtils(org.springframework.util.StringUtils) ApiCommand(com.cloudera.api.swagger.model.ApiCommand) CloudbreakServiceException(com.sequenceiq.cloudbreak.common.exception.CloudbreakServiceException) ApiHostNameList(com.cloudera.api.swagger.model.ApiHostNameList) ApiHostList(com.cloudera.api.swagger.model.ApiHostList) InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) CancellationException(com.sequenceiq.cloudbreak.cloud.scheduler.CancellationException) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) ClouderaManagerResourceApi(com.cloudera.api.swagger.ClouderaManagerResourceApi) ExtendedPollingResult(com.sequenceiq.cloudbreak.polling.ExtendedPollingResult) ApiException(com.cloudera.api.swagger.client.ApiException)

Example 47 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerCommissioner method collectHostsToCommission.

public Map<String, InstanceMetaData> collectHostsToCommission(Stack stack, HostGroup hostGroup, Set<String> hostNames, ApiClient client) {
    Set<InstanceMetaData> hostsInHostGroup = hostGroup.getInstanceGroup().getNotTerminatedAndNotZombieInstanceMetaDataSet();
    Map<String, InstanceMetaData> hostsToCommission = hostsInHostGroup.stream().filter(hostMetadata -> hostNames.contains(hostMetadata.getDiscoveryFQDN())).collect(Collectors.toMap(InstanceMetaData::getDiscoveryFQDN, hostMetadata -> hostMetadata));
    if (hostsToCommission.size() != hostNames.size()) {
        List<String> missingHosts = hostNames.stream().filter(h -> !hostsToCommission.containsKey(h)).collect(Collectors.toList());
        LOGGER.debug("Not all requested hosts found in CB for host group: {}. MissingCount={}, missingHosts=[{}]. Requested hosts: [{}]", hostGroup.getName(), missingHosts.size(), missingHosts, hostNames);
    }
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        List<String> cmHostNames = hostRefList.getItems().stream().map(ApiHost::getHostname).collect(Collectors.toList());
        List<String> matchingCmHosts = hostsToCommission.keySet().stream().filter(hostName -> cmHostNames.contains(hostName)).collect(Collectors.toList());
        Set<String> matchingCmHostSet = new HashSet<>(matchingCmHosts);
        if (matchingCmHosts.size() != hostsToCommission.size()) {
            List<String> missingHostsInCm = hostsToCommission.keySet().stream().filter(h -> !matchingCmHostSet.contains(h)).collect(Collectors.toList());
            LOGGER.debug("Not all requested hosts found in CM. MissingCount={}, missingHosts=[{}]. Requested hosts: [{}]", missingHostsInCm.size(), missingHostsInCm, hostsToCommission.keySet());
        }
        Sets.newHashSet(hostsToCommission.keySet()).stream().filter(hostName -> !matchingCmHostSet.contains(hostName)).forEach(hostsToCommission::remove);
        LOGGER.debug("Collected hosts to commission: [{}]", hostsToCommission);
        return hostsToCommission;
    } catch (ApiException e) {
        // TODO: CB-14929: This exception has to be handled properly. Situations where CM communications fails. There's no retries here right now.
        LOGGER.error("Failed to get host list for cluster: {}", stack.getName(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}
Also used : Stack(com.sequenceiq.cloudbreak.domain.stack.Stack) ApiCommand(com.cloudera.api.swagger.model.ApiCommand) ApiClient(com.cloudera.api.swagger.client.ApiClient) LoggerFactory(org.slf4j.LoggerFactory) ApiHostList(com.cloudera.api.swagger.model.ApiHostList) ApiException(com.cloudera.api.swagger.client.ApiException) ClouderaManagerPollingServiceProvider(com.sequenceiq.cloudbreak.cm.polling.ClouderaManagerPollingServiceProvider) HashSet(java.util.HashSet) Inject(javax.inject.Inject) ApiHost(com.cloudera.api.swagger.model.ApiHost) Map(java.util.Map) ClouderaManagerResourceApi(com.cloudera.api.swagger.ClouderaManagerResourceApi) Logger(org.slf4j.Logger) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) Set(java.util.Set) ExtendedPollingResult(com.sequenceiq.cloudbreak.polling.ExtendedPollingResult) ApiHostNameList(com.cloudera.api.swagger.model.ApiHostNameList) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) Component(org.springframework.stereotype.Component) HostGroup(com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup) InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) CommandsResourceApi(com.cloudera.api.swagger.CommandsResourceApi) CloudbreakServiceException(com.sequenceiq.cloudbreak.common.exception.CloudbreakServiceException) CancellationException(com.sequenceiq.cloudbreak.cloud.scheduler.CancellationException) ClouderaManagerApiFactory(com.sequenceiq.cloudbreak.cm.client.retry.ClouderaManagerApiFactory) CloudbreakServiceException(com.sequenceiq.cloudbreak.common.exception.CloudbreakServiceException) InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) ApiHostList(com.cloudera.api.swagger.model.ApiHostList) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) HashSet(java.util.HashSet) ApiException(com.cloudera.api.swagger.client.ApiException)

Example 48 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerClientActions method checkCmServicesStartedSuccessfully.

public DistroXTestDto checkCmServicesStartedSuccessfully(DistroXTestDto testDto, String user, String password) {
    String serverIp = testDto.getResponse().getCluster().getServerIp();
    ApiClient apiClient = getCmApiClientWithTimeoutDisabledDirect(serverIp, testDto.getName(), V_43, user, password);
    // CHECKSTYLE:OFF
    HostsResourceApi hostsResourceApi = new HostsResourceApi(apiClient);
    // CHECKSTYLE:ON
    try {
        Set<String> masterHostnames = testDto.getResponse().getInstanceGroups().stream().filter(ig -> "master".equals(ig.getName())).flatMap(ig -> ig.getMetadata().stream()).map(InstanceMetaDataV4Response::getDiscoveryFQDN).collect(Collectors.toSet());
        List<ApiHost> apiHosts = hostsResourceApi.readHosts(null, null, "FULL_WITH_HEALTH_CHECK_EXPLANATION").getItems();
        Set<String> servicesNotStarted = apiHosts.stream().filter(apiHost -> masterHostnames.contains(apiHost.getHostname())).flatMap(apiHost -> collectNotStartedServicesOnHost(apiHost).stream()).filter(serviceName -> REQUIRED_SERVICES.contains(serviceName)).collect(Collectors.toSet());
        if (!servicesNotStarted.isEmpty()) {
            LOGGER.error("There are not started required services: {}", servicesNotStarted);
            throw new TestFailException(String.format("There are not started required services: %s", servicesNotStarted));
        }
    } catch (ApiException e) {
        LOGGER.error("Exception when calling HostsResourceApi#readHosts. Response: {}", e.getResponseBody(), e);
        String message = format("Exception when calling HostsResourceApi#readHosts at %s. Response: %s", apiClient.getBasePath(), e.getResponseBody());
        throw new TestFailException(message, e);
    } catch (Exception e) {
        LOGGER.error("Can't read host statuses at: '{}'!", apiClient.getBasePath());
        throw new TestFailException("Can't read host statuses at: " + apiClient.getBasePath(), e);
    }
    return testDto;
}
Also used : Logger(org.slf4j.Logger) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) ApiClient(com.cloudera.api.swagger.client.ApiClient) LoggerFactory(org.slf4j.LoggerFactory) ApiConfigList(com.cloudera.api.swagger.model.ApiConfigList) Log(com.sequenceiq.it.cloudbreak.log.Log) Set(java.util.Set) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ApiException(com.cloudera.api.swagger.client.ApiException) DistroXTestDto(com.sequenceiq.it.cloudbreak.dto.distrox.DistroXTestDto) Value(org.springframework.beans.factory.annotation.Value) RoleConfigGroupsResourceApi(com.cloudera.api.swagger.RoleConfigGroupsResourceApi) ApiRoleState(com.cloudera.api.swagger.model.ApiRoleState) InstanceMetaDataV4Response(com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.response.instancegroup.instancemetadata.InstanceMetaDataV4Response) SdxInternalTestDto(com.sequenceiq.it.cloudbreak.dto.sdx.SdxInternalTestDto) List(java.util.List) Component(org.springframework.stereotype.Component) ClouderaManagerClient(com.sequenceiq.it.cloudbreak.util.clouderamanager.client.ClouderaManagerClient) TestFailException(com.sequenceiq.it.cloudbreak.exception.TestFailException) ApiHost(com.cloudera.api.swagger.model.ApiHost) ListUtils.emptyIfNull(org.apache.commons.collections4.ListUtils.emptyIfNull) ApiRoleRef(com.cloudera.api.swagger.model.ApiRoleRef) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) TestFailException(com.sequenceiq.it.cloudbreak.exception.TestFailException) ApiClient(com.cloudera.api.swagger.client.ApiClient) ApiException(com.cloudera.api.swagger.client.ApiException) TestFailException(com.sequenceiq.it.cloudbreak.exception.TestFailException) ApiHost(com.cloudera.api.swagger.model.ApiHost) ApiException(com.cloudera.api.swagger.client.ApiException)

Example 49 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method enterMaintenanceMode.

public void enterMaintenanceMode(Stack stack, Set<String> hostList, ApiClient client) {
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    String currentHostId = null;
    int successCount = 0;
    List<String> availableHostsIdsFromCm = null;
    LOGGER.debug("Attempting to put {} instances into CM maintenance mode", hostList == null ? 0 : hostList.size());
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        availableHostsIdsFromCm = hostRefList.getItems().stream().filter(apiHostRef -> hostList.contains(apiHostRef.getHostname())).parallel().map(ApiHost::getHostId).collect(Collectors.toList());
        for (String hostId : availableHostsIdsFromCm) {
            currentHostId = hostId;
            hostsResourceApi.enterMaintenanceMode(hostId);
            successCount++;
        }
        LOGGER.debug("Finished putting {} instances into CM maintenance mode. Initial request size: {}, CM availableCount: {}", successCount, hostList == null ? 0 : hostList.size(), availableHostsIdsFromCm == null ? "null" : availableHostsIdsFromCm);
    } catch (ApiException e) {
        LOGGER.error("Failed while putting a node into maintenance mode. nodeId=" + currentHostId + ", successCount=" + successCount, e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}
Also used : ApiHostList(com.cloudera.api.swagger.model.ApiHostList) CloudbreakServiceException(com.sequenceiq.cloudbreak.common.exception.CloudbreakServiceException) HostsResourceApi(com.cloudera.api.swagger.HostsResourceApi) ApiHost(com.cloudera.api.swagger.model.ApiHost) ApiException(com.cloudera.api.swagger.client.ApiException)

Aggregations

ApiHost (com.cloudera.api.swagger.model.ApiHost)49 ApiHostList (com.cloudera.api.swagger.model.ApiHostList)30 InstanceMetaData (com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData)26 Test (org.junit.Test)23 HostsResourceApi (com.cloudera.api.swagger.HostsResourceApi)16 ApiException (com.cloudera.api.swagger.client.ApiException)16 List (java.util.List)14 ApiClient (com.cloudera.api.swagger.client.ApiClient)13 Set (java.util.Set)13 Collectors (java.util.stream.Collectors)13 ClouderaManagerApiFactory (com.sequenceiq.cloudbreak.cm.client.retry.ClouderaManagerApiFactory)12 ApiCommand (com.cloudera.api.swagger.model.ApiCommand)11 Stack (com.sequenceiq.cloudbreak.domain.stack.Stack)11 HostGroup (com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup)11 ApiHealthSummary (com.cloudera.api.swagger.model.ApiHealthSummary)10 ApiRoleRef (com.cloudera.api.swagger.model.ApiRoleRef)10 ClouderaManagerPollingServiceProvider (com.sequenceiq.cloudbreak.cm.polling.ClouderaManagerPollingServiceProvider)10 Logger (org.slf4j.Logger)10 LoggerFactory (org.slf4j.LoggerFactory)10 ClouderaManagerResourceApi (com.cloudera.api.swagger.ClouderaManagerResourceApi)9