Examples with HostsResourceApi com.cloudera.api.swagger.HostsResourceApi used on opensource projects

Example 26 with HostsResourceApi

use of com.cloudera.api.swagger.HostsResourceApi in project cloudbreak by hortonworks.

the class ClouderaManagerCommissioner method recommissionNodes.

/**
 * Attempts to recommission the provided list of hosts
 * @param stack the stack under consideration
 * @param hostsToRecommission hosts to decommission
 * @param client api client to communicate with ClouderaManager
 * @return the hostnames for hosts which were successfully recommissioned
 */
public Set<String> recommissionNodes(Stack stack, Map<String, InstanceMetaData> hostsToRecommission, ApiClient client) {
    // TODO CB-15132: Check status of target nodes / previously issued commands - in case of pod restarts etc.
    // Ideally without needing to persist anything (commandId etc)
    // TODO CB-15132: Deal with situations where CM itself is unavailable. Go back and STOP resources on the cloud-provider.
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    ApiHostList hostRefList;
    try {
        hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
    } catch (ApiException e) {
        LOGGER.error("Failed to communicate with Cloudera Manager", e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
    LOGGER.trace("Target recommissionNodes: count={}, hosts=[{}]", hostsToRecommission.size(), hostsToRecommission.keySet());
    LOGGER.debug("hostsAvailableFromCM: count={}, hosts=[{}]", hostRefList.getItems().size(), hostRefList.getItems().stream().map(ApiHost::getHostname));
    // Not considering the commission states of the nodes. Nodes could be COMMISSIONED with services in STOPPED state.
    List<String> hostsAvailableForRecommission = hostRefList.getItems().stream().filter(apiHostRef -> hostsToRecommission.containsKey(apiHostRef.getHostname())).parallel().map(ApiHost::getHostname).collect(Collectors.toList());
    Set<String> hostsAvailableForRecommissionSet = new HashSet<>(hostsAvailableForRecommission);
    List<String> cmHostsUnavailableForRecommission = hostsToRecommission.keySet().stream().filter(h -> !hostsAvailableForRecommissionSet.contains(h)).collect(Collectors.toList());
    if (cmHostsUnavailableForRecommission.size() != 0) {
        LOGGER.info("Some recommission targets are unavailable in CM: TotalRecommissionTargetCount={}, unavailableInCMCount={}, unavailableInCM=[{}]", hostsToRecommission.size(), cmHostsUnavailableForRecommission.size(), cmHostsUnavailableForRecommission);
    }
    recommissionHosts(stack, client, hostsAvailableForRecommission);
    return hostsAvailableForRecommission.stream().map(hostsToRecommission::get).filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.toSet());
}

Example 27 with HostsResourceApi

use of com.cloudera.api.swagger.HostsResourceApi in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method decommissionNodesStopStart.

public Set<String> decommissionNodesStopStart(Stack stack, Map<String, InstanceMetaData> hostsToRemove, ApiClient client, long pollingTimeout) {
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        LOGGER.trace("Target decommissionNodes: count={}, hosts=[{}]", hostsToRemove.size(), hostsToRemove.keySet());
        LOGGER.debug("hostsAvailableFromCM: count={}, hosts=[{}]", hostRefList.getItems().size(), hostRefList.getItems().stream().map(ApiHost::getHostname));
        List<String> stillAvailableRemovableHosts = hostRefList.getItems().stream().filter(apiHostRef -> hostsToRemove.containsKey(apiHostRef.getHostname())).parallel().map(ApiHost::getHostname).collect(Collectors.toList());
        Set<String> hostsAvailableForDecommissionSet = new HashSet<>(stillAvailableRemovableHosts);
        List<String> cmHostsUnavailableForDecommission = hostsToRemove.keySet().stream().filter(h -> !hostsAvailableForDecommissionSet.contains(h)).collect(Collectors.toList());
        if (cmHostsUnavailableForDecommission.size() != 0) {
            LOGGER.info("Some decommission targets are unavailable in CM: TotalDecommissionTargetCount={}, unavailableInCMCount={}, unavailableInCm=[{}]", hostsToRemove.size(), cmHostsUnavailableForDecommission.size(), cmHostsUnavailableForDecommission);
        }
        ClouderaManagerResourceApi apiInstance = clouderaManagerApiFactory.getClouderaManagerResourceApi(client);
        ApiHostNameList body = new ApiHostNameList().items(stillAvailableRemovableHosts);
        ApiCommand apiCommand = apiInstance.hostsDecommissionCommand(body);
        ExtendedPollingResult pollingResult = clouderaManagerPollingServiceProvider.startPollingCmHostsDecommission(stack, client, apiCommand.getId(), pollingTimeout);
        if (pollingResult.isExited()) {
            throw new CancellationException("Cluster was terminated while waiting for host decommission");
        } else if (pollingResult.isTimeout()) {
            String warningMessage = "Cloudera Manager decommission host command {} polling timed out, " + "thus we are aborting the decommission and we are retrying it for lost nodes once again.";
            abortDecommissionWithWarningMessage(apiCommand, client, warningMessage);
            throw new CloudbreakServiceException(String.format("Timeout while Cloudera Manager decommissioned host. CM command Id: %s", apiCommand.getId()));
        }
        return stillAvailableRemovableHosts.stream().map(hostsToRemove::get).filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.toSet());
    } catch (ApiException e) {
        LOGGER.error("Failed to decommission hosts: {}", hostsToRemove.keySet(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}

Example 28 with HostsResourceApi

use of com.cloudera.api.swagger.HostsResourceApi in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method removeHostsFromCluster.

private void removeHostsFromCluster(List<String> hosts, Stack stack, ApiClient client) {
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        List<String> knownDeletableHosts = hostRefList.getItems().stream().filter(host -> hosts.contains(host.getHostname())).map(ApiHost::getHostname).collect(Collectors.toList());
        if (!knownDeletableHosts.isEmpty()) {
            ApiHostsToRemoveArgs body = new ApiHostsToRemoveArgs();
            body.hostsToRemove(knownDeletableHosts);
            body.deleteHosts(Boolean.TRUE);
            ApiCommand command = hostsResourceApi.removeHostsFromCluster(body);
            LOGGER.debug("Remove hosts from cluster request sent, hosts: {}", knownDeletableHosts);
            ExtendedPollingResult pollingResult = clouderaManagerPollingServiceProvider.startPollingRemoveHostsFromCluster(stack, client, command.getId());
            if (pollingResult.isExited()) {
                throw new CancellationException("Cluster was terminated while waiting for hosts removal from CM.");
            } else if (pollingResult.isTimeout()) {
                throw new CloudbreakServiceException("Timeout while Cloudera Manager tried to remove hosts from cluster.");
            }
        } else {
            LOGGER.debug("Hosts already removed.");
        }
    } catch (ApiException e) {
        LOGGER.error("Failed to remove hosts: {}", Joiner.on(",").join(hosts), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}

Example 29 with HostsResourceApi

use of com.cloudera.api.swagger.HostsResourceApi in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method collectDownscaleCandidates.

public Set<InstanceMetaData> collectDownscaleCandidates(ApiClient client, Stack stack, HostGroup hostGroup, Integer scalingAdjustment, Set<InstanceMetaData> instanceMetaDatasInStack) {
    LOGGER.debug("Collecting downscale candidates");
    Set<InstanceMetaData> instancesForHostGroup = instanceMetaDatasInStack.stream().filter(instanceMetaData -> instanceMetaData.getInstanceGroup().getGroupName().equals(hostGroup.getName())).collect(Collectors.toSet());
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        HostTemplatesResourceApi hostTemplatesResourceApi = clouderaManagerApiFactory.getHostTemplatesResourceApi(client);
        ApiHostTemplateList hostTemplates = hostTemplatesResourceApi.readHostTemplates(stack.getName());
        int replication = hostGroupNodesAreDataNodes(hostTemplates, hostGroup.getName()) ? getReplicationFactor(client, stack.getName()) : 0;
        verifyNodeCount(replication, scalingAdjustment, instancesForHostGroup.size(), 0, stack);
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        Set<InstanceMetaData> instancesToRemove = getUnusedInstances(scalingAdjustment, instancesForHostGroup, hostRefList);
        List<ApiHost> apiHosts = hostRefList.getItems().stream().filter(host -> instancesForHostGroup.stream().filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).anyMatch(instanceMetaData -> instanceMetaData.getDiscoveryFQDN().equals(host.getHostname()))).collect(Collectors.toList());
        Set<String> hostsToRemove = apiHosts.stream().sorted(hostHealthComparator).limit(Math.abs(scalingAdjustment) - instancesToRemove.size()).map(ApiHost::getHostname).collect(Collectors.toSet());
        Set<InstanceMetaData> clouderaManagerNodesToRemove = instancesForHostGroup.stream().filter(instanceMetaData -> hostsToRemove.contains(instanceMetaData.getDiscoveryFQDN())).collect(Collectors.toSet());
        instancesToRemove.addAll(clouderaManagerNodesToRemove);
        LOGGER.debug("Downscale candidates: [{}]", instancesToRemove);
        return instancesToRemove;
    } catch (ApiException e) {
        LOGGER.error("Failed to get host list for cluster: {}", stack.getName(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}

Example 30 with HostsResourceApi

use of com.cloudera.api.swagger.HostsResourceApi in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method decommissionNodes.

public Set<String> decommissionNodes(Stack stack, Map<String, InstanceMetaData> hostsToRemove, ApiClient client) {
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        List<String> stillAvailableRemovableHosts = hostRefList.getItems().stream().filter(apiHostRef -> hostsToRemove.containsKey(apiHostRef.getHostname())).parallel().map(ApiHost::getHostname).collect(Collectors.toList());
        LOGGER.debug("Decommissioning nodes: [{}]", stillAvailableRemovableHosts);
        boolean onlyLostNodesAffected = hostsToRemove.values().stream().allMatch(InstanceMetaData::isDeletedOnProvider);
        String hostGroupName = hostsToRemove.values().stream().map(instanceMetaData -> instanceMetaData.getInstanceGroup().getGroupName()).findFirst().get();
        ClouderaManagerResourceApi apiInstance = clouderaManagerApiFactory.getClouderaManagerResourceApi(client);
        ApiHostNameList body = new ApiHostNameList().items(stillAvailableRemovableHosts);
        ApiCommand apiCommand = apiInstance.hostsDecommissionCommand(body);
        ExtendedPollingResult pollingResult = clouderaManagerPollingServiceProvider.startPollingCmHostDecommissioning(stack, client, apiCommand.getId(), onlyLostNodesAffected, stillAvailableRemovableHosts.size());
        if (pollingResult.isExited()) {
            throw new CancellationException("Cluster was terminated while waiting for host decommission");
        } else if (pollingResult.isTimeout()) {
            if (onlyLostNodesAffected) {
                String warningMessage = "Cloudera Manager decommission host command {} polling timed out, " + "thus we are aborting the decommission and we are retrying it for lost nodes once again.";
                abortDecommissionWithWarningMessage(apiCommand, client, warningMessage);
                retryDecommissionNodes(apiInstance, body, stack, client, stillAvailableRemovableHosts, hostGroupName);
            } else {
                throw new CloudbreakServiceException("Timeout while Cloudera Manager decommissioned host.");
            }
        }
        return stillAvailableRemovableHosts.stream().map(hostsToRemove::get).filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.toSet());
    } catch (ApiException e) {
        LOGGER.error("Failed to decommission hosts: {}", hostsToRemove.keySet(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}