Examples with ApiHost com.cloudera.api.swagger.model.ApiHost used on opensource projects

Example 41 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerHostHealthyStatusChecker method filterForHealthy.

private Set<String> filterForHealthy(ApiHostList hostList) {
    // Recent heartbeat, GOOD_HEALTH
    Set<String> goodHealthSet = new HashSet<>();
    for (ApiHost apiHost : hostList.getItems()) {
        String hostname = apiHost.getHostname();
        Instant lastheatbeat = null;
        if (StringUtils.isNotBlank(apiHost.getLastHeartbeat())) {
            lastheatbeat = Instant.parse(apiHost.getLastHeartbeat());
        }
        ApiHealthSummary healthSummary = apiHost.getHealthSummary();
        boolean inMaintenance = apiHost.getMaintenanceMode();
        ApiCommissionState commissionState = apiHost.getCommissionState();
        LOGGER.trace("CM info for: [{}]: lastHeatbeat={}, lastHeartbeatInstant={}, healthSummary={}, commissionState={}, maint={}", hostname, apiHost.getLastHeartbeat(), lastheatbeat, healthSummary, commissionState, inMaintenance);
        if (lastheatbeat != null && start.isBefore(lastheatbeat) && healthSummary == ApiHealthSummary.GOOD) {
            goodHealthSet.add(hostname);
        }
    }
    LOGGER.info("Found good host count={}", goodHealthSet.size());
    return goodHealthSet;
}

Example 42 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerModificationService method upscaleCluster.

@Override
public List<String> upscaleCluster(Map<HostGroup, Set<InstanceMetaData>> instanceMetaDatasByHostGroup) throws CloudbreakException {
    ClustersResourceApi clustersResourceApi = clouderaManagerApiFactory.getClustersResourceApi(apiClient);
    Set<InstanceMetaData> instanceMetaDatas = instanceMetaDatasByHostGroup.values().stream().flatMap(Collection::stream).collect(Collectors.toSet());
    try {
        LOGGER.debug("Starting cluster upscale with hosts: {}.", instanceMetaDatas.stream().map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.joining(", ")));
        String clusterName = stack.getName();
        Set<String> clusterHostnames = getClusterHostnamesFromCM(clustersResourceApi, clusterName);
        List<ApiHost> hosts = getHostsFromCM();
        LOGGER.debug("Processing outdated cluster hosts. Cluster hostnames from CM: {}", clusterHostnames);
        setHostRackIdForOutdatedClusterHosts(instanceMetaDatas, clusterHostnames, hosts);
        LOGGER.debug("Processing upscaled cluster hosts.");
        Map<String, InstanceMetaData> upscaleInstancesMap = getInstancesMap(clusterHostnames, instanceMetaDatas, true);
        if (!upscaleInstancesMap.isEmpty()) {
            Map<String, ApiHost> upscaleHostsMap = getHostsMap(upscaleInstancesMap, hosts);
            setHostRackIdBatch(upscaleInstancesMap, upscaleHostsMap);
            ApiHostRefList bodyForAddHosts = createUpscaledHostRefList(upscaleInstancesMap, upscaleHostsMap);
            clustersResourceApi.addHosts(clusterName, bodyForAddHosts);
            activateParcels(clustersResourceApi);
            for (HostGroup hostGroup : instanceMetaDatasByHostGroup.keySet()) {
                ApiHostRefList bodyForApplyHostGroupRoles = getBodyForApplyHostGroupRoles(upscaleInstancesMap, upscaleHostsMap, hostGroup);
                applyHostGroupRolesOnUpscaledHosts(bodyForApplyHostGroupRoles, hostGroup.getName());
            }
        } else {
            redistributeParcelsForRecovery();
            activateParcels(clustersResourceApi);
        }
        deployClientConfig(clustersResourceApi, stack);
        clouderaManagerRoleRefreshService.refreshClusterRoles(apiClient, stack);
        LOGGER.debug("Cluster upscale completed.");
        return instanceMetaDatas.stream().map(InstanceMetaData::getDiscoveryFQDN).filter(Objects::nonNull).collect(Collectors.toList());
    } catch (ApiException e) {
        LOGGER.error(String.format("Failed to upscale. Response: %s", e.getResponseBody()), e);
        throw new CloudbreakException("Failed to upscale", e);
    }
}

Example 43 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerCommissioner method recommissionNodes.

/**
 * Attempts to recommission the provided list of hosts
 * @param stack the stack under consideration
 * @param hostsToRecommission hosts to decommission
 * @param client api client to communicate with ClouderaManager
 * @return the hostnames for hosts which were successfully recommissioned
 */
public Set<String> recommissionNodes(Stack stack, Map<String, InstanceMetaData> hostsToRecommission, ApiClient client) {
    // TODO CB-15132: Check status of target nodes / previously issued commands - in case of pod restarts etc.
    // Ideally without needing to persist anything (commandId etc)
    // TODO CB-15132: Deal with situations where CM itself is unavailable. Go back and STOP resources on the cloud-provider.
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    ApiHostList hostRefList;
    try {
        hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
    } catch (ApiException e) {
        LOGGER.error("Failed to communicate with Cloudera Manager", e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
    LOGGER.trace("Target recommissionNodes: count={}, hosts=[{}]", hostsToRecommission.size(), hostsToRecommission.keySet());
    LOGGER.debug("hostsAvailableFromCM: count={}, hosts=[{}]", hostRefList.getItems().size(), hostRefList.getItems().stream().map(ApiHost::getHostname));
    // Not considering the commission states of the nodes. Nodes could be COMMISSIONED with services in STOPPED state.
    List<String> hostsAvailableForRecommission = hostRefList.getItems().stream().filter(apiHostRef -> hostsToRecommission.containsKey(apiHostRef.getHostname())).parallel().map(ApiHost::getHostname).collect(Collectors.toList());
    Set<String> hostsAvailableForRecommissionSet = new HashSet<>(hostsAvailableForRecommission);
    List<String> cmHostsUnavailableForRecommission = hostsToRecommission.keySet().stream().filter(h -> !hostsAvailableForRecommissionSet.contains(h)).collect(Collectors.toList());
    if (cmHostsUnavailableForRecommission.size() != 0) {
        LOGGER.info("Some recommission targets are unavailable in CM: TotalRecommissionTargetCount={}, unavailableInCMCount={}, unavailableInCM=[{}]", hostsToRecommission.size(), cmHostsUnavailableForRecommission.size(), cmHostsUnavailableForRecommission);
    }
    recommissionHosts(stack, client, hostsAvailableForRecommission);
    return hostsAvailableForRecommission.stream().map(hostsToRecommission::get).filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.toSet());
}

Example 44 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method decommissionNodesStopStart.

public Set<String> decommissionNodesStopStart(Stack stack, Map<String, InstanceMetaData> hostsToRemove, ApiClient client, long pollingTimeout) {
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        LOGGER.trace("Target decommissionNodes: count={}, hosts=[{}]", hostsToRemove.size(), hostsToRemove.keySet());
        LOGGER.debug("hostsAvailableFromCM: count={}, hosts=[{}]", hostRefList.getItems().size(), hostRefList.getItems().stream().map(ApiHost::getHostname));
        List<String> stillAvailableRemovableHosts = hostRefList.getItems().stream().filter(apiHostRef -> hostsToRemove.containsKey(apiHostRef.getHostname())).parallel().map(ApiHost::getHostname).collect(Collectors.toList());
        Set<String> hostsAvailableForDecommissionSet = new HashSet<>(stillAvailableRemovableHosts);
        List<String> cmHostsUnavailableForDecommission = hostsToRemove.keySet().stream().filter(h -> !hostsAvailableForDecommissionSet.contains(h)).collect(Collectors.toList());
        if (cmHostsUnavailableForDecommission.size() != 0) {
            LOGGER.info("Some decommission targets are unavailable in CM: TotalDecommissionTargetCount={}, unavailableInCMCount={}, unavailableInCm=[{}]", hostsToRemove.size(), cmHostsUnavailableForDecommission.size(), cmHostsUnavailableForDecommission);
        }
        ClouderaManagerResourceApi apiInstance = clouderaManagerApiFactory.getClouderaManagerResourceApi(client);
        ApiHostNameList body = new ApiHostNameList().items(stillAvailableRemovableHosts);
        ApiCommand apiCommand = apiInstance.hostsDecommissionCommand(body);
        ExtendedPollingResult pollingResult = clouderaManagerPollingServiceProvider.startPollingCmHostsDecommission(stack, client, apiCommand.getId(), pollingTimeout);
        if (pollingResult.isExited()) {
            throw new CancellationException("Cluster was terminated while waiting for host decommission");
        } else if (pollingResult.isTimeout()) {
            String warningMessage = "Cloudera Manager decommission host command {} polling timed out, " + "thus we are aborting the decommission and we are retrying it for lost nodes once again.";
            abortDecommissionWithWarningMessage(apiCommand, client, warningMessage);
            throw new CloudbreakServiceException(String.format("Timeout while Cloudera Manager decommissioned host. CM command Id: %s", apiCommand.getId()));
        }
        return stillAvailableRemovableHosts.stream().map(hostsToRemove::get).filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).map(InstanceMetaData::getDiscoveryFQDN).collect(Collectors.toSet());
    } catch (ApiException e) {
        LOGGER.error("Failed to decommission hosts: {}", hostsToRemove.keySet(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}

Example 45 with ApiHost

use of com.cloudera.api.swagger.model.ApiHost in project cloudbreak by hortonworks.

the class ClouderaManagerDecomissioner method collectDownscaleCandidates.

public Set<InstanceMetaData> collectDownscaleCandidates(ApiClient client, Stack stack, HostGroup hostGroup, Integer scalingAdjustment, Set<InstanceMetaData> instanceMetaDatasInStack) {
    LOGGER.debug("Collecting downscale candidates");
    Set<InstanceMetaData> instancesForHostGroup = instanceMetaDatasInStack.stream().filter(instanceMetaData -> instanceMetaData.getInstanceGroup().getGroupName().equals(hostGroup.getName())).collect(Collectors.toSet());
    HostsResourceApi hostsResourceApi = clouderaManagerApiFactory.getHostsResourceApi(client);
    try {
        HostTemplatesResourceApi hostTemplatesResourceApi = clouderaManagerApiFactory.getHostTemplatesResourceApi(client);
        ApiHostTemplateList hostTemplates = hostTemplatesResourceApi.readHostTemplates(stack.getName());
        int replication = hostGroupNodesAreDataNodes(hostTemplates, hostGroup.getName()) ? getReplicationFactor(client, stack.getName()) : 0;
        verifyNodeCount(replication, scalingAdjustment, instancesForHostGroup.size(), 0, stack);
        ApiHostList hostRefList = hostsResourceApi.readHosts(null, null, SUMMARY_REQUEST_VIEW);
        Set<InstanceMetaData> instancesToRemove = getUnusedInstances(scalingAdjustment, instancesForHostGroup, hostRefList);
        List<ApiHost> apiHosts = hostRefList.getItems().stream().filter(host -> instancesForHostGroup.stream().filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).anyMatch(instanceMetaData -> instanceMetaData.getDiscoveryFQDN().equals(host.getHostname()))).collect(Collectors.toList());
        Set<String> hostsToRemove = apiHosts.stream().sorted(hostHealthComparator).limit(Math.abs(scalingAdjustment) - instancesToRemove.size()).map(ApiHost::getHostname).collect(Collectors.toSet());
        Set<InstanceMetaData> clouderaManagerNodesToRemove = instancesForHostGroup.stream().filter(instanceMetaData -> hostsToRemove.contains(instanceMetaData.getDiscoveryFQDN())).collect(Collectors.toSet());
        instancesToRemove.addAll(clouderaManagerNodesToRemove);
        LOGGER.debug("Downscale candidates: [{}]", instancesToRemove);
        return instancesToRemove;
    } catch (ApiException e) {
        LOGGER.error("Failed to get host list for cluster: {}", stack.getName(), e);
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}