Search in sources :

Example 11 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterHostServiceRunner method collectUpscaleCandidates.

private Map<String, String> collectUpscaleCandidates(Long clusterId, Map<String, Integer> hostGroupWithAdjustment) {
    Map<String, String> hostNames = new HashMap<>();
    for (Map.Entry<String, Integer> entry : hostGroupWithAdjustment.entrySet()) {
        String hostGroupName = entry.getKey();
        Integer adjustment = entry.getValue();
        HostGroup hostGroup = hostGroupService.findHostGroupInClusterByName(clusterId, hostGroupName).orElseThrow(NotFoundException.notFound("hostgroup", hostGroupName));
        if (hostGroup.getInstanceGroup() != null) {
            Long instanceGroupId = hostGroup.getInstanceGroup().getId();
            instanceMetaDataService.findUnusedHostsInInstanceGroup(instanceGroupId).stream().filter(instanceMetaData -> instanceMetaData.getDiscoveryFQDN() != null).sorted(Comparator.comparing(InstanceMetaData::getStartDate)).limit(adjustment.longValue()).forEach(im -> hostNames.put(im.getDiscoveryFQDN(), im.getPrivateIp()));
        }
    }
    return hostNames;
}
Also used : CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) DatalakeService(com.sequenceiq.cloudbreak.service.sharedservice.DatalakeService) ComponentConfigProviderService(com.sequenceiq.cloudbreak.service.ComponentConfigProviderService) StringUtils(org.apache.commons.lang3.StringUtils) ThreadBasedUserCrnProvider(com.sequenceiq.cloudbreak.auth.ThreadBasedUserCrnProvider) Map(java.util.Map) Gateway(com.sequenceiq.cloudbreak.domain.stack.cluster.gateway.Gateway) StringUtils.isEmpty(org.apache.commons.lang3.StringUtils.isEmpty) HostAttributeDecorator(com.sequenceiq.cloudbreak.core.bootstrap.service.host.decorator.HostAttributeDecorator) RecipeEngine(com.sequenceiq.cloudbreak.service.cluster.flow.recipe.RecipeEngine) CLOUDERAMANAGER_VERSION_7_2_1(com.sequenceiq.cloudbreak.cmtemplate.CMRepositoryVersionUtil.CLOUDERAMANAGER_VERSION_7_2_1) CLOUDERAMANAGER_VERSION_7_2_0(com.sequenceiq.cloudbreak.cmtemplate.CMRepositoryVersionUtil.CLOUDERAMANAGER_VERSION_7_2_0) Set(java.util.Set) Crn(com.sequenceiq.cloudbreak.auth.crn.Crn) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) ClouderaManagerRepo(com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo) Serializable(java.io.Serializable) TargetedUpscaleSupportService(com.sequenceiq.cloudbreak.service.stack.TargetedUpscaleSupportService) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) RdsConfigWithoutClusterService(com.sequenceiq.cloudbreak.service.rdsconfig.RdsConfigWithoutClusterService) StackUtil(com.sequenceiq.cloudbreak.util.StackUtil) MDCUtils(com.sequenceiq.cloudbreak.logger.MDCUtils) CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) CollectionUtils(org.springframework.util.CollectionUtils) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig) ExposedServices(com.sequenceiq.cloudbreak.domain.stack.cluster.gateway.ExposedServices) CancellationException(com.sequenceiq.cloudbreak.cloud.scheduler.CancellationException) InstanceGroup(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceGroup) StackService(com.sequenceiq.cloudbreak.service.stack.StackService) Joiner(com.google.common.base.Joiner) Stack(com.sequenceiq.cloudbreak.domain.stack.Stack) CMLicenseParser(com.sequenceiq.cloudbreak.auth.CMLicenseParser) OrchestratorGrainRunnerParams(com.sequenceiq.cloudbreak.orchestrator.host.OrchestratorGrainRunnerParams) HostGroupService(com.sequenceiq.cloudbreak.service.hostgroup.HostGroupService) StringUtils.isNotEmpty(org.apache.commons.lang3.StringUtils.isNotEmpty) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) CsdParcelDecorator(com.sequenceiq.cloudbreak.core.bootstrap.service.host.decorator.CsdParcelDecorator) KerberosDetailService(com.sequenceiq.cloudbreak.template.kerberos.KerberosDetailService) Lists(com.google.common.collect.Lists) KerberosType(com.sequenceiq.cloudbreak.type.KerberosType) LdapView(com.sequenceiq.cloudbreak.dto.LdapView) CMRepositoryVersionUtil.isVersionNewerOrEqualThanLimited(com.sequenceiq.cloudbreak.cmtemplate.CMRepositoryVersionUtil.isVersionNewerOrEqualThanLimited) ClusterComponentConfigProvider(com.sequenceiq.cloudbreak.cluster.service.ClusterComponentConfigProvider) NullUtil.throwIfNull(com.sequenceiq.cloudbreak.util.NullUtil.throwIfNull) DatabaseType(com.sequenceiq.cloudbreak.api.endpoint.v4.database.base.DatabaseType) DnsResolverType(com.sequenceiq.cloudbreak.domain.stack.DnsResolverType) ExitCriteriaModel(com.sequenceiq.cloudbreak.orchestrator.state.ExitCriteriaModel) IOException(java.io.IOException) PostgresConfigService(com.sequenceiq.cloudbreak.core.bootstrap.service.container.postgres.PostgresConfigService) ServiceLocationMap(com.sequenceiq.cloudbreak.cluster.model.ServiceLocationMap) ClusterDeletionBasedExitCriteriaModel(com.sequenceiq.cloudbreak.core.bootstrap.service.ClusterDeletionBasedExitCriteriaModel) GrpcUmsClient(com.sequenceiq.cloudbreak.auth.altus.GrpcUmsClient) HostGroup(com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup) GrainProperties(com.sequenceiq.cloudbreak.orchestrator.model.GrainProperties) InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) GatewayTopology(com.sequenceiq.cloudbreak.domain.stack.cluster.gateway.GatewayTopology) ComponentLocatorService(com.sequenceiq.cloudbreak.service.blueprint.ComponentLocatorService) InetAddresses(com.google.common.net.InetAddresses) NodesUnreachableException(com.sequenceiq.cloudbreak.util.NodesUnreachableException) SaltConfig(com.sequenceiq.cloudbreak.orchestrator.model.SaltConfig) GrainOperation(com.sequenceiq.cloudbreak.orchestrator.host.GrainOperation) EntitlementService(com.sequenceiq.cloudbreak.auth.altus.EntitlementService) CloudbreakOrchestratorCancelledException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorCancelledException) LoggerFactory(org.slf4j.LoggerFactory) LoadBalancerSANProvider(com.sequenceiq.cloudbreak.san.LoadBalancerSANProvider) InstanceGroupService(com.sequenceiq.cloudbreak.service.stack.InstanceGroupService) ClusterPreCreationApi(com.sequenceiq.cloudbreak.cluster.api.ClusterPreCreationApi) KerberosConfigService(com.sequenceiq.cloudbreak.kerberos.KerberosConfigService) ExposedServiceCollector(com.sequenceiq.cloudbreak.api.service.ExposedServiceCollector) Telemetry(com.sequenceiq.common.api.telemetry.model.Telemetry) NotFoundException(com.sequenceiq.cloudbreak.common.exception.NotFoundException) HostOrchestrator(com.sequenceiq.cloudbreak.orchestrator.host.HostOrchestrator) CLOUDERAMANAGER_VERSION_7_0_2(com.sequenceiq.cloudbreak.cmtemplate.CMRepositoryVersionUtil.CLOUDERAMANAGER_VERSION_7_0_2) SaltPillarProperties(com.sequenceiq.cloudbreak.orchestrator.model.SaltPillarProperties) DataBusCredential(com.sequenceiq.common.api.telemetry.model.DataBusCredential) StackTags(com.sequenceiq.cloudbreak.cloud.model.StackTags) ExposedService(com.sequenceiq.cloudbreak.api.service.ExposedService) Collectors(java.util.stream.Collectors) TemporaryStorage(com.sequenceiq.cloudbreak.common.type.TemporaryStorage) RegionAwareInternalCrnGeneratorFactory(com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGeneratorFactory) Json(com.sequenceiq.cloudbreak.common.json.Json) IdBrokerService(com.sequenceiq.cloudbreak.service.idbroker.IdBrokerService) GatewayConfigService(com.sequenceiq.cloudbreak.service.GatewayConfigService) ProxyConfigProvider(com.sequenceiq.cloudbreak.service.proxy.ProxyConfigProvider) List(java.util.List) VirtualGroupService(com.sequenceiq.cloudbreak.auth.altus.VirtualGroupService) RdsView(com.sequenceiq.cloudbreak.template.views.RdsView) Optional(java.util.Optional) SSOType(com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.base.SSOType) Pattern(java.util.regex.Pattern) KerberosConfig(com.sequenceiq.cloudbreak.dto.KerberosConfig) Account(com.cloudera.thunderhead.service.usermanagement.UserManagementProto.Account) Cluster(com.sequenceiq.cloudbreak.domain.stack.cluster.Cluster) LoadBalancerConfigService(com.sequenceiq.cloudbreak.service.LoadBalancerConfigService) CMRepositoryVersionUtil(com.sequenceiq.cloudbreak.cmtemplate.CMRepositoryVersionUtil) IdBroker(com.sequenceiq.cloudbreak.domain.stack.cluster.IdBroker) LdapConfigService(com.sequenceiq.cloudbreak.ldap.LdapConfigService) HashMap(java.util.HashMap) FreeIpaConfigProvider(com.sequenceiq.cloudbreak.service.freeipa.FreeIpaConfigProvider) HashSet(java.util.HashSet) Inject(javax.inject.Inject) InstanceMetaDataService(com.sequenceiq.cloudbreak.service.stack.InstanceMetaDataService) Collections.singletonMap(java.util.Collections.singletonMap) RdsConfigWithoutCluster(com.sequenceiq.cloudbreak.domain.view.RdsConfigWithoutCluster) Nonnull(javax.annotation.Nonnull) TelemetryDecorator(com.sequenceiq.cloudbreak.core.bootstrap.service.host.decorator.TelemetryDecorator) ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel(com.sequenceiq.cloudbreak.core.bootstrap.service.ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel) Logger(org.slf4j.Logger) VirtualGroupRequest(com.sequenceiq.cloudbreak.auth.altus.VirtualGroupRequest) NodeReachabilityResult(com.sequenceiq.cloudbreak.orchestrator.model.NodeReachabilityResult) EnvironmentConfigProvider(com.sequenceiq.cloudbreak.service.environment.EnvironmentConfigProvider) Node(com.sequenceiq.cloudbreak.common.orchestration.Node) MountDisks(com.sequenceiq.cloudbreak.service.stack.flow.MountDisks) Component(org.springframework.stereotype.Component) UmsVirtualGroupRight(com.sequenceiq.cloudbreak.auth.altus.UmsVirtualGroupRight) ClusterApiConnectors(com.sequenceiq.cloudbreak.service.cluster.ClusterApiConnectors) CloudbreakServiceException(com.sequenceiq.cloudbreak.common.exception.CloudbreakServiceException) Comparator(java.util.Comparator) Collections(java.util.Collections) HashMap(java.util.HashMap) HostGroup(com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup) Map(java.util.Map) ServiceLocationMap(com.sequenceiq.cloudbreak.cluster.model.ServiceLocationMap) HashMap(java.util.HashMap) Collections.singletonMap(java.util.Collections.singletonMap)

Example 12 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData in project cloudbreak by hortonworks.

the class GrainPropertiesService method addCloudIdentityRolesForHosts.

private GrainProperties addCloudIdentityRolesForHosts(Cluster cluster, Set<Node> nodes) {
    Set<InstanceMetaData> instanceMetaDataSet = instanceMetaDataService.getAllInstanceMetadataByStackId(cluster.getStack().getId());
    GrainProperties propertiesForIdentityRoles = new GrainProperties();
    Set<String> hostNames = nodes.stream().map(Node::getHostname).collect(toSet());
    for (InstanceMetaData instanceMetaData : instanceMetaDataSet) {
        if (hostNames.contains(instanceMetaData.getDiscoveryFQDN())) {
            setCloudIdentityRoles(propertiesForIdentityRoles, instanceMetaData);
        }
    }
    return propertiesForIdentityRoles.getProperties().isEmpty() ? null : propertiesForIdentityRoles;
}
Also used : InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) GrainProperties(com.sequenceiq.cloudbreak.orchestrator.model.GrainProperties)

Example 13 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData in project cloudbreak by hortonworks.

the class AmbariClusterResetService method resetCluster.

public void resetCluster(Long stackId) throws CloudbreakOrchestratorException {
    Stack stack = stackService.getByIdWithListsInTransaction(stackId);
    InstanceMetaData gatewayInstance = stack.getPrimaryGatewayInstance();
    GatewayConfig gatewayConfig = gatewayConfigService.getGatewayConfig(stack, gatewayInstance, stack.getCluster().hasGateway());
    Set<String> gatewayFQDN = Collections.singleton(gatewayInstance.getDiscoveryFQDN());
    ExitCriteriaModel exitCriteriaModel = clusterDeletionBasedModel(stack.getId(), stack.getCluster().getId());
    hostOrchestrator.resetClusterManager(gatewayConfig, gatewayFQDN, stackUtil.collectNodes(stack), exitCriteriaModel);
}
Also used : InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) ExitCriteriaModel(com.sequenceiq.cloudbreak.orchestrator.state.ExitCriteriaModel) Stack(com.sequenceiq.cloudbreak.domain.stack.Stack) GatewayConfig(com.sequenceiq.cloudbreak.orchestrator.model.GatewayConfig)

Example 14 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterBuilderService method loadInstanceMetadataForHostGroups.

private Map<HostGroup, List<InstanceMetaData>> loadInstanceMetadataForHostGroups(Iterable<HostGroup> hostGroups) {
    Map<HostGroup, List<InstanceMetaData>> instanceMetaDataByHostGroup = new HashMap<>();
    for (HostGroup hostGroup : hostGroups) {
        Long instanceGroupId = hostGroup.getInstanceGroup().getId();
        List<InstanceMetaData> metas = instanceMetaDataService.findAliveInstancesInInstanceGroup(instanceGroupId);
        instanceMetaDataByHostGroup.put(hostGroup, metas);
    }
    return instanceMetaDataByHostGroup;
}
Also used : InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) HashMap(java.util.HashMap) HostGroup(com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup) List(java.util.List)

Example 15 with InstanceMetaData

use of com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData in project cloudbreak by hortonworks.

the class ClusterRepairFlowEventChainFactory method downscaleEvent.

private StackEvent downscaleEvent(boolean singlePrimaryGW, ClusterRepairTriggerEvent event, Map<String, Set<String>> groupsWithHostNames) {
    Set<InstanceMetaData> instanceMetaData = instanceMetaDataService.getAllInstanceMetadataWithoutInstanceGroupByStackId(event.getStackId());
    Map<String, Set<Long>> groupsWithPrivateIds = new HashMap<>();
    Map<String, Integer> groupsWithAdjustment = new HashMap<>();
    for (Entry<String, Set<String>> groupWithHostNames : groupsWithHostNames.entrySet()) {
        Set<String> hostNames = groupWithHostNames.getValue();
        String group = groupWithHostNames.getKey();
        Set<Long> privateIdsForHostNames = stackService.getPrivateIdsForHostNames(instanceMetaData, hostNames);
        groupsWithPrivateIds.put(group, privateIdsForHostNames);
        int size = hostNames != null ? hostNames.size() : 0;
        groupsWithAdjustment.put(group, size);
    }
    LOGGER.info("Downscale groups with adjustments: {}", groupsWithAdjustment);
    LOGGER.info("Downscale groups with privateIds: {}", groupsWithPrivateIds);
    if (!singlePrimaryGW) {
        LOGGER.info("Full downscale for the following: {}", groupsWithHostNames);
        return new ClusterAndStackDownscaleTriggerEvent(FlowChainTriggers.FULL_DOWNSCALE_TRIGGER_EVENT, event.getResourceId(), groupsWithAdjustment, groupsWithPrivateIds, groupsWithHostNames, ScalingType.DOWNSCALE_TOGETHER, event.accepted(), new ClusterDownscaleDetails(true, true));
    } else {
        LOGGER.info("Stack downscale for the following: {}", groupsWithHostNames);
        return new StackDownscaleTriggerEvent(STACK_DOWNSCALE_EVENT.event(), event.getResourceId(), groupsWithAdjustment, groupsWithPrivateIds, groupsWithHostNames, event.getTriggeredStackVariant(), event.accepted()).setRepair();
    }
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) ClusterAndStackDownscaleTriggerEvent(com.sequenceiq.cloudbreak.core.flow2.event.ClusterAndStackDownscaleTriggerEvent) HashMap(java.util.HashMap) InstanceMetaData(com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData) ClusterAndStackDownscaleTriggerEvent(com.sequenceiq.cloudbreak.core.flow2.event.ClusterAndStackDownscaleTriggerEvent) StackDownscaleTriggerEvent(com.sequenceiq.cloudbreak.core.flow2.event.StackDownscaleTriggerEvent) ClusterDownscaleDetails(com.sequenceiq.cloudbreak.core.flow2.event.ClusterDownscaleDetails)

Aggregations

InstanceMetaData (com.sequenceiq.cloudbreak.domain.stack.instance.InstanceMetaData)415 Stack (com.sequenceiq.cloudbreak.domain.stack.Stack)165 InstanceGroup (com.sequenceiq.cloudbreak.domain.stack.instance.InstanceGroup)152 Test (org.junit.jupiter.api.Test)143 Map (java.util.Map)92 HashSet (java.util.HashSet)90 Set (java.util.Set)86 List (java.util.List)84 HostGroup (com.sequenceiq.cloudbreak.domain.stack.cluster.host.HostGroup)77 Cluster (com.sequenceiq.cloudbreak.domain.stack.cluster.Cluster)73 Collectors (java.util.stream.Collectors)71 ArrayList (java.util.ArrayList)62 Test (org.junit.Test)60 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)57 Optional (java.util.Optional)52 StackService (com.sequenceiq.cloudbreak.service.stack.StackService)48 Inject (javax.inject.Inject)47 Logger (org.slf4j.Logger)47 LoggerFactory (org.slf4j.LoggerFactory)47 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)45