Search in sources :

Example 1 with ContainerConstraint

use of com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint in project cloudbreak by hortonworks.

the class ContainerConstraintFactory method getAmbariAgentConstraint.

public ContainerConstraint getAmbariAgentConstraint(String ambariServerHost, String ambariAgentApp, String cloudPlatform, HostGroup hostGroup, Integer adjustment, List<String> hostBlackList, String identifier) {
    String containerInstanceName;
    containerInstanceName = YARN.equals(hostGroup.getCluster().getStack().getOrchestrator().getType()) ? createContainerInstanceNameForYarn(hostGroup, AMBARI_AGENT.getName(), identifier) : createContainerInstanceName(hostGroup, AMBARI_AGENT.getName(), identifier);
    Constraint hgConstraint = hostGroup.getConstraint();
    Builder builder = new Builder().withNamePrefix(containerInstanceName).withAppName(ambariAgentApp).networkMode(HOST_NETWORK_MODE);
    if (hgConstraint.getInstanceGroup() != null) {
        InstanceGroup instanceGroup = hgConstraint.getInstanceGroup();
        Map<String, String> dataVolumeBinds = new HashMap<>();
        dataVolumeBinds.put("/var/log/ambari-agent-container", CONTAINER_VOLUME_PATH);
        dataVolumeBinds.put(HADOOP_MOUNT_DIR, HADOOP_MOUNT_DIR);
        dataVolumeBinds.putAll(ImmutableMap.of("/data/jars", "/data/jars", HOST_VOLUME_PATH, CONTAINER_VOLUME_PATH));
        builder.addVolumeBindings(dataVolumeBinds);
        if (adjustment != null) {
            List<String> candidates = collectUpscaleCandidates(hostGroup.getCluster().getId(), hostGroup.getName(), adjustment);
            builder.addHosts(getHosts(candidates, instanceGroup));
        } else {
            builder.addHosts(getHosts(null, instanceGroup));
        }
        builder.cmd(new String[] { String.format("/usr/sbin/init systemd.setenv=AMBARI_SERVER_ADDR=%s systemd.setenv=CLOUD_PLATFORM=%s", ambariServerHost, cloudPlatform) });
    }
    if (hgConstraint.getConstraintTemplate() != null) {
        builder.cpus(hgConstraint.getConstraintTemplate().getCpu());
        builder.memory(hgConstraint.getConstraintTemplate().getMemory());
        builder.constraints(getConstraints(hostBlackList));
        if (adjustment != null) {
            builder.instances(adjustment);
        } else {
            builder.instances(hgConstraint.getHostCount());
        }
        builder.withDiskSize(hgConstraint.getConstraintTemplate().getDisk());
        Map<String, String> dataVolumeBinds = new HashMap<>();
        dataVolumeBinds.put("/var/log/ambari-agent-container", CONTAINER_VOLUME_PATH);
        builder.addVolumeBindings(dataVolumeBinds);
        builder.cmd(new String[] { String.format("/usr/sbin/init systemd.setenv=AMBARI_SERVER_ADDR=%s systemd.setenv=USE_CONSUL_DNS=false", ambariServerHost) });
    }
    return builder.build();
}
Also used : ContainerConstraint(com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint) Constraint(com.sequenceiq.cloudbreak.domain.Constraint) HashMap(java.util.HashMap) Builder(com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint.Builder) InstanceGroup(com.sequenceiq.cloudbreak.domain.InstanceGroup)

Example 2 with ContainerConstraint

use of com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint in project cloudbreak by hortonworks.

the class YarnContainerOrchestrator method runContainer.

@Override
public List<ContainerInfo> runContainer(ContainerConfig config, OrchestrationCredential cred, ContainerConstraint constraint, ExitCriteriaModel exitCriteriaModel) throws CloudbreakOrchestratorException {
    // Create an application per component
    List<ContainerInfo> containerInfos = new ArrayList<>();
    for (int componentNumber = 1; componentNumber <= constraint.getInstances(); componentNumber++) {
        try {
            submitHandler.submitApplication(config, cred, constraint, componentNumber);
            String applicationName = applicationUtils.getApplicationName(constraint, componentNumber);
            OrchestratorBootstrap bootstrap = new YarnAppBootstrap(applicationName, cred.getApiEndpoint());
            Callable<Boolean> runner = runner(bootstrap, getExitCriteria(), exitCriteriaModel);
            Future<Boolean> appFuture = getParallelOrchestratorComponentRunner().submit(runner);
            appFuture.get();
            containerInfos.add(detailHandler.getContainerInfo(config, cred, constraint, componentNumber));
        } catch (CloudbreakOrchestratorException | InterruptedException | ExecutionException e) {
            throw new CloudbreakOrchestratorFailedException(e);
        }
    }
    return containerInfos;
}
Also used : OrchestratorBootstrap(com.sequenceiq.cloudbreak.orchestrator.OrchestratorBootstrap) ArrayList(java.util.ArrayList) ContainerConstraint(com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) CloudbreakOrchestratorFailedException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorFailedException) ContainerInfo(com.sequenceiq.cloudbreak.orchestrator.model.ContainerInfo) ExecutionException(java.util.concurrent.ExecutionException) YarnAppBootstrap(com.sequenceiq.cloudbreak.orchestrator.yarn.poller.YarnAppBootstrap)

Example 3 with ContainerConstraint

use of com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint in project cloudbreak by hortonworks.

the class ClusterContainerRunner method initializeClusterContainers.

private Map<String, List<Container>> initializeClusterContainers(Stack stack, String cloudPlatform) throws CloudbreakException, CloudbreakOrchestratorException {
    Orchestrator orchestrator = stack.getOrchestrator();
    Map<String, Object> map = new HashMap<>(orchestrator.getAttributes().getMap());
    OrchestrationCredential credential = new OrchestrationCredential(orchestrator.getApiEndpoint(), map);
    ContainerOrchestrator containerOrchestrator = containerOrchestratorResolver.get(orchestrator.getType());
    Map<String, List<ContainerInfo>> containers = new HashMap<>();
    Cluster cluster = clusterService.retrieveClusterByStackId(stack.getId());
    String gatewayHostname = getGatewayHostName(stack);
    try {
        ContainerConstraint ambariServerDbConstraint = constraintFactory.getAmbariServerDbConstraint(gatewayHostname, cluster.getName(), cluster.getId().toString());
        List<ContainerInfo> dbContainer = containerOrchestrator.runContainer(containerConfigService.get(stack, AMBARI_DB), credential, ambariServerDbConstraint, clusterDeletionBasedModel(stack.getId(), cluster.getId()));
        containers.put(AMBARI_DB.name(), dbContainer);
        String serverDbHostName = dbContainer.get(0).getHost();
        ContainerConstraint ambariServerConstraint = constraintFactory.getAmbariServerConstraint(serverDbHostName, gatewayHostname, cloudPlatform, cluster.getName(), cluster.getId().toString());
        List<ContainerInfo> ambariServerContainer = containerOrchestrator.runContainer(containerConfigService.get(stack, AMBARI_SERVER), credential, ambariServerConstraint, clusterDeletionBasedModel(stack.getId(), cluster.getId()));
        containers.put(AMBARI_SERVER.name(), ambariServerContainer);
        String ambariServerHost = ambariServerContainer.get(0).getHost();
        List<String> hostBlackList = new ArrayList<>();
        for (HostGroup hostGroup : hostGroupRepository.findHostGroupsInCluster(stack.getCluster().getId())) {
            ContainerConstraint ambariAgentConstraint = constraintFactory.getAmbariAgentConstraint(ambariServerHost, null, cloudPlatform, hostGroup, null, hostBlackList, cluster.getId().toString());
            List<ContainerInfo> containerInfos = containerOrchestrator.runContainer(containerConfigService.get(stack, AMBARI_AGENT), credential, ambariAgentConstraint, clusterDeletionBasedModel(stack.getId(), cluster.getId()));
            containers.put(hostGroup.getName(), containerInfos);
            hostBlackList.addAll(getHostsFromContainerInfo(containerInfos));
        }
        return saveContainers(containers, cluster);
    } catch (CloudbreakOrchestratorException ex) {
        if (!containers.isEmpty()) {
            saveContainers(containers, cluster);
        }
        checkCancellation(ex);
        throw ex;
    }
}
Also used : ContainerConstraint(com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) OrchestrationCredential(com.sequenceiq.cloudbreak.orchestrator.model.OrchestrationCredential) Cluster(com.sequenceiq.cloudbreak.domain.Cluster) HostGroup(com.sequenceiq.cloudbreak.domain.HostGroup) Orchestrator(com.sequenceiq.cloudbreak.domain.Orchestrator) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) ContainerInfo(com.sequenceiq.cloudbreak.orchestrator.model.ContainerInfo) ArrayList(java.util.ArrayList) List(java.util.List)

Example 4 with ContainerConstraint

use of com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint in project cloudbreak by hortonworks.

the class ClusterContainerRunner method addClusterContainers.

private Map<String, List<Container>> addClusterContainers(Stack stack, String cloudPlatform, String hostGroupName, Integer adjustment) throws CloudbreakException, CloudbreakOrchestratorException {
    Orchestrator orchestrator = stack.getOrchestrator();
    Map<String, Object> map = new HashMap<>(orchestrator.getAttributes().getMap());
    OrchestrationCredential credential = new OrchestrationCredential(orchestrator.getApiEndpoint(), map);
    ContainerOrchestrator containerOrchestrator = containerOrchestratorResolver.get(orchestrator.getType());
    Map<String, List<ContainerInfo>> containers = new HashMap<>();
    Cluster cluster = clusterService.retrieveClusterByStackId(stack.getId());
    try {
        Set<Container> existingContainers = containerService.findContainersInCluster(cluster.getId());
        String ambariServerHost = existingContainers.stream().filter(input -> input.getImage().contains(AMBARI_SERVER.getName())).findFirst().get().getHost();
        HostGroup hostGroup = hostGroupRepository.findHostGroupInClusterByName(cluster.getId(), hostGroupName);
        String ambariAgentApp = existingContainers.stream().filter(input -> hostGroup.getHostNames().contains(input.getHost()) && input.getImage().contains(AMBARI_AGENT.getName())).findFirst().get().getName();
        List<String> hostBlackList = getOtherHostgroupsAgentHostsFromContainer(existingContainers, hostGroupName);
        ContainerConstraint ambariAgentConstraint = constraintFactory.getAmbariAgentConstraint(ambariServerHost, ambariAgentApp, cloudPlatform, hostGroup, adjustment, hostBlackList, cluster.getId().toString());
        containers.put(hostGroup.getName(), containerOrchestrator.runContainer(containerConfigService.get(stack, AMBARI_AGENT), credential, ambariAgentConstraint, clusterDeletionBasedModel(stack.getId(), cluster.getId())));
        return saveContainers(containers, cluster);
    } catch (CloudbreakOrchestratorException ex) {
        if (!containers.isEmpty()) {
            saveContainers(containers, cluster);
        }
        checkCancellation(ex);
        throw ex;
    }
}
Also used : CloudbreakException(com.sequenceiq.cloudbreak.service.CloudbreakException) ContainerConfigService(com.sequenceiq.cloudbreak.core.bootstrap.service.ContainerConfigService) StackRepository(com.sequenceiq.cloudbreak.repository.StackRepository) CloudbreakOrchestratorCancelledException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorCancelledException) HashMap(java.util.HashMap) ClusterService(com.sequenceiq.cloudbreak.service.cluster.ClusterService) HostGroupRepository(com.sequenceiq.cloudbreak.repository.HostGroupRepository) StringUtils(org.apache.commons.lang3.StringUtils) Orchestrator(com.sequenceiq.cloudbreak.domain.Orchestrator) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) AMBARI_SERVER(com.sequenceiq.cloudbreak.orchestrator.container.DockerContainer.AMBARI_SERVER) ContainerService(com.sequenceiq.cloudbreak.service.cluster.ContainerService) OrchestrationCredential(com.sequenceiq.cloudbreak.orchestrator.model.OrchestrationCredential) Map(java.util.Map) ContainerConstraint(com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint) Stack(com.sequenceiq.cloudbreak.domain.Stack) ConversionService(org.springframework.core.convert.ConversionService) ContainerInfo(com.sequenceiq.cloudbreak.orchestrator.model.ContainerInfo) AMBARI_AGENT(com.sequenceiq.cloudbreak.orchestrator.container.DockerContainer.AMBARI_AGENT) HostGroup(com.sequenceiq.cloudbreak.domain.HostGroup) ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel(com.sequenceiq.cloudbreak.core.bootstrap.service.ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel) AMBARI_DB(com.sequenceiq.cloudbreak.orchestrator.container.DockerContainer.AMBARI_DB) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) InstanceMetaData(com.sequenceiq.cloudbreak.domain.InstanceMetaData) Collection(java.util.Collection) Set(java.util.Set) Cluster(com.sequenceiq.cloudbreak.domain.Cluster) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) Collectors(java.util.stream.Collectors) Container(com.sequenceiq.cloudbreak.domain.Container) List(java.util.List) Component(org.springframework.stereotype.Component) Entry(java.util.Map.Entry) CancellationException(com.sequenceiq.cloudbreak.cloud.scheduler.CancellationException) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) ContainerConstraint(com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint) HashMap(java.util.HashMap) OrchestrationCredential(com.sequenceiq.cloudbreak.orchestrator.model.OrchestrationCredential) Cluster(com.sequenceiq.cloudbreak.domain.Cluster) HostGroup(com.sequenceiq.cloudbreak.domain.HostGroup) Orchestrator(com.sequenceiq.cloudbreak.domain.Orchestrator) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) Container(com.sequenceiq.cloudbreak.domain.Container) CloudbreakOrchestratorException(com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException) ContainerOrchestrator(com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator) ArrayList(java.util.ArrayList) List(java.util.List)

Aggregations

ContainerConstraint (com.sequenceiq.cloudbreak.orchestrator.model.ContainerConstraint)4 CloudbreakOrchestratorException (com.sequenceiq.cloudbreak.orchestrator.exception.CloudbreakOrchestratorException)3 ContainerInfo (com.sequenceiq.cloudbreak.orchestrator.model.ContainerInfo)3 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 Cluster (com.sequenceiq.cloudbreak.domain.Cluster)2 HostGroup (com.sequenceiq.cloudbreak.domain.HostGroup)2 Orchestrator (com.sequenceiq.cloudbreak.domain.Orchestrator)2 ContainerOrchestrator (com.sequenceiq.cloudbreak.orchestrator.container.ContainerOrchestrator)2 OrchestrationCredential (com.sequenceiq.cloudbreak.orchestrator.model.OrchestrationCredential)2 List (java.util.List)2 CancellationException (com.sequenceiq.cloudbreak.cloud.scheduler.CancellationException)1 ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel (com.sequenceiq.cloudbreak.core.bootstrap.service.ClusterDeletionBasedExitCriteriaModel.clusterDeletionBasedModel)1 ContainerConfigService (com.sequenceiq.cloudbreak.core.bootstrap.service.ContainerConfigService)1 Constraint (com.sequenceiq.cloudbreak.domain.Constraint)1 Container (com.sequenceiq.cloudbreak.domain.Container)1 InstanceGroup (com.sequenceiq.cloudbreak.domain.InstanceGroup)1 InstanceMetaData (com.sequenceiq.cloudbreak.domain.InstanceMetaData)1 Stack (com.sequenceiq.cloudbreak.domain.Stack)1 OrchestratorBootstrap (com.sequenceiq.cloudbreak.orchestrator.OrchestratorBootstrap)1