Examples with Cluster org.apache.whirr.Cluster used on opensource projects

Example 16 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class BootstrapClusterAction method doAction.

@Override
protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap) throws IOException, InterruptedException {
    LOG.info("Bootstrapping cluster");
    ExecutorService executorService = Executors.newCachedThreadPool();
    Map<InstanceTemplate, Future<Set<? extends NodeMetadata>>> futures = Maps.newHashMap();
    // initialize startup processes per InstanceTemplates
    for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
        final InstanceTemplate instanceTemplate = entry.getKey();
        final ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
        final int maxNumberOfRetries = clusterSpec.getMaxStartupRetries();
        StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
        ComputeServiceContext computeServiceContext = getCompute().apply(clusterSpec);
        final ComputeService computeService = computeServiceContext.getComputeService();
        final Template template = BootstrapTemplate.build(clusterSpec, computeService, statementBuilder, entry.getKey());
        Future<Set<? extends NodeMetadata>> nodesFuture = executorService.submit(new StartupProcess(clusterSpec.getClusterName(), instanceTemplate.getNumberOfInstances(), instanceTemplate.getMinNumberOfInstances(), maxNumberOfRetries, instanceTemplate.getRoles(), computeService, template, executorService, nodeStarterFactory));
        futures.put(instanceTemplate, nodesFuture);
    }
    Set<Instance> instances = Sets.newLinkedHashSet();
    for (Entry<InstanceTemplate, Future<Set<? extends NodeMetadata>>> entry : futures.entrySet()) {
        Set<? extends NodeMetadata> nodes;
        try {
            nodes = entry.getValue().get();
        } catch (ExecutionException e) {
            // nodes after retries
            throw new IOException(e);
        }
        Set<String> roles = entry.getKey().getRoles();
        instances.addAll(getInstances(roles, nodes));
    }
    Cluster cluster = new Cluster(instances);
    for (ClusterActionEvent event : eventMap.values()) {
        event.setCluster(cluster);
    }
}

Example 17 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class ScriptBasedClusterAction method execute.

public Cluster execute(ClusterSpec clusterSpec, Cluster cluster) throws IOException, InterruptedException {
    if (clusterSpec.getInstanceTemplates().size() == 0) {
        throw new IllegalArgumentException("No instance templates specified.");
    }
    Map<InstanceTemplate, ClusterActionEvent> eventMap = Maps.newHashMap();
    Cluster newCluster = cluster;
    for (InstanceTemplate instanceTemplate : clusterSpec.getInstanceTemplates()) {
        if (shouldIgnoreInstanceTemplate(instanceTemplate)) {
            // skip execution if this group of instances is not in target
            continue;
        }
        StatementBuilder statementBuilder = new StatementBuilder();
        ComputeServiceContext computeServiceContext = getCompute().apply(clusterSpec);
        FirewallManager firewallManager = new FirewallManager(computeServiceContext, clusterSpec, newCluster);
        VelocityEngine velocityEngine = TemplateUtils.newVelocityEngine();
        ClusterActionEvent event = new ClusterActionEvent(getAction(), clusterSpec, instanceTemplate, newCluster, statementBuilder, getCompute(), firewallManager, velocityEngine);
        eventMap.put(instanceTemplate, event);
        eventSpecificActions(instanceTemplate, event);
        for (String role : instanceTemplate.getRoles()) {
            if (roleIsInTarget(role)) {
                safeGetActionHandler(role).beforeAction(event);
            }
        }
        // cluster may have been updated by handler
        newCluster = event.getCluster();
    }
    doAction(eventMap);
    // cluster may have been updated by action
    newCluster = Iterables.get(eventMap.values(), 0).getCluster();
    for (InstanceTemplate instanceTemplate : clusterSpec.getInstanceTemplates()) {
        if (shouldIgnoreInstanceTemplate(instanceTemplate)) {
            continue;
        }
        ClusterActionEvent event = eventMap.get(instanceTemplate);
        for (String role : instanceTemplate.getRoles()) {
            if (roleIsInTarget(role)) {
                event.setCluster(newCluster);
                safeGetActionHandler(role).afterAction(event);
                // cluster may have been updated by handler
                newCluster = event.getCluster();
            }
        }
    }
    return newCluster;
}

Example 18 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class ByonClusterAction method doAction.

@Override
protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap) throws IOException, InterruptedException {
    final Collection<Future<ExecResponse>> futures = Sets.newHashSet();
    List<NodeMetadata> nodes = Lists.newArrayList();
    List<NodeMetadata> usedNodes = Lists.newArrayList();
    int numberAllocated = 0;
    Set<Instance> allInstances = Sets.newLinkedHashSet();
    for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
        final ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
        final StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
        if (statementBuilder.isEmpty()) {
            // skip
            continue;
        }
        final ComputeServiceContext computeServiceContext = getCompute().apply(clusterSpec);
        final ComputeService computeService = computeServiceContext.getComputeService();
        LoginCredentials credentials = LoginCredentials.builder().user(clusterSpec.getClusterUser()).privateKey(clusterSpec.getPrivateKey()).build();
        final RunScriptOptions options = overrideLoginCredentials(credentials);
        if (numberAllocated == 0) {
            for (ComputeMetadata compute : computeService.listNodes()) {
                if (!(compute instanceof NodeMetadata)) {
                    throw new IllegalArgumentException("Not an instance of NodeMetadata: " + compute);
                }
                nodes.add((NodeMetadata) compute);
            }
        }
        int num = entry.getKey().getNumberOfInstances();
        Predicate<NodeMetadata> unused = not(in(usedNodes));
        // TODO: This seems very fragile and a bug.  It is not required that someone passes a hardware id,
        // so this is likely to break badly. Even if there was, why do we assume it is splittable?!
        // this logic should be refactored or removed ASAP
        Predicate<NodeMetadata> instancePredicate = Predicates.alwaysTrue();
        if (entry.getKey().getTemplate() != null) {
            String hardwareId = entry.getKey().getTemplate().getHardwareId();
            if (hardwareId != null)
                instancePredicate = new TagsPredicate(StringUtils.split(hardwareId));
        }
        List<NodeMetadata> templateNodes = Lists.newArrayList(filter(nodes, and(unused, instancePredicate)));
        if (templateNodes.size() < num) {
            LOG.warn("Not enough nodes available for template " + StringUtils.join(entry.getKey().getRoles(), "+"));
        }
        templateNodes = templateNodes.subList(0, num);
        usedNodes.addAll(templateNodes);
        numberAllocated = usedNodes.size();
        Set<Instance> templateInstances = getInstances(credentials, entry.getKey().getRoles(), templateNodes);
        allInstances.addAll(templateInstances);
        for (final Instance instance : templateInstances) {
            futures.add(runStatementOnInstanceInCluster(statementBuilder, instance, clusterSpec, options));
        }
    }
    for (Future<ExecResponse> future : futures) {
        try {
            future.get();
        } catch (ExecutionException e) {
            throw new IOException(e.getCause());
        }
    }
    if (action.equals(ClusterActionHandler.BOOTSTRAP_ACTION)) {
        Cluster cluster = new Cluster(allInstances);
        for (ClusterActionEvent event : eventMap.values()) {
            event.setCluster(cluster);
        }
    }
}

Example 19 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class YarnNodeManagerHandler method beforeConfigure.

@Override
protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException {
    ClusterSpec clusterSpec = event.getClusterSpec();
    Configuration conf = getConfiguration(clusterSpec);
    Cluster cluster = event.getCluster();
    Set<Instance> nodeManagers = cluster.getInstancesMatching(role(ROLE));
    if (!nodeManagers.isEmpty()) {
        for (Instance nodeManager : nodeManagers) {
            event.getFirewallManager().addRules(Rule.create().destination(nodeManager).ports(NODE_MANAGER_WEB_UI_PORT));
        }
    }
    handleFirewallRules(event);
    try {
        event.getStatementBuilder().addStatements(build("/tmp/yarn-site.xml", clusterSpec, cluster, ROLE));
    } catch (ConfigurationException e) {
        throw new IOException(e);
    }
    addStatement(event, call(getConfigureFunction(conf)));
    addStatement(event, call(getStartFunction(conf), "nodemanager"));
}

Example 20 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class YarnResourceManagerHandler method afterConfigure.

@Override
protected void afterConfigure(ClusterActionEvent event) throws IOException {
    ClusterSpec clusterSpec = event.getClusterSpec();
    Cluster cluster = event.getCluster();
    Instance resourceManager = cluster.getInstanceMatching(role(ROLE));
    LOG.info("Resource manager web UI available at http://{}:{}", resourceManager.getPublicHostName(), RESOURCE_MANAGER_WEB_UI_PORT);
    Properties mrConfig = createClientSideMapReduceProperties(clusterSpec);
    createClientSideMapReduceSiteFile(clusterSpec, mrConfig);
    Properties yarnConfig = createClientSideYarnProperties(clusterSpec, resourceManager);
    createClientSideYarnSiteFile(clusterSpec, yarnConfig);
    Properties combined = new Properties();
    combined.putAll(cluster.getConfiguration());
    combined.putAll(mrConfig);
    combined.putAll(yarnConfig);
    event.setCluster(new Cluster(cluster.getInstances(), combined));
}