Examples with InstanceTemplate org.apache.whirr.InstanceTemplate used on opensource projects

Example 1 with InstanceTemplate

use of org.apache.whirr.InstanceTemplate in project whirr by apache.

the class BootstrapTemplateTest method assertSpotPriceIs.

@SuppressWarnings("unchecked")
private void assertSpotPriceIs(ClusterSpec clusterSpec, final String templateGroup, float spotPrice) throws MalformedURLException {
    InstanceTemplate instanceTemplate = getOnlyElement(filter(clusterSpec.getInstanceTemplates(), new Predicate<InstanceTemplate>() {

        private Joiner plusJoiner = Joiner.on("+");

        @Override
        public boolean apply(InstanceTemplate group) {
            return plusJoiner.join(group.getRoles()).equals(templateGroup);
        }
    }));
    ComputeService computeService = mock(AWSEC2ComputeService.class);
    ComputeServiceContext context = mock(ComputeServiceContext.class);
    when(computeService.getContext()).thenReturn(context);
    when(context.getComputeService()).thenReturn(computeService);
    TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
    when(computeService.templateBuilder()).thenReturn(templateBuilder);
    when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
    when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
    Template template = mock(Template.class);
    TemplateOptions options = mock(TemplateOptions.class);
    Image image = mock(Image.class);
    when(templateBuilder.build()).thenReturn(template);
    when(template.getOptions()).thenReturn(options);
    when(template.getImage()).thenReturn(image);
    AWSEC2TemplateOptions awsEec2TemplateOptions = mock(AWSEC2TemplateOptions.class);
    when(options.as((Class<TemplateOptions>) any())).thenReturn(awsEec2TemplateOptions);
    BootstrapTemplate.build(clusterSpec, computeService, statementBuilder, instanceTemplate);
    verify(awsEec2TemplateOptions).spotPrice(spotPrice);
}

Example 2 with InstanceTemplate

use of org.apache.whirr.InstanceTemplate in project whirr by apache.

the class BootstrapClusterAction method doAction.

@Override
protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap) throws IOException, InterruptedException {
    LOG.info("Bootstrapping cluster");
    ExecutorService executorService = Executors.newCachedThreadPool();
    Map<InstanceTemplate, Future<Set<? extends NodeMetadata>>> futures = Maps.newHashMap();
    // initialize startup processes per InstanceTemplates
    for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
        final InstanceTemplate instanceTemplate = entry.getKey();
        final ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
        final int maxNumberOfRetries = clusterSpec.getMaxStartupRetries();
        StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
        ComputeServiceContext computeServiceContext = getCompute().apply(clusterSpec);
        final ComputeService computeService = computeServiceContext.getComputeService();
        final Template template = BootstrapTemplate.build(clusterSpec, computeService, statementBuilder, entry.getKey());
        Future<Set<? extends NodeMetadata>> nodesFuture = executorService.submit(new StartupProcess(clusterSpec.getClusterName(), instanceTemplate.getNumberOfInstances(), instanceTemplate.getMinNumberOfInstances(), maxNumberOfRetries, instanceTemplate.getRoles(), computeService, template, executorService, nodeStarterFactory));
        futures.put(instanceTemplate, nodesFuture);
    }
    Set<Instance> instances = Sets.newLinkedHashSet();
    for (Entry<InstanceTemplate, Future<Set<? extends NodeMetadata>>> entry : futures.entrySet()) {
        Set<? extends NodeMetadata> nodes;
        try {
            nodes = entry.getValue().get();
        } catch (ExecutionException e) {
            // nodes after retries
            throw new IOException(e);
        }
        Set<String> roles = entry.getKey().getRoles();
        instances.addAll(getInstances(roles, nodes));
    }
    Cluster cluster = new Cluster(instances);
    for (ClusterActionEvent event : eventMap.values()) {
        event.setCluster(cluster);
    }
}

Example 3 with InstanceTemplate

use of org.apache.whirr.InstanceTemplate in project whirr by apache.

the class ScriptBasedClusterAction method execute.

public Cluster execute(ClusterSpec clusterSpec, Cluster cluster) throws IOException, InterruptedException {
    if (clusterSpec.getInstanceTemplates().size() == 0) {
        throw new IllegalArgumentException("No instance templates specified.");
    }
    Map<InstanceTemplate, ClusterActionEvent> eventMap = Maps.newHashMap();
    Cluster newCluster = cluster;
    for (InstanceTemplate instanceTemplate : clusterSpec.getInstanceTemplates()) {
        if (shouldIgnoreInstanceTemplate(instanceTemplate)) {
            // skip execution if this group of instances is not in target
            continue;
        }
        StatementBuilder statementBuilder = new StatementBuilder();
        ComputeServiceContext computeServiceContext = getCompute().apply(clusterSpec);
        FirewallManager firewallManager = new FirewallManager(computeServiceContext, clusterSpec, newCluster);
        VelocityEngine velocityEngine = TemplateUtils.newVelocityEngine();
        ClusterActionEvent event = new ClusterActionEvent(getAction(), clusterSpec, instanceTemplate, newCluster, statementBuilder, getCompute(), firewallManager, velocityEngine);
        eventMap.put(instanceTemplate, event);
        eventSpecificActions(instanceTemplate, event);
        for (String role : instanceTemplate.getRoles()) {
            if (roleIsInTarget(role)) {
                safeGetActionHandler(role).beforeAction(event);
            }
        }
        // cluster may have been updated by handler
        newCluster = event.getCluster();
    }
    doAction(eventMap);
    // cluster may have been updated by action
    newCluster = Iterables.get(eventMap.values(), 0).getCluster();
    for (InstanceTemplate instanceTemplate : clusterSpec.getInstanceTemplates()) {
        if (shouldIgnoreInstanceTemplate(instanceTemplate)) {
            continue;
        }
        ClusterActionEvent event = eventMap.get(instanceTemplate);
        for (String role : instanceTemplate.getRoles()) {
            if (roleIsInTarget(role)) {
                event.setCluster(newCluster);
                safeGetActionHandler(role).afterAction(event);
                // cluster may have been updated by handler
                newCluster = event.getCluster();
            }
        }
    }
    return newCluster;
}

Example 4 with InstanceTemplate

use of org.apache.whirr.InstanceTemplate in project whirr by apache.

the class ByonClusterAction method doAction.

@Override
protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap) throws IOException, InterruptedException {
    final Collection<Future<ExecResponse>> futures = Sets.newHashSet();
    List<NodeMetadata> nodes = Lists.newArrayList();
    List<NodeMetadata> usedNodes = Lists.newArrayList();
    int numberAllocated = 0;
    Set<Instance> allInstances = Sets.newLinkedHashSet();
    for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
        final ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
        final StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
        if (statementBuilder.isEmpty()) {
            // skip
            continue;
        }
        final ComputeServiceContext computeServiceContext = getCompute().apply(clusterSpec);
        final ComputeService computeService = computeServiceContext.getComputeService();
        LoginCredentials credentials = LoginCredentials.builder().user(clusterSpec.getClusterUser()).privateKey(clusterSpec.getPrivateKey()).build();
        final RunScriptOptions options = overrideLoginCredentials(credentials);
        if (numberAllocated == 0) {
            for (ComputeMetadata compute : computeService.listNodes()) {
                if (!(compute instanceof NodeMetadata)) {
                    throw new IllegalArgumentException("Not an instance of NodeMetadata: " + compute);
                }
                nodes.add((NodeMetadata) compute);
            }
        }
        int num = entry.getKey().getNumberOfInstances();
        Predicate<NodeMetadata> unused = not(in(usedNodes));
        // TODO: This seems very fragile and a bug.  It is not required that someone passes a hardware id,
        // so this is likely to break badly. Even if there was, why do we assume it is splittable?!
        // this logic should be refactored or removed ASAP
        Predicate<NodeMetadata> instancePredicate = Predicates.alwaysTrue();
        if (entry.getKey().getTemplate() != null) {
            String hardwareId = entry.getKey().getTemplate().getHardwareId();
            if (hardwareId != null)
                instancePredicate = new TagsPredicate(StringUtils.split(hardwareId));
        }
        List<NodeMetadata> templateNodes = Lists.newArrayList(filter(nodes, and(unused, instancePredicate)));
        if (templateNodes.size() < num) {
            LOG.warn("Not enough nodes available for template " + StringUtils.join(entry.getKey().getRoles(), "+"));
        }
        templateNodes = templateNodes.subList(0, num);
        usedNodes.addAll(templateNodes);
        numberAllocated = usedNodes.size();
        Set<Instance> templateInstances = getInstances(credentials, entry.getKey().getRoles(), templateNodes);
        allInstances.addAll(templateInstances);
        for (final Instance instance : templateInstances) {
            futures.add(runStatementOnInstanceInCluster(statementBuilder, instance, clusterSpec, options));
        }
    }
    for (Future<ExecResponse> future : futures) {
        try {
            future.get();
        } catch (ExecutionException e) {
            throw new IOException(e.getCause());
        }
    }
    if (action.equals(ClusterActionHandler.BOOTSTRAP_ACTION)) {
        Cluster cluster = new Cluster(allInstances);
        for (ClusterActionEvent event : eventMap.values()) {
            event.setCluster(cluster);
        }
    }
}

Example 5 with InstanceTemplate

use of org.apache.whirr.InstanceTemplate in project whirr by apache.

the class ScriptBasedClusterAction method runScripts.

protected void runScripts(Map<InstanceTemplate, ClusterActionEvent> eventMap) throws InterruptedException, IOException {
    final String phaseName = getAction();
    final Collection<Future<ExecResponse>> futures = Sets.newHashSet();
    final ClusterSpec clusterSpec = eventMap.values().iterator().next().getClusterSpec();
    final RunScriptOptions options = overrideLoginCredentials(LoginCredentials.builder().user(clusterSpec.getClusterUser()).privateKey(clusterSpec.getPrivateKey()).build());
    for (Map.Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
        if (shouldIgnoreInstanceTemplate(entry.getKey())) {
            // skip if not in the target
            continue;
        }
        Cluster cluster = entry.getValue().getCluster();
        StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
        if (statementBuilder.isEmpty()) {
            // skip execution if we have an empty list
            continue;
        }
        Set<Instance> instances = cluster.getInstancesMatching(Predicates.<Instance>and(onlyRolesIn(entry.getKey().getRoles()), not(instanceIsNotInTarget())));
        LOG.info("Starting to run scripts on cluster for phase {} " + "on instances: {}", phaseName, asString(instances));
        for (Instance instance : instances) {
            futures.add(runStatementOnInstanceInCluster(statementBuilder, instance, clusterSpec, options));
        }
    }
    for (Future<ExecResponse> future : futures) {
        try {
            future.get();
        } catch (ExecutionException e) {
            throw new IOException(e.getCause());
        }
    }
    LOG.info("Finished running {} phase scripts on all cluster instances", phaseName);
}