Examples with InstanceTemplate org.apache.whirr.service.ClusterSpec.InstanceTemplate used on opensource projects

Example 1 with InstanceTemplate

use of org.apache.whirr.service.ClusterSpec.InstanceTemplate in project whirr by apache.

the class CassandraService method launchCluster.

@Override
public Cluster launchCluster(ClusterSpec clusterSpec) throws IOException {
    ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
    ComputeService computeService = computeServiceContext.getComputeService();
    byte[] bootScript = RunUrlBuilder.runUrls("sun/java/install", "apache/cassandra/install");
    TemplateBuilder templateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(bootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        templateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template template = templateBuilder.build();
    InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(CASSANDRA_ROLE);
    checkNotNull(instanceTemplate);
    int clusterSize = instanceTemplate.getNumberOfInstances();
    Set<? extends NodeMetadata> nodeMap;
    try {
        nodeMap = computeService.runNodesWithTag(clusterSpec.getClusterName(), clusterSize, template);
    } catch (RunNodesException e) {
        // TODO: can we do better here
        throw new IOException(e);
    }
    FirewallSettings.authorizeIngress(computeServiceContext, nodeMap, clusterSpec, CLIENT_PORT);
    List<NodeMetadata> nodes = Lists.newArrayList(nodeMap);
    List<NodeMetadata> seeds = getSeeds(nodes);
    // Pass list of all servers in cluster to configure script.
    String servers = Joiner.on(' ').join(getPrivateIps(seeds));
    byte[] configureScript = RunUrlBuilder.runUrls("apache/cassandra/post-configure " + servers);
    try {
        Map<? extends NodeMetadata, ExecResponse> responses = computeService.runScriptOnNodesMatching(runningWithTag(clusterSpec.getClusterName()), configureScript);
        assert responses.size() > 0 : "no nodes matched " + clusterSpec.getClusterName();
    } catch (RunScriptOnNodesException e) {
        // TODO: retry
        throw new IOException(e);
    }
    return new Cluster(getInstances(nodes));
}

Example 2 with InstanceTemplate

use of org.apache.whirr.service.ClusterSpec.InstanceTemplate in project whirr by apache.

the class HadoopService method launchCluster.

@Override
public HadoopCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
    ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
    ComputeService computeService = computeServiceContext.getComputeService();
    // Launch Hadoop "master" (NN and JT)
    // deal with user packages and autoshutdown with extra runurls
    String hadoopInstallRunUrl = clusterSpec.getConfiguration().getString("whirr.hadoop-install-runurl", "apache/hadoop/install");
    byte[] nnjtBootScript = RunUrlBuilder.runUrls("sun/java/install", String.format("%s nn,jt -c %s", hadoopInstallRunUrl, clusterSpec.getProvider()));
    TemplateBuilder masterTemplateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(nnjtBootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        masterTemplateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template masterTemplate = masterTemplateBuilder.build();
    InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(MASTER_ROLE);
    checkNotNull(instanceTemplate);
    checkArgument(instanceTemplate.getNumberOfInstances() == 1);
    Set<? extends NodeMetadata> nodes;
    try {
        nodes = computeService.runNodesWithTag(clusterSpec.getClusterName(), 1, masterTemplate);
    } catch (RunNodesException e) {
        // TODO: can we do better here (retry?)
        throw new IOException(e);
    }
    NodeMetadata node = Iterables.getOnlyElement(nodes);
    InetAddress namenodePublicAddress = InetAddress.getByName(Iterables.get(node.getPublicAddresses(), 0));
    InetAddress jobtrackerPublicAddress = InetAddress.getByName(Iterables.get(node.getPublicAddresses(), 0));
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, WEB_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, NAMENODE_WEB_UI_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, JOBTRACKER_WEB_UI_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, namenodePublicAddress.getHostAddress(), NAMENODE_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, namenodePublicAddress.getHostAddress(), JOBTRACKER_PORT);
    if (!namenodePublicAddress.equals(jobtrackerPublicAddress)) {
        FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, jobtrackerPublicAddress.getHostAddress(), NAMENODE_PORT);
        FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, jobtrackerPublicAddress.getHostAddress(), JOBTRACKER_PORT);
    }
    // Launch slaves (DN and TT)
    byte[] slaveBootScript = RunUrlBuilder.runUrls("sun/java/install", String.format("%s dn,tt -n %s -j %s", hadoopInstallRunUrl, namenodePublicAddress.getHostName(), jobtrackerPublicAddress.getHostName()));
    TemplateBuilder slaveTemplateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(slaveBootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        slaveTemplateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template slaveTemplate = slaveTemplateBuilder.build();
    instanceTemplate = clusterSpec.getInstanceTemplate(WORKER_ROLE);
    checkNotNull(instanceTemplate);
    Set<? extends NodeMetadata> workerNodes;
    try {
        workerNodes = computeService.runNodesWithTag(clusterSpec.getClusterName(), instanceTemplate.getNumberOfInstances(), slaveTemplate);
    } catch (RunNodesException e) {
        // TODO: don't bail out if only a few have failed to start
        throw new IOException(e);
    }
    // TODO: wait for TTs to come up (done in test for the moment)
    Set<Instance> instances = Sets.union(getInstances(MASTER_ROLE, Collections.singleton(node)), getInstances(WORKER_ROLE, workerNodes));
    Properties config = createClientSideProperties(namenodePublicAddress, jobtrackerPublicAddress);
    return new HadoopCluster(instances, config);
}

Example 3 with InstanceTemplate

use of org.apache.whirr.service.ClusterSpec.InstanceTemplate in project whirr by apache.

the class ZooKeeperService method launchCluster.

@Override
public ZooKeeperCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
    ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
    ComputeService computeService = computeServiceContext.getComputeService();
    byte[] bootScript = RunUrlBuilder.runUrls("sun/java/install", "apache/zookeeper/install");
    TemplateBuilder templateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(bootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        templateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template template = templateBuilder.build();
    InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(ZOOKEEPER_ROLE);
    checkNotNull(instanceTemplate);
    int ensembleSize = instanceTemplate.getNumberOfInstances();
    Set<? extends NodeMetadata> nodeMap;
    try {
        nodeMap = computeService.runNodesWithTag(clusterSpec.getClusterName(), ensembleSize, template);
    } catch (RunNodesException e) {
        // TODO: can we do better here - proceed if ensemble is big enough?
        throw new IOException(e);
    }
    FirewallSettings.authorizeIngress(computeServiceContext, nodeMap, clusterSpec, CLIENT_PORT);
    List<NodeMetadata> nodes = Lists.newArrayList(nodeMap);
    // Pass list of all servers in ensemble to configure script.
    // Position is significant: i-th server has id i.
    String servers = Joiner.on(' ').join(getPrivateIps(nodes));
    byte[] configureScript = RunUrlBuilder.runUrls("apache/zookeeper/post-configure " + servers);
    try {
        computeService.runScriptOnNodesMatching(runningWithTag(clusterSpec.getClusterName()), configureScript);
    } catch (RunScriptOnNodesException e) {
        // TODO: retry
        throw new IOException(e);
    }
    String hosts = Joiner.on(',').join(getHosts(nodes));
    return new ZooKeeperCluster(getInstances(nodes), hosts);
}