Search in sources :

Example 21 with NodeMetadata

use of org.jclouds.compute.domain.NodeMetadata in project whirr by apache.

the class HadoopService method launchCluster.

@Override
public HadoopCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
    ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
    ComputeService computeService = computeServiceContext.getComputeService();
    // Launch Hadoop "master" (NN and JT)
    // deal with user packages and autoshutdown with extra runurls
    String hadoopInstallRunUrl = clusterSpec.getConfiguration().getString("whirr.hadoop-install-runurl", "apache/hadoop/install");
    byte[] nnjtBootScript = RunUrlBuilder.runUrls("sun/java/install", String.format("%s nn,jt -c %s", hadoopInstallRunUrl, clusterSpec.getProvider()));
    TemplateBuilder masterTemplateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(nnjtBootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        masterTemplateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template masterTemplate = masterTemplateBuilder.build();
    InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(MASTER_ROLE);
    checkNotNull(instanceTemplate);
    checkArgument(instanceTemplate.getNumberOfInstances() == 1);
    Set<? extends NodeMetadata> nodes;
    try {
        nodes = computeService.runNodesWithTag(clusterSpec.getClusterName(), 1, masterTemplate);
    } catch (RunNodesException e) {
        // TODO: can we do better here (retry?)
        throw new IOException(e);
    }
    NodeMetadata node = Iterables.getOnlyElement(nodes);
    InetAddress namenodePublicAddress = InetAddress.getByName(Iterables.get(node.getPublicAddresses(), 0));
    InetAddress jobtrackerPublicAddress = InetAddress.getByName(Iterables.get(node.getPublicAddresses(), 0));
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, WEB_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, NAMENODE_WEB_UI_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, JOBTRACKER_WEB_UI_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, namenodePublicAddress.getHostAddress(), NAMENODE_PORT);
    FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, namenodePublicAddress.getHostAddress(), JOBTRACKER_PORT);
    if (!namenodePublicAddress.equals(jobtrackerPublicAddress)) {
        FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, jobtrackerPublicAddress.getHostAddress(), NAMENODE_PORT);
        FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, jobtrackerPublicAddress.getHostAddress(), JOBTRACKER_PORT);
    }
    // Launch slaves (DN and TT)
    byte[] slaveBootScript = RunUrlBuilder.runUrls("sun/java/install", String.format("%s dn,tt -n %s -j %s", hadoopInstallRunUrl, namenodePublicAddress.getHostName(), jobtrackerPublicAddress.getHostName()));
    TemplateBuilder slaveTemplateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(slaveBootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        slaveTemplateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template slaveTemplate = slaveTemplateBuilder.build();
    instanceTemplate = clusterSpec.getInstanceTemplate(WORKER_ROLE);
    checkNotNull(instanceTemplate);
    Set<? extends NodeMetadata> workerNodes;
    try {
        workerNodes = computeService.runNodesWithTag(clusterSpec.getClusterName(), instanceTemplate.getNumberOfInstances(), slaveTemplate);
    } catch (RunNodesException e) {
        // TODO: don't bail out if only a few have failed to start
        throw new IOException(e);
    }
    // TODO: wait for TTs to come up (done in test for the moment)
    Set<Instance> instances = Sets.union(getInstances(MASTER_ROLE, Collections.singleton(node)), getInstances(WORKER_ROLE, workerNodes));
    Properties config = createClientSideProperties(namenodePublicAddress, jobtrackerPublicAddress);
    return new HadoopCluster(instances, config);
}
Also used : Instance(org.apache.whirr.service.Cluster.Instance) TemplateBuilder(org.jclouds.compute.domain.TemplateBuilder) ComputeServiceContext(org.jclouds.compute.ComputeServiceContext) IOException(java.io.IOException) Properties(java.util.Properties) ComputeService(org.jclouds.compute.ComputeService) Template(org.jclouds.compute.domain.Template) InstanceTemplate(org.apache.whirr.service.ClusterSpec.InstanceTemplate) NodeMetadata(org.jclouds.compute.domain.NodeMetadata) RunNodesException(org.jclouds.compute.RunNodesException) InetAddress(java.net.InetAddress) InstanceTemplate(org.apache.whirr.service.ClusterSpec.InstanceTemplate)

Example 22 with NodeMetadata

use of org.jclouds.compute.domain.NodeMetadata in project whirr by apache.

the class ZooKeeperService method launchCluster.

@Override
public ZooKeeperCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
    ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
    ComputeService computeService = computeServiceContext.getComputeService();
    byte[] bootScript = RunUrlBuilder.runUrls("sun/java/install", "apache/zookeeper/install");
    TemplateBuilder templateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(bootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
    // TODO extract this logic elsewhere
    if (clusterSpec.getProvider().equals("ec2"))
        templateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
    Template template = templateBuilder.build();
    InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(ZOOKEEPER_ROLE);
    checkNotNull(instanceTemplate);
    int ensembleSize = instanceTemplate.getNumberOfInstances();
    Set<? extends NodeMetadata> nodeMap;
    try {
        nodeMap = computeService.runNodesWithTag(clusterSpec.getClusterName(), ensembleSize, template);
    } catch (RunNodesException e) {
        // TODO: can we do better here - proceed if ensemble is big enough?
        throw new IOException(e);
    }
    FirewallSettings.authorizeIngress(computeServiceContext, nodeMap, clusterSpec, CLIENT_PORT);
    List<NodeMetadata> nodes = Lists.newArrayList(nodeMap);
    // Pass list of all servers in ensemble to configure script.
    // Position is significant: i-th server has id i.
    String servers = Joiner.on(' ').join(getPrivateIps(nodes));
    byte[] configureScript = RunUrlBuilder.runUrls("apache/zookeeper/post-configure " + servers);
    try {
        computeService.runScriptOnNodesMatching(runningWithTag(clusterSpec.getClusterName()), configureScript);
    } catch (RunScriptOnNodesException e) {
        // TODO: retry
        throw new IOException(e);
    }
    String hosts = Joiner.on(',').join(getHosts(nodes));
    return new ZooKeeperCluster(getInstances(nodes), hosts);
}
Also used : TemplateBuilder(org.jclouds.compute.domain.TemplateBuilder) ComputeServiceContext(org.jclouds.compute.ComputeServiceContext) IOException(java.io.IOException) ComputeService(org.jclouds.compute.ComputeService) Template(org.jclouds.compute.domain.Template) InstanceTemplate(org.apache.whirr.service.ClusterSpec.InstanceTemplate) NodeMetadata(org.jclouds.compute.domain.NodeMetadata) RunNodesException(org.jclouds.compute.RunNodesException) RunScriptOnNodesException(org.jclouds.compute.RunScriptOnNodesException) InstanceTemplate(org.apache.whirr.service.ClusterSpec.InstanceTemplate)

Example 23 with NodeMetadata

use of org.jclouds.compute.domain.NodeMetadata in project hive by apache.

the class CloudExecutionContextProvider method terminate.

private void terminate(Set<String> hosts, boolean warnIfHostsNotFound) {
    LOG.info("Requesting termination of " + hosts);
    Set<NodeMetadata> nodesToTerminate = Sets.newHashSet();
    for (NodeMetadata node : getRunningNodes()) {
        String ip = publicIpOrHostname(node);
        if (hosts.contains(ip)) {
            nodesToTerminate.add(node);
        }
    }
    terminateInternal(nodesToTerminate);
    if (warnIfHostsNotFound && nodesToTerminate.size() != hosts.size()) {
        LOG.error("Requested termination of " + hosts.size() + " but found only " + nodesToTerminate.size());
    }
}
Also used : NodeMetadata(org.jclouds.compute.domain.NodeMetadata)

Example 24 with NodeMetadata

use of org.jclouds.compute.domain.NodeMetadata in project legacy-jclouds-examples by jclouds.

the class MainApp method main.

public static void main(String[] args) {
    if (args.length < PARAMETERS)
        throw new IllegalArgumentException(INVALID_SYNTAX);
    // Args
    String accesskeyid = args[0];
    String secretkey = args[1];
    String group = args[2];
    String command = args[3];
    // Init
    ComputeService compute = new ComputeServiceContextFactory().createContext("aws-ec2", accesskeyid, secretkey).getComputeService();
    // wait up to 60 seconds for ssh to be accessible
    RetryablePredicate<IPSocket> socketTester = new RetryablePredicate<IPSocket>(new InetSocketAddressConnect(), 60, 1, 1, TimeUnit.SECONDS);
    try {
        if (command.equals("create")) {
            Template template = compute.templateBuilder().build();
            template.getOptions().as(AWSEC2TemplateOptions.class).spotPrice(0.03f).authorizePublicKey(Files.toString(new File(System.getProperty("user.home") + "/.ssh/id_rsa.pub"), Charsets.UTF_8));
            System.out.printf(">> running one spot node type(%s) with ami(%s) in group(%s)%n", template.getHardware().getProviderId(), template.getImage().getId(), group);
            // run only a single node
            NodeMetadata node = Iterables.getOnlyElement(compute.createNodesInGroup(group, 1, template));
            System.out.printf("<< running node(%s)%n", node.getId());
            IPSocket socket = new IPSocket(Iterables.get(node.getPublicAddresses(), 0), node.getLoginPort());
            if (socketTester.apply(socket)) {
                System.out.printf("<< socket ready [%s] node(%s)%n", socket, node.getId());
                System.out.printf("ssh to node with the following command:%n ssh %s@%s%n", node.getCredentials().identity, socket.getAddress());
                System.exit(0);
            } else {
                System.out.printf("<< socket not ready [%s] node(%s)%n", socket, node.getId());
            }
        } else if (command.equals("destroy")) {
            System.out.printf(">> destroying nodes in group(%s)%n", group);
            Set<? extends NodeMetadata> destroyed = compute.destroyNodesMatching(NodePredicates.inGroup(group));
            System.out.printf("<< destroyed(%d)%n", destroyed.size());
            System.exit(0);
        } else {
            System.err.println(INVALID_SYNTAX);
            System.exit(1);
        }
    } catch (RunNodesException e) {
        System.err.println(e.getMessage());
        for (NodeMetadata node : e.getNodeErrors().keySet()) compute.destroyNode(node.getId());
        System.exit(1);
    } catch (IOException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    } finally {
        compute.getContext().close();
    }
}
Also used : RetryablePredicate(org.jclouds.predicates.RetryablePredicate) Set(java.util.Set) InetSocketAddressConnect(org.jclouds.predicates.InetSocketAddressConnect) IOException(java.io.IOException) ComputeService(org.jclouds.compute.ComputeService) ComputeServiceContextFactory(org.jclouds.compute.ComputeServiceContextFactory) IPSocket(org.jclouds.net.IPSocket) Template(org.jclouds.compute.domain.Template) NodeMetadata(org.jclouds.compute.domain.NodeMetadata) RunNodesException(org.jclouds.compute.RunNodesException) File(java.io.File)

Example 25 with NodeMetadata

use of org.jclouds.compute.domain.NodeMetadata in project legacy-jclouds-examples by jclouds.

the class CreateVolumeAndAttach method createServer.

private NodeMetadata createServer() throws RunNodesException, TimeoutException {
    Template template = compute.templateBuilder().locationId(Constants.ZONE).osDescriptionMatches(".*CentOS 6.2.*").minRam(512).build();
    System.out.println("Create Server");
    Set<? extends NodeMetadata> nodes = compute.createNodesInGroup(Constants.NAME, 1, template);
    NodeMetadata nodeMetadata = nodes.iterator().next();
    String publicAddress = nodeMetadata.getPublicAddresses().iterator().next();
    // We set the password to something we know so we can login in the DetachVolume example
    nova.getApi().getServerApiForZone(Constants.ZONE).changeAdminPass(nodeMetadata.getProviderId(), Constants.PASSWORD);
    System.out.println("  " + nodeMetadata);
    System.out.println("  Login: ssh " + nodeMetadata.getCredentials().getUser() + "@" + publicAddress);
    System.out.println("  Password: " + Constants.PASSWORD);
    return nodeMetadata;
}
Also used : NodeMetadata(org.jclouds.compute.domain.NodeMetadata) Template(org.jclouds.compute.domain.Template)

Aggregations

NodeMetadata (org.jclouds.compute.domain.NodeMetadata)35 RunNodesException (org.jclouds.compute.RunNodesException)11 Template (org.jclouds.compute.domain.Template)9 ComputeService (org.jclouds.compute.ComputeService)7 IOException (java.io.IOException)6 TemplateBuilder (org.jclouds.compute.domain.TemplateBuilder)6 Test (org.junit.Test)5 Exchange (org.apache.camel.Exchange)4 ComputeServiceContext (org.jclouds.compute.ComputeServiceContext)4 RunScriptOnNodesException (org.jclouds.compute.RunScriptOnNodesException)4 Statement (org.jclouds.scriptbuilder.domain.Statement)4 File (java.io.File)3 Set (java.util.Set)3 InstanceTemplate (org.apache.whirr.service.ClusterSpec.InstanceTemplate)3 ComputeMetadata (org.jclouds.compute.domain.ComputeMetadata)3 LoginCredentials (org.jclouds.domain.LoginCredentials)3 Predicate (com.google.common.base.Predicate)2 Properties (java.util.Properties)2 Host (org.apache.hive.ptest.execution.conf.Host)2 RunListBuilder (org.jclouds.chef.util.RunListBuilder)2