use of org.jclouds.compute.domain.NodeMetadata in project whirr by apache.
the class HadoopService method launchCluster.
@Override
public HadoopCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
ComputeService computeService = computeServiceContext.getComputeService();
// Launch Hadoop "master" (NN and JT)
// deal with user packages and autoshutdown with extra runurls
String hadoopInstallRunUrl = clusterSpec.getConfiguration().getString("whirr.hadoop-install-runurl", "apache/hadoop/install");
byte[] nnjtBootScript = RunUrlBuilder.runUrls("sun/java/install", String.format("%s nn,jt -c %s", hadoopInstallRunUrl, clusterSpec.getProvider()));
TemplateBuilder masterTemplateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(nnjtBootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
// TODO extract this logic elsewhere
if (clusterSpec.getProvider().equals("ec2"))
masterTemplateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
Template masterTemplate = masterTemplateBuilder.build();
InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(MASTER_ROLE);
checkNotNull(instanceTemplate);
checkArgument(instanceTemplate.getNumberOfInstances() == 1);
Set<? extends NodeMetadata> nodes;
try {
nodes = computeService.runNodesWithTag(clusterSpec.getClusterName(), 1, masterTemplate);
} catch (RunNodesException e) {
// TODO: can we do better here (retry?)
throw new IOException(e);
}
NodeMetadata node = Iterables.getOnlyElement(nodes);
InetAddress namenodePublicAddress = InetAddress.getByName(Iterables.get(node.getPublicAddresses(), 0));
InetAddress jobtrackerPublicAddress = InetAddress.getByName(Iterables.get(node.getPublicAddresses(), 0));
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, WEB_PORT);
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, NAMENODE_WEB_UI_PORT);
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, JOBTRACKER_WEB_UI_PORT);
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, namenodePublicAddress.getHostAddress(), NAMENODE_PORT);
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, namenodePublicAddress.getHostAddress(), JOBTRACKER_PORT);
if (!namenodePublicAddress.equals(jobtrackerPublicAddress)) {
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, jobtrackerPublicAddress.getHostAddress(), NAMENODE_PORT);
FirewallSettings.authorizeIngress(computeServiceContext, node, clusterSpec, jobtrackerPublicAddress.getHostAddress(), JOBTRACKER_PORT);
}
// Launch slaves (DN and TT)
byte[] slaveBootScript = RunUrlBuilder.runUrls("sun/java/install", String.format("%s dn,tt -n %s -j %s", hadoopInstallRunUrl, namenodePublicAddress.getHostName(), jobtrackerPublicAddress.getHostName()));
TemplateBuilder slaveTemplateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(slaveBootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
// TODO extract this logic elsewhere
if (clusterSpec.getProvider().equals("ec2"))
slaveTemplateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
Template slaveTemplate = slaveTemplateBuilder.build();
instanceTemplate = clusterSpec.getInstanceTemplate(WORKER_ROLE);
checkNotNull(instanceTemplate);
Set<? extends NodeMetadata> workerNodes;
try {
workerNodes = computeService.runNodesWithTag(clusterSpec.getClusterName(), instanceTemplate.getNumberOfInstances(), slaveTemplate);
} catch (RunNodesException e) {
// TODO: don't bail out if only a few have failed to start
throw new IOException(e);
}
// TODO: wait for TTs to come up (done in test for the moment)
Set<Instance> instances = Sets.union(getInstances(MASTER_ROLE, Collections.singleton(node)), getInstances(WORKER_ROLE, workerNodes));
Properties config = createClientSideProperties(namenodePublicAddress, jobtrackerPublicAddress);
return new HadoopCluster(instances, config);
}
use of org.jclouds.compute.domain.NodeMetadata in project whirr by apache.
the class ZooKeeperService method launchCluster.
@Override
public ZooKeeperCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
ComputeService computeService = computeServiceContext.getComputeService();
byte[] bootScript = RunUrlBuilder.runUrls("sun/java/install", "apache/zookeeper/install");
TemplateBuilder templateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(bootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
// TODO extract this logic elsewhere
if (clusterSpec.getProvider().equals("ec2"))
templateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
Template template = templateBuilder.build();
InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(ZOOKEEPER_ROLE);
checkNotNull(instanceTemplate);
int ensembleSize = instanceTemplate.getNumberOfInstances();
Set<? extends NodeMetadata> nodeMap;
try {
nodeMap = computeService.runNodesWithTag(clusterSpec.getClusterName(), ensembleSize, template);
} catch (RunNodesException e) {
// TODO: can we do better here - proceed if ensemble is big enough?
throw new IOException(e);
}
FirewallSettings.authorizeIngress(computeServiceContext, nodeMap, clusterSpec, CLIENT_PORT);
List<NodeMetadata> nodes = Lists.newArrayList(nodeMap);
// Pass list of all servers in ensemble to configure script.
// Position is significant: i-th server has id i.
String servers = Joiner.on(' ').join(getPrivateIps(nodes));
byte[] configureScript = RunUrlBuilder.runUrls("apache/zookeeper/post-configure " + servers);
try {
computeService.runScriptOnNodesMatching(runningWithTag(clusterSpec.getClusterName()), configureScript);
} catch (RunScriptOnNodesException e) {
// TODO: retry
throw new IOException(e);
}
String hosts = Joiner.on(',').join(getHosts(nodes));
return new ZooKeeperCluster(getInstances(nodes), hosts);
}
use of org.jclouds.compute.domain.NodeMetadata in project hive by apache.
the class CloudExecutionContextProvider method terminate.
private void terminate(Set<String> hosts, boolean warnIfHostsNotFound) {
LOG.info("Requesting termination of " + hosts);
Set<NodeMetadata> nodesToTerminate = Sets.newHashSet();
for (NodeMetadata node : getRunningNodes()) {
String ip = publicIpOrHostname(node);
if (hosts.contains(ip)) {
nodesToTerminate.add(node);
}
}
terminateInternal(nodesToTerminate);
if (warnIfHostsNotFound && nodesToTerminate.size() != hosts.size()) {
LOG.error("Requested termination of " + hosts.size() + " but found only " + nodesToTerminate.size());
}
}
use of org.jclouds.compute.domain.NodeMetadata in project legacy-jclouds-examples by jclouds.
the class MainApp method main.
public static void main(String[] args) {
if (args.length < PARAMETERS)
throw new IllegalArgumentException(INVALID_SYNTAX);
// Args
String accesskeyid = args[0];
String secretkey = args[1];
String group = args[2];
String command = args[3];
// Init
ComputeService compute = new ComputeServiceContextFactory().createContext("aws-ec2", accesskeyid, secretkey).getComputeService();
// wait up to 60 seconds for ssh to be accessible
RetryablePredicate<IPSocket> socketTester = new RetryablePredicate<IPSocket>(new InetSocketAddressConnect(), 60, 1, 1, TimeUnit.SECONDS);
try {
if (command.equals("create")) {
Template template = compute.templateBuilder().build();
template.getOptions().as(AWSEC2TemplateOptions.class).spotPrice(0.03f).authorizePublicKey(Files.toString(new File(System.getProperty("user.home") + "/.ssh/id_rsa.pub"), Charsets.UTF_8));
System.out.printf(">> running one spot node type(%s) with ami(%s) in group(%s)%n", template.getHardware().getProviderId(), template.getImage().getId(), group);
// run only a single node
NodeMetadata node = Iterables.getOnlyElement(compute.createNodesInGroup(group, 1, template));
System.out.printf("<< running node(%s)%n", node.getId());
IPSocket socket = new IPSocket(Iterables.get(node.getPublicAddresses(), 0), node.getLoginPort());
if (socketTester.apply(socket)) {
System.out.printf("<< socket ready [%s] node(%s)%n", socket, node.getId());
System.out.printf("ssh to node with the following command:%n ssh %s@%s%n", node.getCredentials().identity, socket.getAddress());
System.exit(0);
} else {
System.out.printf("<< socket not ready [%s] node(%s)%n", socket, node.getId());
}
} else if (command.equals("destroy")) {
System.out.printf(">> destroying nodes in group(%s)%n", group);
Set<? extends NodeMetadata> destroyed = compute.destroyNodesMatching(NodePredicates.inGroup(group));
System.out.printf("<< destroyed(%d)%n", destroyed.size());
System.exit(0);
} else {
System.err.println(INVALID_SYNTAX);
System.exit(1);
}
} catch (RunNodesException e) {
System.err.println(e.getMessage());
for (NodeMetadata node : e.getNodeErrors().keySet()) compute.destroyNode(node.getId());
System.exit(1);
} catch (IOException e) {
System.err.println(e.getMessage());
System.exit(1);
} finally {
compute.getContext().close();
}
}
use of org.jclouds.compute.domain.NodeMetadata in project legacy-jclouds-examples by jclouds.
the class CreateVolumeAndAttach method createServer.
private NodeMetadata createServer() throws RunNodesException, TimeoutException {
Template template = compute.templateBuilder().locationId(Constants.ZONE).osDescriptionMatches(".*CentOS 6.2.*").minRam(512).build();
System.out.println("Create Server");
Set<? extends NodeMetadata> nodes = compute.createNodesInGroup(Constants.NAME, 1, template);
NodeMetadata nodeMetadata = nodes.iterator().next();
String publicAddress = nodeMetadata.getPublicAddresses().iterator().next();
// We set the password to something we know so we can login in the DetachVolume example
nova.getApi().getServerApiForZone(Constants.ZONE).changeAdminPass(nodeMetadata.getProviderId(), Constants.PASSWORD);
System.out.println(" " + nodeMetadata);
System.out.println(" Login: ssh " + nodeMetadata.getCredentials().getUser() + "@" + publicAddress);
System.out.println(" Password: " + Constants.PASSWORD);
return nodeMetadata;
}
Aggregations