use of org.jclouds.compute.domain.TemplateBuilder in project whirr by apache.
the class CassandraService method launchCluster.
@Override
public Cluster launchCluster(ClusterSpec clusterSpec) throws IOException {
ComputeServiceContext computeServiceContext = ComputeServiceContextBuilder.build(clusterSpec);
ComputeService computeService = computeServiceContext.getComputeService();
byte[] bootScript = RunUrlBuilder.runUrls("sun/java/install", "apache/cassandra/install");
TemplateBuilder templateBuilder = computeService.templateBuilder().osFamily(UBUNTU).options(runScript(bootScript).installPrivateKey(clusterSpec.readPrivateKey()).authorizePublicKey(clusterSpec.readPublicKey()));
// TODO extract this logic elsewhere
if (clusterSpec.getProvider().equals("ec2"))
templateBuilder.imageNameMatches(".*10\\.?04.*").osDescriptionMatches("^ubuntu-images.*").architecture(Architecture.X86_32);
Template template = templateBuilder.build();
InstanceTemplate instanceTemplate = clusterSpec.getInstanceTemplate(CASSANDRA_ROLE);
checkNotNull(instanceTemplate);
int clusterSize = instanceTemplate.getNumberOfInstances();
Set<? extends NodeMetadata> nodeMap;
try {
nodeMap = computeService.runNodesWithTag(clusterSpec.getClusterName(), clusterSize, template);
} catch (RunNodesException e) {
// TODO: can we do better here
throw new IOException(e);
}
FirewallSettings.authorizeIngress(computeServiceContext, nodeMap, clusterSpec, CLIENT_PORT);
List<NodeMetadata> nodes = Lists.newArrayList(nodeMap);
List<NodeMetadata> seeds = getSeeds(nodes);
// Pass list of all servers in cluster to configure script.
String servers = Joiner.on(' ').join(getPrivateIps(seeds));
byte[] configureScript = RunUrlBuilder.runUrls("apache/cassandra/post-configure " + servers);
try {
Map<? extends NodeMetadata, ExecResponse> responses = computeService.runScriptOnNodesMatching(runningWithTag(clusterSpec.getClusterName()), configureScript);
assert responses.size() > 0 : "no nodes matched " + clusterSpec.getClusterName();
} catch (RunScriptOnNodesException e) {
// TODO: retry
throw new IOException(e);
}
return new Cluster(getInstances(nodes));
}
use of org.jclouds.compute.domain.TemplateBuilder in project iobserve-analysis by research-iobserve.
the class AcquireActionScript method execute.
@Override
public void execute() throws RunNodesException {
final ResourceContainer container = this.action.getSourceResourceContainer();
final ResourceContainerCloud cloudContainer = this.getResourceContainerCloud(container);
final ComputeService client = this.getComputeServiceForContainer(cloudContainer);
final TemplateBuilder templateBuilder = client.templateBuilder();
final VMType instanceType = cloudContainer.getInstanceType();
templateBuilder.hardwareId(instanceType.getName());
templateBuilder.locationId(instanceType.getLocation());
// TODO maybe make this configurable
templateBuilder.osFamily(OsFamily.UBUNTU);
final Statement setupAdminInstructions = AdminAccess.standard();
// Necessary to set hostname to allow mapping for later events
final TemplateOptions options = Builder.runScript(setupAdminInstructions).runScript(AcquireActionScript.getChangeHostnameScript(cloudContainer)).runScript(this.getStartupScript());
templateBuilder.options(options);
final Template template = templateBuilder.build();
final String groupName = ModelHelper.getGroupName(cloudContainer);
final NodeMetadata node = Iterables.getOnlyElement(client.createNodesInGroup(groupName, 1, template));
AcquireActionScript.LOGGER.info(String.format("Acquired node for resource container '%s'. NodeID: %s, Hostname: %s, Adresses: %s", cloudContainer.getEntityName(), node.getId(), node.getHostname(), Iterables.concat(node.getPrivateAddresses(), node.getPublicAddresses())));
// TODO write resource container to original model to enable mapping
}
use of org.jclouds.compute.domain.TemplateBuilder in project whirr by apache.
the class BootstrapTemplateTest method assertSpotPriceIs.
@SuppressWarnings("unchecked")
private void assertSpotPriceIs(ClusterSpec clusterSpec, final String templateGroup, float spotPrice) throws MalformedURLException {
InstanceTemplate instanceTemplate = getOnlyElement(filter(clusterSpec.getInstanceTemplates(), new Predicate<InstanceTemplate>() {
private Joiner plusJoiner = Joiner.on("+");
@Override
public boolean apply(InstanceTemplate group) {
return plusJoiner.join(group.getRoles()).equals(templateGroup);
}
}));
ComputeService computeService = mock(AWSEC2ComputeService.class);
ComputeServiceContext context = mock(ComputeServiceContext.class);
when(computeService.getContext()).thenReturn(context);
when(context.getComputeService()).thenReturn(computeService);
TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
when(computeService.templateBuilder()).thenReturn(templateBuilder);
when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
Template template = mock(Template.class);
TemplateOptions options = mock(TemplateOptions.class);
Image image = mock(Image.class);
when(templateBuilder.build()).thenReturn(template);
when(template.getOptions()).thenReturn(options);
when(template.getImage()).thenReturn(image);
AWSEC2TemplateOptions awsEec2TemplateOptions = mock(AWSEC2TemplateOptions.class);
when(options.as((Class<TemplateOptions>) any())).thenReturn(awsEec2TemplateOptions);
BootstrapTemplate.build(clusterSpec, computeService, statementBuilder, instanceTemplate);
verify(awsEec2TemplateOptions).spotPrice(spotPrice);
}
use of org.jclouds.compute.domain.TemplateBuilder in project whirr by apache.
the class BootstrapClusterActionTest method testDoActionRetriesSucceed.
@SuppressWarnings("unchecked")
@Test
public void testDoActionRetriesSucceed() throws Exception {
CompositeConfiguration config = new CompositeConfiguration();
if (System.getProperty("config") != null) {
config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
}
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.service-name", "test-service");
conf.addProperty("whirr.cluster-name", "test-cluster");
conf.addProperty("whirr.instance-templates", "1 hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.instance-templates-max-percent-failures", "60 hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.provider", "ec2");
config.addConfiguration(conf);
ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(conf);
Set<String> jtnn = new HashSet<String>();
jtnn.add("hadoop-jobtracker");
jtnn.add("hadoop-namenode");
Set<String> dntt = new HashSet<String>();
dntt.add("hadoop-datanode");
dntt.add("hadoop-tasktracker");
TestNodeStarterFactory nodeStarterFactory = null;
ClusterActionHandler handler = mock(ClusterActionHandler.class);
LoadingCache<String, ClusterActionHandler> handlerMap = convertMapToLoadingCache(ImmutableMap.<String, ClusterActionHandler>builder().put("hadoop-jobtracker", handler).put("hadoop-namenode", handler).put("hadoop-datanode", handler).put("hadoop-tasktracker", handler).build());
Function<ClusterSpec, ComputeServiceContext> getCompute = mock(Function.class);
ComputeServiceContext serviceContext = mock(ComputeServiceContext.class);
ComputeService computeService = mock(ComputeService.class);
TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
Template template = mock(Template.class);
TemplateOptions templateOptions = mock(TemplateOptions.class);
when(getCompute.apply(clusterSpec)).thenReturn(serviceContext);
when(serviceContext.getComputeService()).thenReturn(computeService);
when(computeService.getContext()).thenReturn(serviceContext);
when(serviceContext.getBackendType()).thenReturn(TypeToken.class.cast(TypeToken.of(Context.class)));
when(computeService.templateBuilder()).thenReturn(templateBuilder);
when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
when(templateBuilder.build()).thenReturn(template);
when(template.getOptions()).thenReturn(templateOptions);
// here is a scenario when jt+nn fails once, then the retry is successful
// and from the dn+tt one node fails, then the retry is successful
Map<Set<String>, Stack<Integer>> reaction = Maps.newHashMap();
Stack<Integer> jtnnStack = new Stack<Integer>();
// then ok
jtnnStack.push(1);
// initially fail
jtnnStack.push(0);
reaction.put(jtnn, jtnnStack);
Stack<Integer> ddttStack = new Stack<Integer>();
// 3 from 4, just enough
ddttStack.push(3);
reaction.put(dntt, ddttStack);
nodeStarterFactory = new TestNodeStarterFactory(reaction);
BootstrapClusterAction bootstrapper = new BootstrapClusterAction(getCompute, handlerMap, nodeStarterFactory);
bootstrapper.execute(clusterSpec, null);
if (nodeStarterFactory != null) {
nodeStarterFactory.validateCompletion();
}
}
use of org.jclouds.compute.domain.TemplateBuilder in project whirr by apache.
the class BootstrapClusterActionTest method testDoActionRetriesExceeds.
@SuppressWarnings("unchecked")
@Test(expected = IOException.class)
public void testDoActionRetriesExceeds() throws Exception {
CompositeConfiguration config = new CompositeConfiguration();
if (System.getProperty("config") != null) {
config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
}
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.service-name", "test-service");
conf.addProperty("whirr.cluster-name", "test-cluster");
conf.addProperty("whirr.instance-templates", "1 hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.instance-templates-max-percent-failures", "60 hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.provider", "ec2");
config.addConfiguration(conf);
ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(conf);
Set<String> jtnn = new HashSet<String>();
jtnn.add("hadoop-jobtracker");
jtnn.add("hadoop-namenode");
Set<String> dntt = new HashSet<String>();
dntt.add("hadoop-datanode");
dntt.add("hadoop-tasktracker");
TestNodeStarterFactory nodeStarterFactory = null;
ClusterActionHandler handler = mock(ClusterActionHandler.class);
LoadingCache<String, ClusterActionHandler> handlerMap = convertMapToLoadingCache(ImmutableMap.<String, ClusterActionHandler>builder().put("hadoop-jobtracker", handler).put("hadoop-namenode", handler).put("hadoop-datanode", handler).put("hadoop-tasktracker", handler).build());
Function<ClusterSpec, ComputeServiceContext> getCompute = mock(Function.class);
ComputeServiceContext serviceContext = mock(ComputeServiceContext.class);
ComputeService computeService = mock(ComputeService.class);
TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
Template template = mock(Template.class);
TemplateOptions templateOptions = mock(TemplateOptions.class);
when(getCompute.apply(clusterSpec)).thenReturn(serviceContext);
when(serviceContext.getComputeService()).thenReturn(computeService);
when(computeService.getContext()).thenReturn(serviceContext);
when(serviceContext.getBackendType()).thenReturn(TypeToken.class.cast(TypeToken.of(Context.class)));
when(computeService.templateBuilder()).thenReturn(templateBuilder);
when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
when(templateBuilder.build()).thenReturn(template);
when(template.getOptions()).thenReturn(templateOptions);
// here is a scenario when jt+nn does not fail
// but the dn+tt one node fails 3, then in the retry fails 2
// at the end result, the cluster will fail, throwing IOException
Map<Set<String>, Stack<Integer>> reaction = Maps.newHashMap();
Stack<Integer> jtnnStack = new Stack<Integer>();
jtnnStack.push(new Integer(1));
reaction.put(jtnn, jtnnStack);
Stack<Integer> ddttStack = new Stack<Integer>();
// 1 from 4, retryRequired
ddttStack.push(new Integer(1));
// 1 from 4, still retryRequired
ddttStack.push(new Integer(1));
reaction.put(dntt, ddttStack);
nodeStarterFactory = new TestNodeStarterFactory(reaction);
BootstrapClusterAction bootstrapper = new BootstrapClusterAction(getCompute, handlerMap, nodeStarterFactory);
// this should file with too many retries
bootstrapper.execute(clusterSpec, null);
if (nodeStarterFactory != null) {
nodeStarterFactory.validateCompletion();
}
}
Aggregations