Examples with ClusterSpec org.apache.whirr.ClusterSpec used on opensource projects

Example 6 with ClusterSpec

use of org.apache.whirr.ClusterSpec in project whirr by apache.

the class BootstrapClusterActionTest method testDoActionRetriesExceeds.

@SuppressWarnings("unchecked")
@Test(expected = IOException.class)
public void testDoActionRetriesExceeds() throws Exception {
    CompositeConfiguration config = new CompositeConfiguration();
    if (System.getProperty("config") != null) {
        config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
    }
    Configuration conf = new PropertiesConfiguration();
    conf.addProperty("whirr.service-name", "test-service");
    conf.addProperty("whirr.cluster-name", "test-cluster");
    conf.addProperty("whirr.instance-templates", "1 hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
    conf.addProperty("whirr.instance-templates-max-percent-failures", "60 hadoop-datanode+hadoop-tasktracker");
    conf.addProperty("whirr.provider", "ec2");
    config.addConfiguration(conf);
    ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(conf);
    Set<String> jtnn = new HashSet<String>();
    jtnn.add("hadoop-jobtracker");
    jtnn.add("hadoop-namenode");
    Set<String> dntt = new HashSet<String>();
    dntt.add("hadoop-datanode");
    dntt.add("hadoop-tasktracker");
    TestNodeStarterFactory nodeStarterFactory = null;
    ClusterActionHandler handler = mock(ClusterActionHandler.class);
    LoadingCache<String, ClusterActionHandler> handlerMap = convertMapToLoadingCache(ImmutableMap.<String, ClusterActionHandler>builder().put("hadoop-jobtracker", handler).put("hadoop-namenode", handler).put("hadoop-datanode", handler).put("hadoop-tasktracker", handler).build());
    Function<ClusterSpec, ComputeServiceContext> getCompute = mock(Function.class);
    ComputeServiceContext serviceContext = mock(ComputeServiceContext.class);
    ComputeService computeService = mock(ComputeService.class);
    TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
    Template template = mock(Template.class);
    TemplateOptions templateOptions = mock(TemplateOptions.class);
    when(getCompute.apply(clusterSpec)).thenReturn(serviceContext);
    when(serviceContext.getComputeService()).thenReturn(computeService);
    when(computeService.getContext()).thenReturn(serviceContext);
    when(serviceContext.getBackendType()).thenReturn(TypeToken.class.cast(TypeToken.of(Context.class)));
    when(computeService.templateBuilder()).thenReturn(templateBuilder);
    when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
    when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
    when(templateBuilder.build()).thenReturn(template);
    when(template.getOptions()).thenReturn(templateOptions);
    // here is a scenario when jt+nn does not fail
    // but the dn+tt one node fails 3, then in the retry fails 2
    // at the end result, the cluster will fail, throwing IOException
    Map<Set<String>, Stack<Integer>> reaction = Maps.newHashMap();
    Stack<Integer> jtnnStack = new Stack<Integer>();
    jtnnStack.push(new Integer(1));
    reaction.put(jtnn, jtnnStack);
    Stack<Integer> ddttStack = new Stack<Integer>();
    // 1 from 4, retryRequired
    ddttStack.push(new Integer(1));
    // 1 from 4, still retryRequired
    ddttStack.push(new Integer(1));
    reaction.put(dntt, ddttStack);
    nodeStarterFactory = new TestNodeStarterFactory(reaction);
    BootstrapClusterAction bootstrapper = new BootstrapClusterAction(getCompute, handlerMap, nodeStarterFactory);
    // this should file with too many retries
    bootstrapper.execute(clusterSpec, null);
    if (nodeStarterFactory != null) {
        nodeStarterFactory.validateCompletion();
    }
}

Example 7 with ClusterSpec

use of org.apache.whirr.ClusterSpec in project whirr by apache.

the class DryRunModuleTest method testNoInitScriptsAfterConfigurationStartedAndNoConfigScriptsAfterDestroy.

/**
 * Simple test that tests dry run module and at the same time enforces clear
 * separation of script execution phases.
 */
@Test
public void testNoInitScriptsAfterConfigurationStartedAndNoConfigScriptsAfterDestroy() throws ConfigurationException, JSchException, IOException, InterruptedException {
    final List<String> expectedExecutionOrder = ImmutableList.of("bootstrap", "configure", "start", "destroy");
    CompositeConfiguration config = new CompositeConfiguration();
    config.setProperty("whirr.provider", "stub");
    config.setProperty("whirr.cluster-name", "stub-test");
    config.setProperty("whirr.instance-templates", "10 noop+noop3,10 noop2+noop,10 noop3+noop2");
    config.setProperty("whirr.state-store", "memory");
    ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(config);
    ClusterController controller = new ClusterController();
    DryRun dryRun = getDryRunInControllerForCluster(controller, clusterSpec);
    dryRun.reset();
    controller.launchCluster(clusterSpec);
    controller.destroyCluster(clusterSpec);
    ListMultimap<NodeMetadata, Statement> perNodeExecutions = dryRun.getExecutions();
    List<StatementOnNode> totalExecutions = dryRun.getTotallyOrderedExecutions();
    for (Entry<NodeMetadata, Collection<Statement>> entry : perNodeExecutions.asMap().entrySet()) {
        assertSame("An incorrect number of scripts was executed in the node: " + entry.getValue(), entry.getValue().size(), expectedExecutionOrder.size());
        List<Statement> asList = Lists.newArrayList(entry.getValue());
        int count = 0;
        for (String phase : expectedExecutionOrder) {
            String scriptName = getScriptName(asList.get(count));
            assertTrue("The '" + phase + "' script was executed in the wrong order, found: " + scriptName, scriptName.startsWith(phase));
            count += 1;
        }
    }
    // This tests the barrier by making sure that once a configure
    // script is executed no more setup scripts are executed
    Stack<String> executedPhases = new Stack<String>();
    for (StatementOnNode script : totalExecutions) {
        String[] parts = getScriptName(script.getStatement()).split("-");
        if ((!executedPhases.empty() && !executedPhases.peek().equals(parts[0])) || executedPhases.empty()) {
            executedPhases.push(parts[0]);
        }
    }
    // Assert that all scripts executed in the right order with no overlaps
    assertEquals(expectedExecutionOrder.size(), executedPhases.size());
    for (String phaseName : Lists.reverse(expectedExecutionOrder)) {
        assertEquals(executedPhases.pop(), phaseName);
    }
}

Example 8 with ClusterSpec

use of org.apache.whirr.ClusterSpec in project whirr by apache.

the class DryRunModuleTest method testExecuteOnlyBootstrapForNoopWithListener.

@Test
public void testExecuteOnlyBootstrapForNoopWithListener() throws Exception {
    CompositeConfiguration config = new CompositeConfiguration();
    config.setProperty("whirr.provider", "stub");
    config.setProperty("whirr.cluster-name", "stub-test");
    config.setProperty("whirr.instance-templates", "1 noop");
    config.setProperty("whirr.state-store", "memory");
    ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(config);
    MockClusterActionHandlerListener listener = new MockClusterActionHandlerListener();
    clusterSpec.setHandlerListener(listener);
    ClusterController controller = new ClusterController();
    DryRun dryRun = getDryRunInControllerForCluster(controller, clusterSpec);
    dryRun.reset();
    controller.launchCluster(clusterSpec);
    controller.destroyCluster(clusterSpec);
    ListMultimap<NodeMetadata, Statement> perNodeExecutions = dryRun.getExecutions();
    for (Entry<NodeMetadata, Collection<Statement>> entry : perNodeExecutions.asMap().entrySet()) {
        assertSame("An incorrect number of scripts was executed in the node " + entry, entry.getValue().size(), 1);
    }
    assertEquals("beforeCalls should be 4", listener.beforeCalls, 4);
    assertEquals("afterCalls should be 4", listener.afterCalls, 4);
}

Example 9 with ClusterSpec

use of org.apache.whirr.ClusterSpec in project whirr by apache.

the class ScriptBasedClusterActionTest method testEmptyInstanceTemplates.

@Test(expected = IllegalArgumentException.class)
public void testEmptyInstanceTemplates() throws Exception {
    T action = newClusterActionInstance(EMPTYSET, EMPTYSET);
    DryRun dryRun = getDryRunForAction(action).reset();
    ClusterSpec tempSpec = ClusterSpec.withTemporaryKeys();
    tempSpec.setClusterName("test-cluster-for-script-exection");
    tempSpec.setProvider("stub");
    tempSpec.setIdentity("dummy");
    tempSpec.setStateStore("none");
    ClusterController controller = new ClusterController();
    Cluster tempCluster = controller.launchCluster(tempSpec);
    action.execute(tempSpec, tempCluster);
    List<StatementOnNode> executions = dryRun.getTotallyOrderedExecutions();
}

Example 10 with ClusterSpec

use of org.apache.whirr.ClusterSpec in project whirr by apache.

the class AbstractClusterCommandTest method testOverrides.

@Test
public void testOverrides() throws Exception {
    AbstractClusterCommand clusterCommand = new AbstractClusterCommand("name", "description", new ClusterControllerFactory()) {

        @Override
        public int run(InputStream in, PrintStream out, PrintStream err, List<String> args) throws Exception {
            return 0;
        }
    };
    Map<String, File> keys = KeyPair.generateTemporaryFiles();
    OptionSet optionSet = clusterCommand.parser.parse("--quiet", "--service-name", "overridden-test-service", "--config", "whirr-override-test.properties", "--private-key-file", keys.get("private").getAbsolutePath());
    ClusterSpec clusterSpec = clusterCommand.getClusterSpec(optionSet);
    assertThat(optionSet.has("quiet"), is(true));
    assertThat(clusterSpec.isQuiet(), is(true));
    assertThat(clusterSpec.getServiceName(), is("overridden-test-service"));
    assertThat(clusterSpec.getClusterName(), is("test-cluster"));
    optionSet = clusterCommand.parser.parse("--quiet", "true", "--service-name", "overridden-test-service", "--config", "whirr-override-test.properties", "--private-key-file", keys.get("private").getAbsolutePath());
    clusterSpec = clusterCommand.getClusterSpec(optionSet);
    assertThat(optionSet.has("quiet"), is(true));
    assertThat(clusterSpec.isQuiet(), is(true));
    assertThat(clusterSpec.getServiceName(), is("overridden-test-service"));
    assertThat(clusterSpec.getClusterName(), is("test-cluster"));
    optionSet = clusterCommand.parser.parse("--quiet", "false", "--service-name", "overridden-test-service", "--config", "whirr-override-test.properties", "--private-key-file", keys.get("private").getAbsolutePath());
    clusterSpec = clusterCommand.getClusterSpec(optionSet);
    assertThat(optionSet.has("quiet"), is(true));
    assertThat(clusterSpec.isQuiet(), is(false));
    assertThat(clusterSpec.getServiceName(), is("overridden-test-service"));
    assertThat(clusterSpec.getClusterName(), is("test-cluster"));
    optionSet = clusterCommand.parser.parse("--quiet", "some-value", "--service-name", "overridden-test-service", "--config", "whirr-override-test.properties", "--private-key-file", keys.get("private").getAbsolutePath());
    clusterSpec = clusterCommand.getClusterSpec(optionSet);
    assertThat(optionSet.has("quiet"), is(true));
    assertThat(clusterSpec.isQuiet(), is(false));
    assertThat(clusterSpec.getServiceName(), is("overridden-test-service"));
    assertThat(clusterSpec.getClusterName(), is("test-cluster"));
    optionSet = clusterCommand.parser.parse("--service-name", "overridden-test-service", "--config", "whirr-override-test.properties", "--private-key-file", keys.get("private").getAbsolutePath());
    clusterSpec = clusterCommand.getClusterSpec(optionSet);
    assertThat(optionSet.has("quiet"), is(false));
    assertThat(clusterSpec.isQuiet(), is(false));
    assertThat(clusterSpec.getServiceName(), is("overridden-test-service"));
    assertThat(clusterSpec.getClusterName(), is("test-cluster"));
}