Examples with Cluster org.apache.whirr.Cluster used on opensource projects

Example 41 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class BlobClusterStateStoreTest method testStoreAndLoadState.

@Test(timeout = TestConstants.ITEST_TIMEOUT)
public void testStoreAndLoadState() throws Exception {
    ClusterSpec spec = getTestClusterSpec();
    BlobStoreContext context = BlobStoreContextBuilder.build(spec);
    String container = generateRandomContainerName(context);
    try {
        spec.setStateStore("blob");
        spec.setStateStoreContainer(container);
        Cluster expected = createTestCluster(new String[] { "region/id1", "region/id2" }, new String[] { "role1", "role2" });
        BlobClusterStateStore store = new BlobClusterStateStore(spec);
        store.save(expected);
        /* load and check the stored state */
        Cluster stored = store.load();
        Cluster.Instance first = Iterables.getFirst(stored.getInstances(), null);
        assertNotNull(first);
        assertThat(first.getId(), is("region/id1"));
        assertThat(first.getRoles().contains("role1"), is(true));
        assertThat(stored.getInstances().size(), is(2));
        /* destroy stored state and check it no longer exists */
        store.destroy();
        expected = store.load();
        assertNull(expected);
    } finally {
        LOG.info("Removing temporary container '{}'", container);
        context.getBlobStore().deleteContainer(container);
    }
}

Example 42 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class BlobClusterStateStoreTest method createTestCluster.

private Cluster createTestCluster(String[] ids, String[] roles) {
    checkArgument(ids.length == roles.length, "each ID should have a role");
    Credentials credentials = new Credentials("dummy", "dummy");
    Set<Cluster.Instance> instances = Sets.newLinkedHashSet();
    for (int i = 0; i < ids.length; i++) {
        String ip = "127.0.0." + (i + 1);
        instances.add(new Cluster.Instance(credentials, Sets.newHashSet(roles[i]), ip, ip, ids[i], null));
    }
    return new Cluster(instances);
}

Example 43 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HadoopClusterExample method main.

@Override
public int main(String[] args) throws Exception {
    if (!System.getenv().containsKey("AWS_ACCESS_KEY_ID")) {
        LOG.error("AWS_ACCESS_KEY_ID is undefined in the current environment");
        return -1;
    }
    if (!System.getenv().containsKey("AWS_SECRET_ACCESS_KEY")) {
        LOG.error("AWS_SECRET_ACCESS_KEY is undefined in the current environment");
        return -2;
    }
    /**
     * Start by loading cluster configuration file and creating a ClusterSpec object
     *
     * You can find the file in the resources folder.
     */
    ClusterSpec spec = new ClusterSpec(new PropertiesConfiguration("whirr-hadoop-example.properties"));
    /**
     * Create an instance of the generic cluster controller
     */
    ClusterControllerFactory factory = new ClusterControllerFactory();
    ClusterController controller = factory.create(spec.getServiceName());
    /**
     * Start the cluster as defined in the configuration file
     */
    HadoopProxy proxy = null;
    try {
        LOG.info("Starting cluster {}", spec.getClusterName());
        Cluster cluster = controller.launchCluster(spec);
        LOG.info("Starting local SOCKS proxy");
        proxy = new HadoopProxy(spec, cluster);
        proxy.start();
        /**
         * Obtain a Hadoop configuration object and wait for services to start
         */
        Configuration config = getHadoopConfiguration(cluster);
        JobConf job = new JobConf(config, HadoopClusterExample.class);
        JobClient client = new JobClient(job);
        waitToExitSafeMode(client);
        waitForTaskTrackers(client);
        /**
         * Run a simple job to show that the cluster is available for work.
         */
        runWordCountingJob(config);
    } finally {
        /**
         * Stop the proxy and terminate all the cluster instances.
         */
        if (proxy != null) {
            proxy.stop();
        }
        controller.destroyCluster(spec);
        return 0;
    }
}

Example 44 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class UtilsTest method testPrintAccess.

@Test
public void testPrintAccess() {
    final Instance instance = mock(Instance.class);
    when(instance.getPublicIp()).thenReturn("test-public-ip");
    when(instance.getRoles()).thenReturn(ImmutableSet.of("test-role"));
    Cluster cluster = mock(Cluster.class);
    when(cluster.getInstances()).thenReturn(ImmutableSet.<Cluster.Instance>of(instance));
    ClusterSpec spec = mock(ClusterSpec.class);
    when(spec.getClusterUser()).thenReturn("test-identity");
    when(spec.getPrivateKeyFile()).thenReturn(new File("/test/key/path"));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    Utils.printSSHConnectionDetails(ps, spec, cluster, 20);
    assertEquals("The ssh command did not match", EXPECTED_SSH_COMMAND, new String(baos.toByteArray()));
}

Example 45 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class ElasticSearchHandler method beforeConfigure.

@Override
protected void beforeConfigure(ClusterActionEvent event) throws IOException {
    ClusterSpec spec = event.getClusterSpec();
    Cluster cluster = event.getCluster();
    event.getFirewallManager().addRule(Rule.create().destination(cluster.getInstancesMatching(role(ROLE))).port(HTTP_CLIENT_PORT));
    handleFirewallRules(event);
    Configuration config = ElasticSearchConfigurationBuilder.buildConfig(spec, cluster);
    addStatement(event, call("retry_helpers"));
    addStatement(event, ElasticSearchConfigurationBuilder.build("/tmp/elasticsearch.yml", config));
    addStatement(event, call("configure_elasticsearch", config.getStringArray("es.plugins")));
}