Examples with Cluster org.apache.whirr.Cluster used on opensource projects

Example 31 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HamaMasterClusterActionHandler method beforeConfigure.

@Override
protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException {
    ClusterSpec clusterSpec = event.getClusterSpec();
    Cluster cluster = event.getCluster();
    LOG.info("Authorizing firewall");
    Instance instance = cluster.getInstanceMatching(role(ROLE));
    InetAddress masterPublicAddress = instance.getPublicAddress();
    event.getFirewallManager().addRules(Rule.create().destination(instance).ports(MASTER_WEB_UI_PORT), Rule.create().destination(instance).ports(MASTER_PORT));
    handleFirewallRules(event);
    String hamaConfigureFunction = getConfiguration(clusterSpec).getString(HamaConstants.KEY_CONFIGURE_FUNCTION, HamaConstants.FUNCTION_POST_CONFIGURE);
    String master = masterPublicAddress.getHostName();
    String quorum = ZooKeeperCluster.getHosts(cluster);
    String tarurl = prepareRemoteFileUrl(event, getConfiguration(clusterSpec).getString(HamaConstants.KEY_TARBALL_URL));
    addStatement(event, call("retry_helpers"));
    addStatement(event, call(hamaConfigureFunction, ROLE, HamaConstants.PARAM_MASTER, master, HamaConstants.PARAM_QUORUM, quorum, HamaConstants.PARAM_TARBALL_URL, tarurl));
    String hamaStartFunction = getConfiguration(clusterSpec).getString(HamaConstants.KEY_START_FUNCTION, HamaConstants.FUNCTION_START);
    addStatement(event, call(hamaStartFunction, ROLE, HamaConstants.PARAM_TARBALL_URL, tarurl));
}

Example 32 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HamaMasterClusterActionHandler method afterConfigure.

@Override
protected void afterConfigure(ClusterActionEvent event) throws IOException {
    ClusterSpec clusterSpec = event.getClusterSpec();
    Cluster cluster = event.getCluster();
    LOG.info("Completed configuration of {}", clusterSpec.getClusterName());
    Instance instance = cluster.getInstanceMatching(role(ROLE));
    InetAddress masterPublicAddress = instance.getPublicAddress();
    LOG.info("BSPMaster web UI available at http://{}:{}", masterPublicAddress.getHostName(), MASTER_WEB_UI_PORT);
    String quorum = ZooKeeperCluster.getHosts(cluster);
    Properties config = createClientSideProperties(masterPublicAddress, quorum);
    createClientSideHadoopSiteFile(clusterSpec, config);
    createProxyScript(clusterSpec, cluster);
    event.setCluster(new Cluster(cluster.getInstances(), config));
}

Example 33 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class ListClusterCommandTest method createTestCluster.

private Cluster createTestCluster(String[] ids, String[] roles) {
    checkArgument(ids.length == roles.length, "each ID should have a role");
    Credentials credentials = new Credentials("dummy", "dummy");
    Set<Cluster.Instance> instances = Sets.newHashSet();
    for (int i = 0; i < ids.length; i++) {
        String ip = "127.0.0." + (i + 1);
        instances.add(new Cluster.Instance(credentials, Sets.newHashSet(roles[i]), ip, ip, ids[i], null));
    }
    return new Cluster(instances);
}

Example 34 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class LaunchClusterCommandTest method testAllOptions.

@Test
public void testAllOptions() throws Exception {
    ClusterControllerFactory factory = mock(ClusterControllerFactory.class);
    ClusterController controller = mock(ClusterController.class);
    Cluster cluster = mock(Cluster.class);
    when(factory.create((String) any())).thenReturn(controller);
    when(controller.launchCluster((ClusterSpec) any())).thenReturn(cluster);
    LaunchClusterCommand command = new LaunchClusterCommand(factory);
    Map<String, File> keys = KeyPair.generateTemporaryFiles();
    int rc = command.run(null, out, null, Lists.newArrayList("--service-name", "test-service", "--cluster-name", "test-cluster", "--instance-templates", "1 role1+role2,2 role3", "--provider", "rackspace", "--endpoint", "http://endpoint", "--blobstore-endpoint", "http://blobstore-endpoint", "--identity", "myusername", "--credential", "mypassword", "--private-key-file", keys.get("private").getAbsolutePath(), "--version", "version-string"));
    assertThat(rc, is(0));
    Configuration conf = new PropertiesConfiguration();
    conf.addProperty("whirr.version", "version-string");
    ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
    expectedClusterSpec.setInstanceTemplates(Lists.newArrayList(InstanceTemplate.builder().numberOfInstance(1).roles("role1", "role2").build(), InstanceTemplate.builder().numberOfInstance(2).roles("role3").build()));
    expectedClusterSpec.setServiceName("test-service");
    expectedClusterSpec.setProvider("rackspace");
    expectedClusterSpec.setEndpoint("http://endpoint");
    expectedClusterSpec.setIdentity("myusername");
    expectedClusterSpec.setCredential("mypassword");
    expectedClusterSpec.setBlobStoreEndpoint("http://blobstore-endpoint");
    expectedClusterSpec.setClusterName("test-cluster");
    expectedClusterSpec.setPrivateKey(keys.get("private"));
    expectedClusterSpec.setPublicKey(keys.get("public"));
    verify(factory).create("test-service");
    verify(controller).launchCluster(expectedClusterSpec);
    assertThat(outBytes.toString(), containsString("Started cluster of 0 instances"));
}

Example 35 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class LaunchClusterCommandTest method testMaxPercentFailure.

@Test
public void testMaxPercentFailure() throws Exception {
    ClusterControllerFactory factory = mock(ClusterControllerFactory.class);
    ClusterController controller = mock(ClusterController.class);
    Cluster cluster = mock(Cluster.class);
    when(factory.create((String) any())).thenReturn(controller);
    when(controller.launchCluster((ClusterSpec) any())).thenReturn(cluster);
    LaunchClusterCommand command = new LaunchClusterCommand(factory);
    Map<String, File> keys = KeyPair.generateTemporaryFiles();
    int rc = command.run(null, out, null, Lists.newArrayList("--service-name", "hadoop", "--cluster-name", "test-cluster", "--instance-templates", "1 hadoop-namenode+hadoop-jobtracker,3 hadoop-datanode+hadoop-tasktracker", "--instance-templates-max-percent-failures", "60 hadoop-datanode+hadoop-tasktracker", "--provider", "aws-ec2", "--identity", "myusername", "--credential", "mypassword", "--private-key-file", keys.get("private").getAbsolutePath(), "--version", "version-string"));
    assertThat(rc, is(0));
    Configuration conf = new PropertiesConfiguration();
    conf.addProperty("whirr.provider", "aws-ec2");
    conf.addProperty("whirr.version", "version-string");
    conf.addProperty("whirr.instance-templates-max-percent-failure", "60 hadoop-datanode+hadoop-tasktracker");
    ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
    expectedClusterSpec.setInstanceTemplates(Lists.newArrayList(InstanceTemplate.builder().numberOfInstance(1).minNumberOfInstances(1).roles("hadoop-namenode", "hadoop-jobtracker").build(), InstanceTemplate.builder().numberOfInstance(3).minNumberOfInstances(2).roles("hadoop-datanode", "hadoop-tasktracker").build()));
    expectedClusterSpec.setServiceName("hadoop");
    expectedClusterSpec.setIdentity("myusername");
    expectedClusterSpec.setCredential("mypassword");
    expectedClusterSpec.setClusterName("test-cluster");
    expectedClusterSpec.setPrivateKey(keys.get("private"));
    expectedClusterSpec.setPublicKey(keys.get("public"));
    verify(factory).create("hadoop");
    verify(controller).launchCluster(expectedClusterSpec);
    assertThat(outBytes.toString(), containsString("Started cluster of 0 instances"));
}