Examples with Cluster org.apache.whirr.Cluster used on opensource projects

Example 11 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HadoopNameNodeClusterActionHandler method doBeforeConfigure.

@Override
protected void doBeforeConfigure(ClusterActionEvent event) throws IOException {
    Cluster cluster = event.getCluster();
    Instance namenode = cluster.getInstanceMatching(role(ROLE));
    event.getFirewallManager().addRules(Rule.create().destination(namenode).ports(HadoopCluster.NAMENODE_WEB_UI_PORT), Rule.create().source(namenode.getPublicAddress().getHostAddress()).destination(namenode).ports(HadoopCluster.NAMENODE_PORT, HadoopCluster.JOBTRACKER_PORT));
}

Example 12 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HadoopNameNodeClusterActionHandler method afterConfigure.

@Override
protected void afterConfigure(ClusterActionEvent event) throws IOException {
    ClusterSpec clusterSpec = event.getClusterSpec();
    Cluster cluster = event.getCluster();
    // TODO: wait for TTs to come up (done in test for the moment)
    LOG.info("Completed configuration of {} role {}", clusterSpec.getClusterName(), getRole());
    InetAddress namenodePublicAddress = HadoopCluster.getNamenodePublicAddress(cluster);
    InetAddress jobtrackerPublicAddress = HadoopCluster.getJobTrackerPublicAddress(cluster);
    LOG.info("Namenode web UI available at http://{}:{}", namenodePublicAddress.getHostName(), HadoopCluster.NAMENODE_WEB_UI_PORT);
    Properties config = createClientSideProperties(clusterSpec, namenodePublicAddress, jobtrackerPublicAddress);
    createClientSideHadoopSiteFile(clusterSpec, config);
    createProxyScript(clusterSpec, cluster);
    Properties combined = new Properties();
    combined.putAll(cluster.getConfiguration());
    combined.putAll(config);
    event.setCluster(new Cluster(cluster.getInstances(), combined));
}

Example 13 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HadoopJobTrackerClusterActionHandler method afterConfigure.

@Override
protected void afterConfigure(ClusterActionEvent event) throws IOException {
    ClusterSpec clusterSpec = event.getClusterSpec();
    Cluster cluster = event.getCluster();
    LOG.info("Completed configuration of {} role {}", clusterSpec.getClusterName(), getRole());
    InetAddress jobtrackerPublicAddress = HadoopCluster.getJobTrackerPublicAddress(cluster);
    LOG.info("Jobtracker web UI available at http://{}:{}", jobtrackerPublicAddress.getHostName(), HadoopCluster.JOBTRACKER_WEB_UI_PORT);
}

Example 14 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HadoopJobTrackerClusterActionHandler method doBeforeConfigure.

@Override
protected void doBeforeConfigure(ClusterActionEvent event) throws IOException {
    Cluster cluster = event.getCluster();
    Instance jobtracker = cluster.getInstanceMatching(role(ROLE));
    event.getFirewallManager().addRules(Rule.create().destination(jobtracker).ports(HadoopCluster.JOBTRACKER_WEB_UI_PORT), Rule.create().source(HadoopCluster.getNamenodePublicAddress(cluster).getHostAddress()).destination(jobtracker).ports(HadoopCluster.JOBTRACKER_PORT));
}

Example 15 with Cluster

use of org.apache.whirr.Cluster in project whirr by apache.

the class HadoopConfigurationBuilderTest method testMapReduce.

@Test
public void testMapReduce() throws Exception {
    Cluster cluster = newCluster(5);
    Configuration conf = HadoopConfigurationBuilder.buildMapReduceConfiguration(clusterSpec, cluster, defaults, Sets.newLinkedHashSet(Lists.newArrayList("/data0", "/data1")));
    assertThat(conf.getString("p1"), is("mapred1"));
    assertThat(conf.getString("mapred.job.tracker"), matches(".+:8021"));
    assertThat(conf.getString("mapred.tasktracker.map.tasks.maximum"), is("4"));
    assertThat(conf.getString("mapred.tasktracker.reduce.tasks.maximum"), is("3"));
    assertThat(conf.getString("mapred.reduce.tasks"), is("15"));
    assertThat(conf.getString("mapred.local.dir"), is("/data0/hadoop/mapred/local,/data1/hadoop/mapred/local"));
}