Examples with Processor org.jclouds.compute.domain.Processor used on opensource projects

Example 1 with Processor

use of org.jclouds.compute.domain.Processor in project whirr by apache.

the class HadoopConfigurationBuilder method buildMapReduceConfiguration.

@VisibleForTesting
static Configuration buildMapReduceConfiguration(ClusterSpec clusterSpec, Cluster cluster, Configuration defaults, Set<String> dataDirectories) throws ConfigurationException, IOException {
    Configuration config = build(clusterSpec, cluster, defaults, "hadoop-mapreduce");
    setIfAbsent(config, "mapred.local.dir", appendToDataDirectories(dataDirectories, "/hadoop/mapred/local"));
    Set<Instance> taskTrackers = cluster.getInstancesMatching(role(HadoopTaskTrackerClusterActionHandler.ROLE));
    if (!taskTrackers.isEmpty()) {
        Hardware hardware = Iterables.getFirst(taskTrackers, null).getNodeMetadata().getHardware();
        /* null when using the BYON jclouds compute provider */
        if (hardware != null) {
            int coresPerNode = 0;
            for (Processor processor : hardware.getProcessors()) {
                coresPerNode += processor.getCores();
            }
            int mapTasksPerNode = (int) Math.ceil(coresPerNode * 1.0);
            int reduceTasksPerNode = (int) Math.ceil(coresPerNode * 0.75);
            setIfAbsent(config, "mapred.tasktracker.map.tasks.maximum", mapTasksPerNode + "");
            setIfAbsent(config, "mapred.tasktracker.reduce.tasks.maximum", reduceTasksPerNode + "");
            int clusterReduceSlots = taskTrackers.size() * reduceTasksPerNode;
            setIfAbsent(config, "mapred.reduce.tasks", clusterReduceSlots + "");
        }
    }
    Set<Instance> jobtracker = cluster.getInstancesMatching(role(HadoopJobTrackerClusterActionHandler.ROLE));
    if (!jobtracker.isEmpty()) {
        config.setProperty("mapred.job.tracker", String.format("%s:8021", Iterables.getOnlyElement(jobtracker).getPublicHostName()));
    }
    return config;
}

Example 2 with Processor

use of org.jclouds.compute.domain.Processor in project whirr by apache.

the class HadoopConfigurationBuilderTest method newCluster.

private Cluster newCluster(int numberOfWorkers) {
    DnsResolver fakeDnsResolver = new FakeDnsResolver();
    NodeMetadata node = mock(NodeMetadata.class);
    List<Processor> processors = ImmutableList.of(new Processor(4, 1.0));
    Hardware hardware = new HardwareImpl(null, null, "id", null, null, ImmutableMap.<String, String>of(), ImmutableSet.<String>of(), processors, 1024, ImmutableList.<Volume>of(), null, "xen");
    when(node.getHardware()).thenReturn(hardware);
    Builder<Instance> instances = ImmutableSet.<Instance>builder();
    Instance master = new Instance(new Credentials("", ""), ImmutableSet.of(HadoopNameNodeClusterActionHandler.ROLE, HadoopJobTrackerClusterActionHandler.ROLE), "10.0.0.1", "10.0.0.1", "1", node, fakeDnsResolver);
    instances.add(master);
    for (int i = 0; i < numberOfWorkers; i++) {
        int id = i + 2;
        instances.add(new Instance(new Credentials("", ""), ImmutableSet.of(HadoopDataNodeClusterActionHandler.ROLE, HadoopTaskTrackerClusterActionHandler.ROLE), "10.0.0." + id, "10.0.0." + id, id + "", node, fakeDnsResolver));
    }
    return new Cluster(instances.build());
}