Examples with JobTracker com.hazelcast.mapreduce.JobTracker used on opensource projects

Example 36 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testExceptionDistribution.

@Test(expected = ExecutionException.class)
public void testExceptionDistribution() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new ExceptionThrowingMapper()).submit();
    try {
        Map<String, List<Integer>> result = future.get();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        assertTrue(e.getCause() instanceof NullPointerException);
        throw e;
    }
}

Example 37 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class DistributedMapperClientMapReduceTest method testMapperReducerCollator.

@Test(timeout = 120000)
public void testMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient(null);
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).combiner(new TestCombinerFactory()).reducer(new TestReducerFactory()).submit(new TestCollator());
    int result = future.get();
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result);
    }
}

Example 38 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class DistributedMapperClientMultiMapReduceTest method testMapperReducerCollator.

@Test(timeout = 120000)
public void testMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient(null);
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).combiner(new TestCombinerFactory()).reducer(new TestReducerFactory()).submit(new TestCollator());
    int result = future.get();
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result);
    }
}

Example 39 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ObjectMultiMapProxy method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation) {
    HazelcastInstance hazelcastInstance = getNodeEngine().getHazelcastInstance();
    JobTracker jobTracker = hazelcastInstance.getJobTracker("hz::aggregation-multimap-" + getName());
    return aggregate(supplier, aggregation, jobTracker);
}

Example 40 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class MapProxyImpl method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation) {
    checkTrue(NATIVE != mapConfig.getInMemoryFormat(), "NATIVE storage format is not supported for MapReduce");
    HazelcastInstance hazelcastInstance = getNodeEngine().getHazelcastInstance();
    JobTracker jobTracker = hazelcastInstance.getJobTracker("hz::aggregation-map-" + getName());
    return aggregate(supplier, aggregation, jobTracker);
}