Examples with JobTracker com.hazelcast.mapreduce.JobTracker used on opensource projects

Example 31 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testKeyedAsyncMapper.

@Test(timeout = 120000)
public void testKeyedAsyncMapper() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final Map<String, List<Integer>> listenerResults = new HashMap<String, List<Integer>>();
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, List<Integer>>> future = job.onKeys(50).mapper(new TestMapper()).submit();
    future.andThen(new ExecutionCallback<Map<String, List<Integer>>>() {

        @Override
        public void onResponse(Map<String, List<Integer>> response) {
            listenerResults.putAll(response);
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    semaphore.acquire();
    assertEquals(1, listenerResults.size());
    for (List<Integer> value : listenerResults.values()) {
        assertEquals(1, value.size());
    }
}

Example 32 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testKeyedMapperCollator.

@Test(timeout = 120000)
public void testKeyedMapperCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 10000; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.onKeys(50).mapper(new TestMapper()).submit(new GroupingTestCollator());
    int result = future.get();
    assertEquals(50, result);
}

Example 33 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testKeyPredicateMapperCollator.

@Test(timeout = 120000)
public void testKeyPredicateMapperCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 10000; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.keyPredicate(new TestKeyPredicate()).mapper(new TestMapper()).submit(new GroupingTestCollator());
    int result = future.get();
    assertEquals(50, result);
}

Example 34 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testAsyncMapper.

@Test(timeout = 120000)
public void testAsyncMapper() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final Map<String, List<Integer>> listenerResults = new HashMap<String, List<Integer>>();
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TestMapper()).submit();
    future.andThen(new ExecutionCallback<Map<String, List<Integer>>>() {

        @Override
        public void onResponse(Map<String, List<Integer>> response) {
            listenerResults.putAll(response);
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    semaphore.acquire();
    assertEquals(100, listenerResults.size());
    for (List<Integer> value : listenerResults.values()) {
        assertEquals(1, value.size());
    }
}

Example 35 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testAsyncMapperReducerCollator.

@Test(timeout = 120000)
public void testAsyncMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    final int[] result = new int[1];
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory()).submit(new TestCollator());
    future.andThen(new ExecutionCallback<Integer>() {

        @Override
        public void onResponse(Integer response) {
            result[0] = response.intValue();
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result[0]);
    }
}