Search in sources :

Example 6 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMapReduceTest method testKeyedMapperCollator.

@Test(timeout = 120000)
public void testKeyedMapperCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 10000; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.onKeys(50).mapper(new TestMapper()).submit(new GroupingTestCollator());
    int result = future.get();
    assertEquals(50, result);
}
Also used : HazelcastInstance(com.hazelcast.core.HazelcastInstance) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 7 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMapReduceTest method testMapperReducerCollator.

@Test(timeout = 120000)
public void testMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory()).submit(new TestCollator());
    int result = future.get();
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result);
    }
}
Also used : HazelcastInstance(com.hazelcast.core.HazelcastInstance) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 8 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMapReduceTest method testAsyncMapperReducerCollator.

@Test(timeout = 120000)
public void testAsyncMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final int[] result = new int[1];
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory()).submit(new TestCollator());
    future.andThen(new ExecutionCallback<Integer>() {

        @Override
        public void onResponse(Integer response) {
            result[0] = response.intValue();
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result[0]);
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 9 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testMapperReducerCollator.

@Test(timeout = 120000)
public void testMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory()).submit(new TestCollator());
    int result = future.get();
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result);
    }
}
Also used : HazelcastInstance(com.hazelcast.core.HazelcastInstance) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 10 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testAsyncMapperCollator.

@Test(timeout = 120000)
public void testAsyncMapperCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    final int[] result = new int[1];
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = //
    job.mapper(new GroupingTestMapper()).submit(new GroupingTestCollator());
    future.andThen(new ExecutionCallback<Integer>() {

        @Override
        public void onResponse(Integer response) {
            result[0] = response.intValue();
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result[0]);
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Aggregations

HazelcastInstance (com.hazelcast.core.HazelcastInstance)41 JobTracker (com.hazelcast.mapreduce.JobTracker)41 Test (org.junit.Test)37 Config (com.hazelcast.config.Config)36 SlowTest (com.hazelcast.test.annotation.SlowTest)26 Map (java.util.Map)20 HashMap (java.util.HashMap)18 Semaphore (java.util.concurrent.Semaphore)14 List (java.util.List)10 IMap (com.hazelcast.core.IMap)9 MultiMap (com.hazelcast.core.MultiMap)9 NightlyTest (com.hazelcast.test.annotation.NightlyTest)8 CancellationException (java.util.concurrent.CancellationException)4 ExecutionException (java.util.concurrent.ExecutionException)4 ParallelTest (com.hazelcast.test.annotation.ParallelTest)3 QuickTest (com.hazelcast.test.annotation.QuickTest)3 ListSetMapReduceTest (com.hazelcast.mapreduce.ListSetMapReduceTest)2