Search in sources :

Example 11 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testMapper.

@Test(timeout = 120000)
public void testMapper() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TestMapper()).submit();
    Map<String, List<Integer>> result = future.get();
    assertEquals(100, result.size());
    for (List<Integer> value : result.values()) {
        assertEquals(1, value.size());
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) HazelcastInstance(com.hazelcast.core.HazelcastInstance) List(java.util.List) HashMap(java.util.HashMap) MultiMap(com.hazelcast.core.MultiMap) Map(java.util.Map) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 12 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class DistributedMapperClientMapReduceTest method testAsyncMapperReducerCollator.

@Test(timeout = 120000)
public void testAsyncMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient(null);
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final int[] result = new int[1];
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).combiner(new TestCombinerFactory()).reducer(new TestReducerFactory()).submit(new TestCollator());
    future.andThen(new ExecutionCallback<Integer>() {

        @Override
        public void onResponse(Integer response) {
            try {
                result[0] = response.intValue();
            } finally {
                semaphore.release();
            }
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result[0]);
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 13 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class DistributedMapperClientMapReduceTest method testAsyncMapperReducer.

@Test(timeout = 120000)
public void testAsyncMapperReducer() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient(null);
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final Map<String, Integer> listenerResults = new HashMap<String, Integer>();
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, Integer>> future = job.mapper(new GroupingTestMapper()).combiner(new TestCombinerFactory()).reducer(new TestReducerFactory()).submit();
    future.andThen(new ExecutionCallback<Map<String, Integer>>() {

        @Override
        public void onResponse(Map<String, Integer> response) {
            try {
                listenerResults.putAll(response);
            } finally {
                semaphore.release();
            }
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate results
    int[] expectedResults = new int[4];
    for (int i = 0; i < 100; i++) {
        int index = i % 4;
        expectedResults[index] += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResults[i], (int) listenerResults.get(String.valueOf(i)));
    }
}
Also used : HashMap(java.util.HashMap) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) HashMap(java.util.HashMap) Map(java.util.Map) IMap(com.hazelcast.core.IMap) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 14 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class DistributedMapperClientMapReduceTest method testMapperReducer.

@Test(timeout = 120000)
public void testMapperReducer() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient(null);
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, Integer>> future = job.mapper(new GroupingTestMapper()).combiner(new TestCombinerFactory()).reducer(new TestReducerFactory()).submit();
    Map<String, Integer> result = future.get();
    // Precalculate results
    int[] expectedResults = new int[4];
    for (int i = 0; i < 100; i++) {
        int index = i % 4;
        expectedResults[index] += i;
    }
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResults[i], (int) result.get(String.valueOf(i)));
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) HazelcastInstance(com.hazelcast.core.HazelcastInstance) HashMap(java.util.HashMap) Map(java.util.Map) IMap(com.hazelcast.core.IMap) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 15 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class DistributedMapperClientMultiMapReduceTest method testAsyncMapperReducerCollator.

@Test(timeout = 120000)
public void testAsyncMapperReducerCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient(null);
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    final int[] result = new int[1];
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).combiner(new TestCombinerFactory()).reducer(new TestReducerFactory()).submit(new TestCollator());
    future.andThen(new ExecutionCallback<Integer>() {

        @Override
        public void onResponse(Integer response) {
            try {
                result[0] = response.intValue();
            } finally {
                semaphore.release();
            }
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result[0]);
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Aggregations

HazelcastInstance (com.hazelcast.core.HazelcastInstance)41 JobTracker (com.hazelcast.mapreduce.JobTracker)41 Test (org.junit.Test)37 Config (com.hazelcast.config.Config)36 SlowTest (com.hazelcast.test.annotation.SlowTest)26 Map (java.util.Map)20 HashMap (java.util.HashMap)18 Semaphore (java.util.concurrent.Semaphore)14 List (java.util.List)10 IMap (com.hazelcast.core.IMap)9 MultiMap (com.hazelcast.core.MultiMap)9 NightlyTest (com.hazelcast.test.annotation.NightlyTest)8 CancellationException (java.util.concurrent.CancellationException)4 ExecutionException (java.util.concurrent.ExecutionException)4 ParallelTest (com.hazelcast.test.annotation.ParallelTest)3 QuickTest (com.hazelcast.test.annotation.QuickTest)3 ListSetMapReduceTest (com.hazelcast.mapreduce.ListSetMapReduceTest)2