Search in sources :

Example 26 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMapReduceTest method testAsyncMapperCollator.

@Test(timeout = 120000)
public void testAsyncMapperCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final int[] result = new int[1];
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = //
    job.mapper(new GroupingTestMapper()).submit(new GroupingTestCollator());
    future.andThen(new ExecutionCallback<Integer>() {

        @Override
        public void onResponse(Integer response) {
            result[0] = response.intValue();
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResult, result[0]);
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 27 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMapReduceTest method testMapperCollator.

@Test(timeout = 120000)
public void testMapperCollator() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).submit(new GroupingTestCollator());
    int result = future.get();
    // Precalculate result
    int expectedResult = 0;
    for (int i = 0; i < 100; i++) {
        expectedResult += i;
    }
}
Also used : HazelcastInstance(com.hazelcast.core.HazelcastInstance) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 28 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMapReduceTest method testKeyedAsyncMapper.

@Test(timeout = 120000)
public void testKeyedAsyncMapper() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    IMap<Integer, Integer> m1 = client.getMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final Map<String, List<Integer>> listenerResults = new HashMap<String, List<Integer>>();
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, List<Integer>>> future = job.onKeys(50).mapper(new TestMapper()).submit();
    future.andThen(new ExecutionCallback<Map<String, List<Integer>>>() {

        @Override
        public void onResponse(Map<String, List<Integer>> response) {
            listenerResults.putAll(response);
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    semaphore.acquire();
    assertEquals(1, listenerResults.size());
    for (List<Integer> value : listenerResults.values()) {
        assertEquals(1, value.size());
    }
}
Also used : HashMap(java.util.HashMap) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) IMap(com.hazelcast.core.IMap) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 29 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testAsyncMapperReducer.

@Test(timeout = 120000)
public void testAsyncMapperReducer() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i, i);
    }
    final Map<String, Integer> listenerResults = new HashMap<String, Integer>();
    final Semaphore semaphore = new Semaphore(1);
    semaphore.acquire();
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, Integer>> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory()).submit();
    future.andThen(new ExecutionCallback<Map<String, Integer>>() {

        @Override
        public void onResponse(Map<String, Integer> response) {
            listenerResults.putAll(response);
            semaphore.release();
        }

        @Override
        public void onFailure(Throwable t) {
            semaphore.release();
        }
    });
    // Precalculate results
    int[] expectedResults = new int[4];
    for (int i = 0; i < 100; i++) {
        int index = i % 4;
        expectedResults[index] += i;
    }
    semaphore.acquire();
    for (int i = 0; i < 4; i++) {
        assertEquals(expectedResults[i], (int) listenerResults.get(String.valueOf(i)));
    }
}
Also used : HashMap(java.util.HashMap) Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) Semaphore(java.util.concurrent.Semaphore) HazelcastInstance(com.hazelcast.core.HazelcastInstance) HashMap(java.util.HashMap) MultiMap(com.hazelcast.core.MultiMap) Map(java.util.Map) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Example 30 with JobTracker

use of com.hazelcast.mapreduce.JobTracker in project hazelcast by hazelcast.

the class ClientMultiMapReduceTest method testInProcessCancellation.

@Test(timeout = 60000, expected = CancellationException.class)
public void testInProcessCancellation() throws Exception {
    Config config = buildConfig();
    HazelcastInstance h1 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h2 = hazelcastFactory.newHazelcastInstance(config);
    HazelcastInstance h3 = hazelcastFactory.newHazelcastInstance(config);
    assertClusterSizeEventually(3, h1);
    assertClusterSizeEventually(3, h2);
    assertClusterSizeEventually(3, h3);
    HazelcastInstance client = hazelcastFactory.newHazelcastClient();
    MultiMap<Integer, Integer> m1 = client.getMultiMap(randomString());
    for (int i = 0; i < 100; i++) {
        m1.put(i / 2, i);
    }
    JobTracker tracker = client.getJobTracker("default");
    Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
    ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TimeConsumingMapper()).submit();
    future.cancel(true);
    try {
        Map<String, List<Integer>> result = future.get();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}
Also used : Config(com.hazelcast.config.Config) JobTracker(com.hazelcast.mapreduce.JobTracker) CancellationException(java.util.concurrent.CancellationException) ExecutionException(java.util.concurrent.ExecutionException) HazelcastInstance(com.hazelcast.core.HazelcastInstance) List(java.util.List) HashMap(java.util.HashMap) MultiMap(com.hazelcast.core.MultiMap) Map(java.util.Map) Test(org.junit.Test) SlowTest(com.hazelcast.test.annotation.SlowTest)

Aggregations

HazelcastInstance (com.hazelcast.core.HazelcastInstance)41 JobTracker (com.hazelcast.mapreduce.JobTracker)41 Test (org.junit.Test)37 Config (com.hazelcast.config.Config)36 SlowTest (com.hazelcast.test.annotation.SlowTest)26 Map (java.util.Map)20 HashMap (java.util.HashMap)18 Semaphore (java.util.concurrent.Semaphore)14 List (java.util.List)10 IMap (com.hazelcast.core.IMap)9 MultiMap (com.hazelcast.core.MultiMap)9 NightlyTest (com.hazelcast.test.annotation.NightlyTest)8 CancellationException (java.util.concurrent.CancellationException)4 ExecutionException (java.util.concurrent.ExecutionException)4 ParallelTest (com.hazelcast.test.annotation.ParallelTest)3 QuickTest (com.hazelcast.test.annotation.QuickTest)3 ListSetMapReduceTest (com.hazelcast.mapreduce.ListSetMapReduceTest)2