Examples with CombinerFactory com.hazelcast.mapreduce.CombinerFactory used on opensource projects

Example 1 with CombinerFactory

use of com.hazelcast.mapreduce.CombinerFactory in project hazelcast by hazelcast.

the class ClientMultiMapProxy method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    try {
        Preconditions.isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMultiMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory != null) {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        throw new HazelcastException(e);
    }
}

Example 2 with CombinerFactory

use of com.hazelcast.mapreduce.CombinerFactory in project hazelcast by hazelcast.

the class ObjectMultiMapProxy method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    try {
        isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMultiMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory != null) {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        throw new HazelcastException(e);
    }
}

Example 3 with CombinerFactory

use of com.hazelcast.mapreduce.CombinerFactory in project hazelcast by hazelcast.

the class ThreadsafeCombinerTest method github_issue_3625.

/**
     * Combiner creation is not threadsafe
     *
     * @throws Exception
     */
@Test
public void github_issue_3625() throws Exception {
    class TestCombinerFactory implements CombinerFactory {

        @Override
        public Combiner newCombiner(Object key) {
            return new Combiner() {

                @Override
                public void combine(Object value) {
                }

                @Override
                public Object finalizeChunk() {
                    return null;
                }
            };
        }
    }
    class CreationTask implements Runnable {

        private final CountDownLatch latchStart;

        private final CountDownLatch latchEnd;

        private final AtomicReferenceArray<Combiner> array;

        private final DefaultContext<Integer, Integer> defaultContext;

        private final int index;

        CreationTask(CountDownLatch latchStart, CountDownLatch latchEnd, AtomicReferenceArray<Combiner> array, DefaultContext<Integer, Integer> defaultContext, int index) {
            this.latchStart = latchStart;
            this.latchEnd = latchEnd;
            this.array = array;
            this.defaultContext = defaultContext;
            this.index = index;
        }

        @Override
        public void run() {
            try {
                latchStart.await();
                Combiner combiner = defaultContext.getOrCreateCombiner(1);
                array.set(index, combiner);
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                latchEnd.countDown();
            }
        }
    }
    int threadCount = 20;
    AtomicReferenceArray<Combiner> combiners = new AtomicReferenceArray<Combiner>(threadCount);
    DefaultContext<Integer, Integer> context = new DefaultContext<Integer, Integer>(new TestCombinerFactory(), null);
    CountDownLatch latchStart = new CountDownLatch(1);
    CountDownLatch latchEnd = new CountDownLatch(threadCount);
    for (int i = 0; i < threadCount; i++) {
        Thread t = new Thread(new CreationTask(latchStart, latchEnd, combiners, context, i));
        t.start();
    }
    latchStart.countDown();
    latchEnd.await(1, TimeUnit.MINUTES);
    for (int i = 0; i < threadCount - 1; i++) {
        Combiner c1 = combiners.get(i);
        Combiner c2 = combiners.get(i + 1);
        assertTrue("Returned combiners are not identical: " + c1 + " -> " + c2, c1 == c2);
    }
}

Example 4 with CombinerFactory

use of com.hazelcast.mapreduce.CombinerFactory in project hazelcast by hazelcast.

the class MapProxyImpl method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    checkTrue(NATIVE != mapConfig.getInMemoryFormat(), "NATIVE storage format is not supported for MapReduce");
    try {
        isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory == null) {
            reducingJob = mappingJob.reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        // TODO: not what we want, because it can lead to wrapping of HazelcastException
        throw new HazelcastException(e);
    }
}

Example 5 with CombinerFactory

use of com.hazelcast.mapreduce.CombinerFactory in project hazelcast by hazelcast.

the class AbstractMapReduceTask method startSupervisionTask.

private void startSupervisionTask(JobTracker jobTracker) {
    final MapReduceService mapReduceService = getService(MapReduceService.SERVICE_NAME);
    final JobTrackerConfig config = ((AbstractJobTracker) jobTracker).getJobTrackerConfig();
    final boolean communicateStats = config.isCommunicateStats();
    final int chunkSize = getChunkSizeOrConfigChunkSize(config);
    final TopologyChangedStrategy topologyChangedStrategy = getTopologyChangedStrategyOrConfigTopologyChangedStrategy(config);
    final String name = getDistributedObjectName();
    final String jobId = getJobId();
    final KeyValueSource keyValueSource = getKeyValueSource();
    final Mapper mapper = getMapper();
    final CombinerFactory combinerFactory = getCombinerFactory();
    final ReducerFactory reducerFactory = getReducerFactory();
    final Collection keys = getKeys();
    final Collection<Object> keyObjects = getKeyObjects(keys);
    final KeyPredicate predicate = getPredicate();
    final ClusterService clusterService = nodeEngine.getClusterService();
    for (Member member : clusterService.getMembers(KeyValueJobOperation.MEMBER_SELECTOR)) {
        Operation operation = new KeyValueJobOperation(name, jobId, chunkSize, keyValueSource, mapper, combinerFactory, reducerFactory, communicateStats, topologyChangedStrategy);
        executeOperation(operation, member.getAddress(), mapReduceService, nodeEngine);
    }
    // After we prepared all the remote systems we can now start the processing
    for (Member member : clusterService.getMembers(DATA_MEMBER_SELECTOR)) {
        Operation operation = new StartProcessingJobOperation(name, jobId, keyObjects, predicate);
        executeOperation(operation, member.getAddress(), mapReduceService, nodeEngine);
    }
}