Search in sources :

Example 1 with Mapper

use of com.hazelcast.mapreduce.Mapper in project hazelcast by hazelcast.

the class ObjectMultiMapProxy method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    try {
        isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMultiMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory != null) {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        throw new HazelcastException(e);
    }
}
Also used : MappingJob(com.hazelcast.mapreduce.MappingJob) HazelcastException(com.hazelcast.core.HazelcastException) ReducerFactory(com.hazelcast.mapreduce.ReducerFactory) HazelcastException(com.hazelcast.core.HazelcastException) Collator(com.hazelcast.mapreduce.Collator) CombinerFactory(com.hazelcast.mapreduce.CombinerFactory) Mapper(com.hazelcast.mapreduce.Mapper) ReducingSubmittableJob(com.hazelcast.mapreduce.ReducingSubmittableJob)

Example 2 with Mapper

use of com.hazelcast.mapreduce.Mapper in project hazelcast by hazelcast.

the class ClientMultiMapProxy method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    try {
        Preconditions.isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMultiMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory != null) {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        throw new HazelcastException(e);
    }
}
Also used : MappingJob(com.hazelcast.mapreduce.MappingJob) HazelcastException(com.hazelcast.core.HazelcastException) ReducerFactory(com.hazelcast.mapreduce.ReducerFactory) HazelcastException(com.hazelcast.core.HazelcastException) Collator(com.hazelcast.mapreduce.Collator) CombinerFactory(com.hazelcast.mapreduce.CombinerFactory) Mapper(com.hazelcast.mapreduce.Mapper) ReducingSubmittableJob(com.hazelcast.mapreduce.ReducingSubmittableJob)

Example 3 with Mapper

use of com.hazelcast.mapreduce.Mapper in project hazelcast by hazelcast.

the class AbstractMapReduceTask method startSupervisionTask.

private void startSupervisionTask(JobTracker jobTracker) {
    final MapReduceService mapReduceService = getService(MapReduceService.SERVICE_NAME);
    final JobTrackerConfig config = ((AbstractJobTracker) jobTracker).getJobTrackerConfig();
    final boolean communicateStats = config.isCommunicateStats();
    final int chunkSize = getChunkSizeOrConfigChunkSize(config);
    final TopologyChangedStrategy topologyChangedStrategy = getTopologyChangedStrategyOrConfigTopologyChangedStrategy(config);
    final String name = getDistributedObjectName();
    final String jobId = getJobId();
    final KeyValueSource keyValueSource = getKeyValueSource();
    final Mapper mapper = getMapper();
    final CombinerFactory combinerFactory = getCombinerFactory();
    final ReducerFactory reducerFactory = getReducerFactory();
    final Collection keys = getKeys();
    final Collection<Object> keyObjects = getKeyObjects(keys);
    final KeyPredicate predicate = getPredicate();
    final ClusterService clusterService = nodeEngine.getClusterService();
    for (Member member : clusterService.getMembers(KeyValueJobOperation.MEMBER_SELECTOR)) {
        Operation operation = new KeyValueJobOperation(name, jobId, chunkSize, keyValueSource, mapper, combinerFactory, reducerFactory, communicateStats, topologyChangedStrategy);
        executeOperation(operation, member.getAddress(), mapReduceService, nodeEngine);
    }
    // After we prepared all the remote systems we can now start the processing
    for (Member member : clusterService.getMembers(DATA_MEMBER_SELECTOR)) {
        Operation operation = new StartProcessingJobOperation(name, jobId, keyObjects, predicate);
        executeOperation(operation, member.getAddress(), mapReduceService, nodeEngine);
    }
}
Also used : JobTrackerConfig(com.hazelcast.config.JobTrackerConfig) KeyValueJobOperation(com.hazelcast.mapreduce.impl.operation.KeyValueJobOperation) ReducerFactory(com.hazelcast.mapreduce.ReducerFactory) AbstractJobTracker(com.hazelcast.mapreduce.impl.AbstractJobTracker) StartProcessingJobOperation(com.hazelcast.mapreduce.impl.operation.StartProcessingJobOperation) Operation(com.hazelcast.spi.Operation) MapReduceUtil.executeOperation(com.hazelcast.mapreduce.impl.MapReduceUtil.executeOperation) KeyValueJobOperation(com.hazelcast.mapreduce.impl.operation.KeyValueJobOperation) KeyPredicate(com.hazelcast.mapreduce.KeyPredicate) KeyValueSource(com.hazelcast.mapreduce.KeyValueSource) CombinerFactory(com.hazelcast.mapreduce.CombinerFactory) Mapper(com.hazelcast.mapreduce.Mapper) TopologyChangedStrategy(com.hazelcast.mapreduce.TopologyChangedStrategy) ClusterService(com.hazelcast.internal.cluster.ClusterService) MapReduceService(com.hazelcast.mapreduce.impl.MapReduceService) Collection(java.util.Collection) StartProcessingJobOperation(com.hazelcast.mapreduce.impl.operation.StartProcessingJobOperation) Member(com.hazelcast.core.Member)

Example 4 with Mapper

use of com.hazelcast.mapreduce.Mapper in project hazelcast by hazelcast.

the class MapProxyImpl method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    checkTrue(NATIVE != mapConfig.getInMemoryFormat(), "NATIVE storage format is not supported for MapReduce");
    try {
        isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory == null) {
            reducingJob = mappingJob.reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        // TODO: not what we want, because it can lead to wrapping of HazelcastException
        throw new HazelcastException(e);
    }
}
Also used : MappingJob(com.hazelcast.mapreduce.MappingJob) HazelcastException(com.hazelcast.core.HazelcastException) ReducerFactory(com.hazelcast.mapreduce.ReducerFactory) HazelcastException(com.hazelcast.core.HazelcastException) Collator(com.hazelcast.mapreduce.Collator) AggregationResult(com.hazelcast.map.impl.query.AggregationResult) QueryResult(com.hazelcast.map.impl.query.QueryResult) CombinerFactory(com.hazelcast.mapreduce.CombinerFactory) Mapper(com.hazelcast.mapreduce.Mapper) ReducingSubmittableJob(com.hazelcast.mapreduce.ReducingSubmittableJob)

Example 5 with Mapper

use of com.hazelcast.mapreduce.Mapper in project hazelcast by hazelcast.

the class ClientMapProxy method aggregate.

@Override
public <SuppliedValue, Result> Result aggregate(Supplier<K, V, SuppliedValue> supplier, Aggregation<K, SuppliedValue, Result> aggregation, JobTracker jobTracker) {
    try {
        Preconditions.isNotNull(jobTracker, "jobTracker");
        KeyValueSource<K, V> keyValueSource = KeyValueSource.fromMap(this);
        Job<K, V> job = jobTracker.newJob(keyValueSource);
        Mapper mapper = aggregation.getMapper(supplier);
        CombinerFactory combinerFactory = aggregation.getCombinerFactory();
        ReducerFactory reducerFactory = aggregation.getReducerFactory();
        Collator collator = aggregation.getCollator();
        MappingJob mappingJob = job.mapper(mapper);
        ReducingSubmittableJob reducingJob;
        if (combinerFactory != null) {
            reducingJob = mappingJob.combiner(combinerFactory).reducer(reducerFactory);
        } else {
            reducingJob = mappingJob.reducer(reducerFactory);
        }
        ICompletableFuture<Result> future = reducingJob.submit(collator);
        return future.get();
    } catch (Exception e) {
        throw new HazelcastException(e);
    }
}
Also used : MappingJob(com.hazelcast.mapreduce.MappingJob) HazelcastException(com.hazelcast.core.HazelcastException) ReducerFactory(com.hazelcast.mapreduce.ReducerFactory) HazelcastException(com.hazelcast.core.HazelcastException) Collator(com.hazelcast.mapreduce.Collator) CombinerFactory(com.hazelcast.mapreduce.CombinerFactory) Mapper(com.hazelcast.mapreduce.Mapper) ReducingSubmittableJob(com.hazelcast.mapreduce.ReducingSubmittableJob)

Aggregations

CombinerFactory (com.hazelcast.mapreduce.CombinerFactory)5 Mapper (com.hazelcast.mapreduce.Mapper)5 ReducerFactory (com.hazelcast.mapreduce.ReducerFactory)5 HazelcastException (com.hazelcast.core.HazelcastException)4 Collator (com.hazelcast.mapreduce.Collator)4 MappingJob (com.hazelcast.mapreduce.MappingJob)4 ReducingSubmittableJob (com.hazelcast.mapreduce.ReducingSubmittableJob)4 JobTrackerConfig (com.hazelcast.config.JobTrackerConfig)1 Member (com.hazelcast.core.Member)1 ClusterService (com.hazelcast.internal.cluster.ClusterService)1 AggregationResult (com.hazelcast.map.impl.query.AggregationResult)1 QueryResult (com.hazelcast.map.impl.query.QueryResult)1 KeyPredicate (com.hazelcast.mapreduce.KeyPredicate)1 KeyValueSource (com.hazelcast.mapreduce.KeyValueSource)1 TopologyChangedStrategy (com.hazelcast.mapreduce.TopologyChangedStrategy)1 AbstractJobTracker (com.hazelcast.mapreduce.impl.AbstractJobTracker)1 MapReduceService (com.hazelcast.mapreduce.impl.MapReduceService)1 MapReduceUtil.executeOperation (com.hazelcast.mapreduce.impl.MapReduceUtil.executeOperation)1 KeyValueJobOperation (com.hazelcast.mapreduce.impl.operation.KeyValueJobOperation)1 StartProcessingJobOperation (com.hazelcast.mapreduce.impl.operation.StartProcessingJobOperation)1