Examples with LastChunkNotification com.hazelcast.mapreduce.impl.notification.LastChunkNotification used on opensource projects

Example 1 with LastChunkNotification

use of com.hazelcast.mapreduce.impl.notification.LastChunkNotification in project hazelcast by hazelcast.

the class JobSupervisor method onNotification.

public void onNotification(MapReduceNotification notification) {
    if (notification instanceof IntermediateChunkNotification) {
        IntermediateChunkNotification icn = (IntermediateChunkNotification) notification;
        ReducerTask reducerTask = jobTracker.getReducerTask(icn.getJobId());
        reducerTask.processChunk(icn.getChunk());
    } else if (notification instanceof LastChunkNotification) {
        LastChunkNotification lcn = (LastChunkNotification) notification;
        ReducerTask reducerTask = jobTracker.getReducerTask(lcn.getJobId());
        reducerTask.processChunk(lcn.getPartitionId(), lcn.getSender(), lcn.getChunk());
    } else if (notification instanceof ReducingFinishedNotification) {
        final ReducingFinishedNotification rfn = (ReducingFinishedNotification) notification;
        // Just offload it to free the event queue
        executorService.submit(new Runnable() {

            @Override
            public void run() {
                processReducerFinished0(rfn);
            }
        });
    }
}

Example 2 with LastChunkNotification

use of com.hazelcast.mapreduce.impl.notification.LastChunkNotification in project hazelcast by hazelcast.

the class MapCombineTask method sendLastChunkToAssignedReducers.

private void sendLastChunkToAssignedReducers(int partitionId, Map<KeyOut, Chunk> chunkMap) {
    Address sender = mapReduceService.getLocalAddress();
    // Wrap into LastChunkNotification object
    Map<Address, Map<KeyOut, Chunk>> mapping = mapResultToMember(supervisor, chunkMap);
    // Register remote addresses and partitionId for receiving reducer events
    supervisor.registerReducerEventInterests(partitionId, mapping.keySet());
    // Send LastChunk notifications
    for (Map.Entry<Address, Map<KeyOut, Chunk>> entry : mapping.entrySet()) {
        Address receiver = entry.getKey();
        Map<KeyOut, Chunk> chunk = entry.getValue();
        mapReduceService.sendNotification(receiver, new LastChunkNotification(receiver, name, jobId, sender, partitionId, chunk));
    }
    // Send LastChunk notification to notify reducers that received at least one chunk
    Set<Address> addresses = mapping.keySet();
    Collection<Address> reducerInterests = supervisor.getReducerEventInterests(partitionId);
    if (reducerInterests != null) {
        for (Address address : reducerInterests) {
            if (!addresses.contains(address)) {
                mapReduceService.sendNotification(address, new LastChunkNotification(address, name, jobId, sender, partitionId, Collections.emptyMap()));
            }
        }
    }
}

Example 3 with LastChunkNotification

use of com.hazelcast.mapreduce.impl.notification.LastChunkNotification in project hazelcast by hazelcast.

the class MapReduceDataSerializerHook method createFactory.

@Override
public DataSerializableFactory createFactory() {
    ConstructorFunction<Integer, IdentifiedDataSerializable>[] constructors = new ConstructorFunction[LEN];
    constructors[KEY_VALUE_SOURCE_MAP] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new MapKeyValueSource();
        }
    };
    constructors[KEY_VALUE_SOURCE_MULTIMAP] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new MultiMapKeyValueSource();
        }
    };
    constructors[REDUCER_CHUNK_MESSAGE] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new IntermediateChunkNotification();
        }
    };
    constructors[REDUCER_LAST_CHUNK_MESSAGE] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new LastChunkNotification();
        }
    };
    constructors[TRACKED_JOB_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new KeyValueJobOperation();
        }
    };
    constructors[REQUEST_PARTITION_MAPPING] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new RequestPartitionMapping();
        }
    };
    constructors[REQUEST_PARTITION_REDUCING] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new RequestPartitionReducing();
        }
    };
    constructors[REQUEST_PARTITION_PROCESSED] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new RequestPartitionProcessed();
        }
    };
    constructors[GET_RESULT_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new GetResultOperation();
        }
    };
    constructors[START_PROCESSING_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new StartProcessingJobOperation();
        }
    };
    constructors[REQUEST_PARTITION_RESULT] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new RequestPartitionResult();
        }
    };
    constructors[REDUCING_FINISHED_MESSAGE] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new ReducingFinishedNotification();
        }
    };
    constructors[FIRE_NOTIFICATION_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new FireNotificationOperation();
        }
    };
    constructors[REQUEST_MEMBERID_ASSIGNMENT] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new RequestMemberIdAssignment();
        }
    };
    constructors[PROCESS_STATS_UPDATE_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new ProcessStatsUpdateOperation();
        }
    };
    constructors[NOTIFY_REMOTE_EXCEPTION_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new NotifyRemoteExceptionOperation();
        }
    };
    constructors[CANCEL_JOB_SUPERVISOR_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new CancelJobSupervisorOperation();
        }
    };
    constructors[KEY_VALUE_SOURCE_LIST] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new ListKeyValueSource();
        }
    };
    constructors[KEY_VALUE_SOURCE_SET] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new SetKeyValueSource();
        }
    };
    constructors[KEYS_ASSIGNMENT_RESULT] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new KeysAssignmentResult();
        }
    };
    constructors[KEYS_ASSIGNMENT_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new KeysAssignmentOperation();
        }
    };
    constructors[POSTPONE_PARTITION_PROCESSING_OPERATION] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new PostPonePartitionProcessing();
        }
    };
    constructors[HASH_MAP_ADAPTER] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new HashMapAdapter();
        }
    };
    constructors[COMBINER_RESULT_LIST] = new ConstructorFunction<Integer, IdentifiedDataSerializable>() {

        @Override
        public IdentifiedDataSerializable createNew(Integer arg) {
            return new CombinerResultList();
        }
    };
    return new ArrayDataSerializableFactory(constructors);
}