Examples with HadoopMultimap org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap used on opensource projects

Example 6 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopShuffleJob method collectUpdatesAndSend.

/**
 * Send updates to concrete remote reducer.
 *
 * @param rmtMapIdx Remote map index.
 * @param flush Flush flag.
 * @throws IgniteCheckedException If failed.
 */
private void collectUpdatesAndSend(int rmtMapIdx, boolean flush) throws IgniteCheckedException {
    final int rmtRdcIdx = stripeMappers ? rmtMapIdx % totalReducerCnt : rmtMapIdx;
    HadoopMultimap map = rmtMaps.get(rmtMapIdx);
    if (map == null)
        return;
    if (msgs[rmtMapIdx] == null)
        msgs[rmtMapIdx] = new HadoopShuffleMessage(job.id(), rmtRdcIdx, msgSize);
    visit(map, rmtMapIdx, rmtRdcIdx);
    if (flush && msgs[rmtMapIdx].offset() != 0)
        send(rmtMapIdx, rmtRdcIdx, 0);
}

Example 7 with HadoopMultimap

use of org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap in project ignite by apache.

the class HadoopShuffleJob method onDirectShuffleMessage.

/**
 * Process shuffle message.
 *
 * @param src Source.
 * @param msg Message.
 * @throws IgniteCheckedException Exception.
 */
public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
    byte[] buf = extractBuffer(msg);
    assert buf != null;
    int rdc = msg.reducer();
    HadoopTaskContext taskCtx = locReducersCtx.get(rdc).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(rdc, U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, rdc);
    HadoopSerialization keySer = taskCtx.keySerialization();
    HadoopSerialization valSer = taskCtx.valueSerialization();
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        HadoopDirectDataInput in = new HadoopDirectDataInput(buf);
        Object key = null;
        Object val = null;
        for (int i = 0; i < msg.count(); i++) {
            key = keySer.read(in, key);
            val = valSer.read(in, val);
            adder.write(key, val);
        }
    }
    if (localShuffleState(src).onShuffleMessage())
        sendFinishResponse(src, msg.jobId());
}