Examples with HadoopPerformanceCounter org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter used on opensource projects

Example 6 with HadoopPerformanceCounter

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter in project ignite by apache.

the class HadoopShuffleJob method onDirectShuffleMessage.

/**
 * Process shuffle message.
 *
 * @param src Source.
 * @param msg Message.
 * @throws IgniteCheckedException Exception.
 */
public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
    byte[] buf = extractBuffer(msg);
    assert buf != null;
    int rdc = msg.reducer();
    HadoopTaskContext taskCtx = locReducersCtx.get(rdc).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(rdc, U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, rdc);
    HadoopSerialization keySer = taskCtx.keySerialization();
    HadoopSerialization valSer = taskCtx.valueSerialization();
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        HadoopDirectDataInput in = new HadoopDirectDataInput(buf);
        Object key = null;
        Object val = null;
        for (int i = 0; i < msg.count(); i++) {
            key = keySer.read(in, key);
            val = valSer.read(in, val);
            adder.write(key, val);
        }
    }
    if (localShuffleState(src).onShuffleMessage())
        sendFinishResponse(src, msg.jobId());
}