Examples with HadoopDirectDataInput org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataInput used on opensource projects

Example 1 with HadoopDirectDataInput

use of org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataInput in project ignite by apache.

the class HadoopDataStreamSelfTest method readLineByHadoopDirectDataInput.

/**
     * @param val String value.
     * @return List of strings are returned by readLine().
     * @throws IOException On error.
     */
List<String> readLineByHadoopDirectDataInput(String val) throws IOException {
    HadoopDirectDataOutput out = new HadoopDirectDataOutput(BUFF_SIZE);
    out.write(val.getBytes());
    byte[] inBuf = Arrays.copyOf(out.buffer(), out.position());
    HadoopDirectDataInput in = new HadoopDirectDataInput(inBuf);
    return readLineStrings(in);
}

Example 2 with HadoopDirectDataInput

use of org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataInput in project ignite by apache.

the class HadoopDataStreamSelfTest method testDirectStreams.

/**
     * @throws IOException If failed.
     */
public void testDirectStreams() throws IOException {
    HadoopDirectDataOutput out = new HadoopDirectDataOutput(BUFF_SIZE);
    write(out);
    byte[] inBuf = Arrays.copyOf(out.buffer(), out.position());
    HadoopDirectDataInput in = new HadoopDirectDataInput(inBuf);
    checkRead(in);
}

Example 3 with HadoopDirectDataInput

use of org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataInput in project ignite by apache.

the class HadoopShuffleJob method onDirectShuffleMessage.

/**
     * Process shuffle message.
     *
     * @param src Source.
     * @param msg Message.
     * @throws IgniteCheckedException Exception.
     */
public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
    byte[] buf = extractBuffer(msg);
    assert buf != null;
    int rdc = msg.reducer();
    HadoopTaskContext taskCtx = locReducersCtx.get(rdc).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(rdc, U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, rdc);
    HadoopSerialization keySer = taskCtx.keySerialization();
    HadoopSerialization valSer = taskCtx.valueSerialization();
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        HadoopDirectDataInput in = new HadoopDirectDataInput(buf);
        Object key = null;
        Object val = null;
        for (int i = 0; i < msg.count(); i++) {
            key = keySer.read(in, key);
            val = valSer.read(in, val);
            adder.write(key, val);
        }
    }
    if (localShuffleState(src).onShuffleMessage())
        sendFinishResponse(src, msg.jobId());
}