Examples with DataType org.apache.flink.runtime.io.network.buffer.Buffer.DataType used on opensource projects

Example 1 with DataType

use of org.apache.flink.runtime.io.network.buffer.Buffer.DataType in project flink by apache.

the class RemoteInputChannel method getNextBuffer.

@Override
Optional<BufferAndAvailability> getNextBuffer() throws IOException {
    checkPartitionRequestQueueInitialized();
    final SequenceBuffer next;
    final DataType nextDataType;
    synchronized (receivedBuffers) {
        next = receivedBuffers.poll();
        if (next != null) {
            totalQueueSizeInBytes -= next.buffer.getSize();
        }
        nextDataType = receivedBuffers.peek() != null ? receivedBuffers.peek().buffer.getDataType() : DataType.NONE;
    }
    if (next == null) {
        if (isReleased.get()) {
            throw new CancelTaskException("Queried for a buffer after channel has been released.");
        }
        return Optional.empty();
    }
    NetworkActionsLogger.traceInput("RemoteInputChannel#getNextBuffer", next.buffer, inputGate.getOwningTaskName(), channelInfo, channelStatePersister, next.sequenceNumber);
    numBytesIn.inc(next.buffer.getSize());
    numBuffersIn.inc();
    return Optional.of(new BufferAndAvailability(next.buffer, nextDataType, 0, next.sequenceNumber));
}

Example 2 with DataType

use of org.apache.flink.runtime.io.network.buffer.Buffer.DataType in project flink by apache.

the class RemoteInputChannel method onBuffer.

/**
 * Handles the input buffer. This method is taking over the ownership of the buffer and is fully
 * responsible for cleaning it up both on the happy path and in case of an error.
 */
public void onBuffer(Buffer buffer, int sequenceNumber, int backlog) throws IOException {
    boolean recycleBuffer = true;
    try {
        if (expectedSequenceNumber != sequenceNumber) {
            onError(new BufferReorderingException(expectedSequenceNumber, sequenceNumber));
            return;
        }
        if (buffer.getDataType().isBlockingUpstream()) {
            onBlockingUpstream();
            checkArgument(backlog == 0, "Illegal number of backlog: %s, should be 0.", backlog);
        }
        final boolean wasEmpty;
        boolean firstPriorityEvent = false;
        synchronized (receivedBuffers) {
            NetworkActionsLogger.traceInput("RemoteInputChannel#onBuffer", buffer, inputGate.getOwningTaskName(), channelInfo, channelStatePersister, sequenceNumber);
            // (see above for details).
            if (isReleased.get()) {
                return;
            }
            wasEmpty = receivedBuffers.isEmpty();
            SequenceBuffer sequenceBuffer = new SequenceBuffer(buffer, sequenceNumber);
            DataType dataType = buffer.getDataType();
            if (dataType.hasPriority()) {
                firstPriorityEvent = addPriorityBuffer(sequenceBuffer);
                recycleBuffer = false;
            } else {
                receivedBuffers.add(sequenceBuffer);
                recycleBuffer = false;
                if (dataType.requiresAnnouncement()) {
                    firstPriorityEvent = addPriorityBuffer(announce(sequenceBuffer));
                }
            }
            totalQueueSizeInBytes += buffer.getSize();
            final OptionalLong barrierId = channelStatePersister.checkForBarrier(sequenceBuffer.buffer);
            if (barrierId.isPresent() && barrierId.getAsLong() > lastBarrierId) {
                // checkpoint was not yet started by task thread,
                // so remember the numbers of buffers to spill for the time when
                // it will be started
                lastBarrierId = barrierId.getAsLong();
                lastBarrierSequenceNumber = sequenceBuffer.sequenceNumber;
            }
            channelStatePersister.maybePersist(buffer);
            ++expectedSequenceNumber;
        }
        if (firstPriorityEvent) {
            notifyPriorityEvent(sequenceNumber);
        }
        if (wasEmpty) {
            notifyChannelNonEmpty();
        }
        if (backlog >= 0) {
            onSenderBacklog(backlog);
        }
    } finally {
        if (recycleBuffer) {
            buffer.recycleBuffer();
        }
    }
}

Example 3 with DataType

use of org.apache.flink.runtime.io.network.buffer.Buffer.DataType in project flink by apache.

the class SortBasedDataBuffer method getNextBuffer.

@Override
public BufferWithChannel getNextBuffer(MemorySegment transitBuffer) {
    checkState(isFull, "Sort buffer is not ready to be read.");
    checkState(!isReleased, "Sort buffer is already released.");
    if (!hasRemaining()) {
        return null;
    }
    int numBytesCopied = 0;
    DataType bufferDataType = DataType.DATA_BUFFER;
    int channelIndex = subpartitionReadOrder[readOrderIndex];
    do {
        int sourceSegmentIndex = getSegmentIndexFromPointer(readIndexEntryAddress);
        int sourceSegmentOffset = getSegmentOffsetFromPointer(readIndexEntryAddress);
        MemorySegment sourceSegment = segments.get(sourceSegmentIndex);
        long lengthAndDataType = sourceSegment.getLong(sourceSegmentOffset);
        int length = getSegmentIndexFromPointer(lengthAndDataType);
        DataType dataType = DataType.values()[getSegmentOffsetFromPointer(lengthAndDataType)];
        // return the data read directly if the next to read is an event
        if (dataType.isEvent() && numBytesCopied > 0) {
            break;
        }
        bufferDataType = dataType;
        // get the next index entry address and move the read position forward
        long nextReadIndexEntryAddress = sourceSegment.getLong(sourceSegmentOffset + 8);
        sourceSegmentOffset += INDEX_ENTRY_SIZE;
        // allocate a temp buffer for the event if the target buffer is not big enough
        if (bufferDataType.isEvent() && transitBuffer.size() < length) {
            transitBuffer = MemorySegmentFactory.allocateUnpooledSegment(length);
        }
        numBytesCopied += copyRecordOrEvent(transitBuffer, numBytesCopied, sourceSegmentIndex, sourceSegmentOffset, length);
        if (recordRemainingBytes == 0) {
            // move to next channel if the current channel has been finished
            if (readIndexEntryAddress == lastIndexEntryAddresses[channelIndex]) {
                updateReadChannelAndIndexEntryAddress();
                break;
            }
            readIndexEntryAddress = nextReadIndexEntryAddress;
        }
    } while (numBytesCopied < transitBuffer.size() && bufferDataType.isBuffer());
    numTotalBytesRead += numBytesCopied;
    Buffer buffer = new NetworkBuffer(transitBuffer, (buf) -> {
    }, bufferDataType, numBytesCopied);
    return new BufferWithChannel(buffer, channelIndex);
}

Example 4 with DataType

use of org.apache.flink.runtime.io.network.buffer.Buffer.DataType in project flink by apache.

the class DataBufferTest method testWriteAndReadDataBuffer.

@Test
public void testWriteAndReadDataBuffer() throws Exception {
    int numSubpartitions = 10;
    int bufferSize = 1024;
    int bufferPoolSize = 512;
    Random random = new Random(1111);
    // used to store data written to and read from sort buffer for correctness check
    Queue<DataAndType>[] dataWritten = new Queue[numSubpartitions];
    Queue<Buffer>[] buffersRead = new Queue[numSubpartitions];
    for (int i = 0; i < numSubpartitions; ++i) {
        dataWritten[i] = new ArrayDeque<>();
        buffersRead[i] = new ArrayDeque<>();
    }
    int[] numBytesWritten = new int[numSubpartitions];
    int[] numBytesRead = new int[numSubpartitions];
    Arrays.fill(numBytesWritten, 0);
    Arrays.fill(numBytesRead, 0);
    // fill the sort buffer with randomly generated data
    int totalBytesWritten = 0;
    DataBuffer dataBuffer = createDataBuffer(bufferPoolSize, bufferSize, numSubpartitions, getRandomSubpartitionOrder(numSubpartitions));
    int numDataBuffers = 5;
    while (numDataBuffers > 0) {
        // record size may be larger than buffer size so a record may span multiple segments
        int recordSize = random.nextInt(bufferSize * 4 - 1) + 1;
        byte[] bytes = new byte[recordSize];
        // fill record with random value
        random.nextBytes(bytes);
        ByteBuffer record = ByteBuffer.wrap(bytes);
        // select a random subpartition to write
        int subpartition = random.nextInt(numSubpartitions);
        // select a random data type
        boolean isBuffer = random.nextBoolean();
        DataType dataType = isBuffer ? DataType.DATA_BUFFER : DataType.EVENT_BUFFER;
        boolean isFull = dataBuffer.append(record, subpartition, dataType);
        record.flip();
        if (record.hasRemaining()) {
            dataWritten[subpartition].add(new DataAndType(record, dataType));
            numBytesWritten[subpartition] += record.remaining();
            totalBytesWritten += record.remaining();
        }
        while (isFull && dataBuffer.hasRemaining()) {
            BufferWithChannel buffer = copyIntoSegment(bufferSize, dataBuffer);
            if (buffer == null) {
                break;
            }
            addBufferRead(buffer, buffersRead, numBytesRead);
        }
        if (isFull) {
            --numDataBuffers;
            dataBuffer.reset();
        }
    }
    // read all data from the sort buffer
    if (dataBuffer.hasRemaining()) {
        assertTrue(dataBuffer instanceof HashBasedDataBuffer);
        dataBuffer.reset();
        dataBuffer.finish();
        while (dataBuffer.hasRemaining()) {
            addBufferRead(copyIntoSegment(bufferSize, dataBuffer), buffersRead, numBytesRead);
        }
    }
    assertEquals(totalBytesWritten, dataBuffer.numTotalBytes());
    checkWriteReadResult(numSubpartitions, numBytesWritten, numBytesRead, dataWritten, buffersRead);
}