Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 26 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpillableSubpartitionView method releaseMemory.

int releaseMemory() throws IOException {
    synchronized (buffers) {
        if (spilledView != null || nextBuffer == null) {
            // Already spilled or nothing in-memory
            return 0;
        } else {
            // We don't touch next buffer, because a notification has
            // already been sent for it. Only when it is consumed, will
            // it be recycled.
            // Create the spill writer and write all buffers to disk
            BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
            long spilledBytes = 0;
            int numBuffers = buffers.size();
            for (int i = 0; i < numBuffers; i++) {
                Buffer buffer = buffers.remove();
                spilledBytes += buffer.getSize();
                try {
                    spillWriter.writeBlock(buffer);
                } finally {
                    buffer.recycle();
                }
            }
            spilledView = new SpilledSubpartitionView(parent, memorySegmentSize, spillWriter, numBuffers, listener);
            LOG.debug("Spilling {} bytes for sub partition {} of {}.", spilledBytes, parent.index, parent.parent.getPartitionId());
            return numBuffers;
        }
    }
}

Example 27 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RemoteInputChannel method getNextBuffer.

@Override
BufferAndAvailability getNextBuffer() throws IOException {
    checkState(!isReleased.get(), "Queried for a buffer after channel has been closed.");
    checkState(partitionRequestClient != null, "Queried for a buffer before requesting a queue.");
    checkError();
    final Buffer next;
    final int remaining;
    synchronized (receivedBuffers) {
        next = receivedBuffers.poll();
        remaining = receivedBuffers.size();
    }
    numBytesIn.inc(next.getSize());
    return new BufferAndAvailability(next, remaining > 0);
}

Example 28 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SingleInputGate method getNextBufferOrEvent.

// ------------------------------------------------------------------------
// Consume
// ------------------------------------------------------------------------
@Override
public BufferOrEvent getNextBufferOrEvent() throws IOException, InterruptedException {
    if (hasReceivedAllEndOfPartitionEvents) {
        return null;
    }
    if (isReleased) {
        throw new IllegalStateException("Released");
    }
    requestPartitions();
    InputChannel currentChannel;
    boolean moreAvailable;
    synchronized (inputChannelsWithData) {
        while (inputChannelsWithData.size() == 0) {
            if (isReleased) {
                throw new IllegalStateException("Released");
            }
            inputChannelsWithData.wait();
        }
        currentChannel = inputChannelsWithData.remove();
        moreAvailable = inputChannelsWithData.size() > 0;
    }
    final BufferAndAvailability result = currentChannel.getNextBuffer();
    // Sanity check that notifications only happen when data is available
    if (result == null) {
        throw new IllegalStateException("Bug in input gate/channel logic: input gate got " + "notified by channel about available data, but none was available.");
    }
    // will come for that channel
    if (result.moreAvailable()) {
        queueChannel(currentChannel);
    }
    final Buffer buffer = result.buffer();
    if (buffer.isBuffer()) {
        return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
    } else {
        final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
        if (event.getClass() == EndOfPartitionEvent.class) {
            channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());
            if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
                hasReceivedAllEndOfPartitionEvents = true;
            }
            currentChannel.notifySubpartitionConsumed();
            currentChannel.releaseAllResources();
        }
        return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable);
    }
}

Example 29 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpillingAdaptiveSpanningRecordDeserializer method getCurrentBuffer.

@Override
public Buffer getCurrentBuffer() {
    Buffer tmp = currentBuffer;
    currentBuffer = null;
    return tmp;
}

Example 30 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordWriter method flush.

public void flush() throws IOException {
    for (int targetChannel = 0; targetChannel < numChannels; targetChannel++) {
        RecordSerializer<T> serializer = serializers[targetChannel];
        synchronized (serializer) {
            try {
                Buffer buffer = serializer.getCurrentBuffer();
                if (buffer != null) {
                    numBytesOut.inc(buffer.getSize());
                    targetPartition.writeBuffer(buffer, targetChannel);
                }
            } finally {
                serializer.clear();
            }
        }
    }
}