Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 1 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SequenceNumberingViewReader method getNextBuffer.

public BufferAndAvailability getNextBuffer() throws IOException, InterruptedException {
    Buffer next = subpartitionView.getNextBuffer();
    if (next != null) {
        long remaining = numBuffersAvailable.decrementAndGet();
        sequenceNumber++;
        if (remaining >= 0) {
            return new BufferAndAvailability(next, remaining > 0);
        } else {
            throw new IllegalStateException("no buffer available");
        }
    } else {
        return null;
    }
}

Example 2 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpillableSubpartition method release.

@Override
public void release() throws IOException {
    final ResultSubpartitionView view;
    synchronized (buffers) {
        if (isReleased) {
            return;
        }
        view = readView;
        // below).
        if (view == null) {
            for (Buffer buffer : buffers) {
                buffer.recycle();
            }
            buffers.clear();
            // which can bring down the network.
            if (spillWriter != null) {
                spillWriter.closeAndDelete();
            }
        }
        isReleased = true;
    }
    if (view != null) {
        view.releaseAllResources();
    }
}

Example 3 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpillableSubpartitionView method releaseMemory.

int releaseMemory() throws IOException {
    synchronized (buffers) {
        if (spilledView != null || nextBuffer == null) {
            // Already spilled or nothing in-memory
            return 0;
        } else {
            // We don't touch next buffer, because a notification has
            // already been sent for it. Only when it is consumed, will
            // it be recycled.
            // Create the spill writer and write all buffers to disk
            BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
            long spilledBytes = 0;
            int numBuffers = buffers.size();
            for (int i = 0; i < numBuffers; i++) {
                Buffer buffer = buffers.remove();
                spilledBytes += buffer.getSize();
                try {
                    spillWriter.writeBlock(buffer);
                } finally {
                    buffer.recycle();
                }
            }
            spilledView = new SpilledSubpartitionView(parent, memorySegmentSize, spillWriter, numBuffers, listener);
            LOG.debug("Spilling {} bytes for sub partition {} of {}.", spilledBytes, parent.index, parent.parent.getPartitionId());
            return numBuffers;
        }
    }
}

Example 4 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SingleInputGate method getNextBufferOrEvent.

// ------------------------------------------------------------------------
// Consume
// ------------------------------------------------------------------------
@Override
public BufferOrEvent getNextBufferOrEvent() throws IOException, InterruptedException {
    if (hasReceivedAllEndOfPartitionEvents) {
        return null;
    }
    if (isReleased) {
        throw new IllegalStateException("Released");
    }
    requestPartitions();
    InputChannel currentChannel;
    boolean moreAvailable;
    synchronized (inputChannelsWithData) {
        while (inputChannelsWithData.size() == 0) {
            if (isReleased) {
                throw new IllegalStateException("Released");
            }
            inputChannelsWithData.wait();
        }
        currentChannel = inputChannelsWithData.remove();
        moreAvailable = inputChannelsWithData.size() > 0;
    }
    final BufferAndAvailability result = currentChannel.getNextBuffer();
    // Sanity check that notifications only happen when data is available
    if (result == null) {
        throw new IllegalStateException("Bug in input gate/channel logic: input gate got " + "notified by channel about available data, but none was available.");
    }
    // will come for that channel
    if (result.moreAvailable()) {
        queueChannel(currentChannel);
    }
    final Buffer buffer = result.buffer();
    if (buffer.isBuffer()) {
        return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
    } else {
        final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
        if (event.getClass() == EndOfPartitionEvent.class) {
            channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());
            if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
                hasReceivedAllEndOfPartitionEvents = true;
            }
            currentChannel.notifySubpartitionConsumed();
            currentChannel.releaseAllResources();
        }
        return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable);
    }
}

Example 5 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpillingAdaptiveSpanningRecordDeserializer method getCurrentBuffer.

@Override
public Buffer getCurrentBuffer() {
    Buffer tmp = currentBuffer;
    currentBuffer = null;
    return tmp;
}