Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 21 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class BufferSpiller method add.

/**
	 * Adds a buffer or event to the sequence of spilled buffers and events.
	 * 
	 * @param boe The buffer or event to add and spill.
	 * @throws IOException Thrown, if the buffer of event could not be spilled.
	 */
public void add(BufferOrEvent boe) throws IOException {
    try {
        ByteBuffer contents;
        if (boe.isBuffer()) {
            Buffer buf = boe.getBuffer();
            contents = buf.getMemorySegment().wrap(0, buf.getSize());
        } else {
            contents = EventSerializer.toSerializedEvent(boe.getEvent());
        }
        headBuffer.clear();
        headBuffer.putInt(boe.getChannelIndex());
        headBuffer.putInt(contents.remaining());
        headBuffer.put((byte) (boe.isBuffer() ? 0 : 1));
        headBuffer.flip();
        bytesWritten += (headBuffer.remaining() + contents.remaining());
        sources[1] = contents;
        currentChannel.write(sources);
    } finally {
        if (boe.isBuffer()) {
            boe.getBuffer().recycle();
        }
    }
}

Example 22 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class StreamTestSingleInputGate method setupInputChannels.

@SuppressWarnings("unchecked")
private void setupInputChannels() throws IOException, InterruptedException {
    for (int i = 0; i < numInputChannels; i++) {
        final int channelIndex = i;
        final RecordSerializer<SerializationDelegate<Object>> recordSerializer = new SpanningRecordSerializer<SerializationDelegate<Object>>();
        final SerializationDelegate<Object> delegate = (SerializationDelegate<Object>) (SerializationDelegate<?>) new SerializationDelegate<StreamElement>(new StreamElementSerializer<T>(serializer));
        inputQueues[channelIndex] = new ConcurrentLinkedQueue<InputValue<Object>>();
        inputChannels[channelIndex] = new TestInputChannel(inputGate, i);
        final Answer<BufferAndAvailability> answer = new Answer<BufferAndAvailability>() {

            @Override
            public BufferAndAvailability answer(InvocationOnMock invocationOnMock) throws Throwable {
                InputValue<Object> input = inputQueues[channelIndex].poll();
                if (input != null && input.isStreamEnd()) {
                    when(inputChannels[channelIndex].getInputChannel().isReleased()).thenReturn(true);
                    return new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE), false);
                } else if (input != null && input.isStreamRecord()) {
                    Object inputElement = input.getStreamRecord();
                    final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(bufferSize), mock(BufferRecycler.class));
                    recordSerializer.setNextBuffer(buffer);
                    delegate.setInstance(inputElement);
                    recordSerializer.addRecord(delegate);
                    // Call getCurrentBuffer to ensure size is set
                    return new BufferAndAvailability(recordSerializer.getCurrentBuffer(), false);
                } else if (input != null && input.isEvent()) {
                    AbstractEvent event = input.getEvent();
                    return new BufferAndAvailability(EventSerializer.toBuffer(event), false);
                } else {
                    synchronized (inputQueues[channelIndex]) {
                        inputQueues[channelIndex].wait();
                        return answer(invocationOnMock);
                    }
                }
            }
        };
        when(inputChannels[channelIndex].getInputChannel().getNextBuffer()).thenAnswer(answer);
        inputGate.setInputChannel(new IntermediateResultPartitionID(), inputChannels[channelIndex].getInputChannel());
    }
}

Example 23 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SequenceNumberingViewReader method getNextBuffer.

public BufferAndAvailability getNextBuffer() throws IOException, InterruptedException {
    Buffer next = subpartitionView.getNextBuffer();
    if (next != null) {
        long remaining = numBuffersAvailable.decrementAndGet();
        sequenceNumber++;
        if (remaining >= 0) {
            return new BufferAndAvailability(next, remaining > 0);
        } else {
            throw new IllegalStateException("no buffer available");
        }
    } else {
        return null;
    }
}

Example 24 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PipelinedSubpartition method release.

@Override
public void release() {
    // view reference accessible outside the lock, but assigned inside the locked scope
    final PipelinedSubpartitionView view;
    synchronized (buffers) {
        if (isReleased) {
            return;
        }
        // Release all available buffers
        Buffer buffer;
        while ((buffer = buffers.poll()) != null) {
            buffer.recycle();
        }
        // Get the view...
        view = readView;
        readView = null;
        // Make sure that no further buffers are added to the subpartition
        isReleased = true;
    }
    LOG.debug("Released {}.", this);
    // Release all resources of the view
    if (view != null) {
        view.releaseAllResources();
    }
}

Example 25 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SpillableSubpartition method release.

@Override
public void release() throws IOException {
    final ResultSubpartitionView view;
    synchronized (buffers) {
        if (isReleased) {
            return;
        }
        view = readView;
        // below).
        if (view == null) {
            for (Buffer buffer : buffers) {
                buffer.recycle();
            }
            buffers.clear();
            // which can bring down the network.
            if (spillWriter != null) {
                spillWriter.closeAndDelete();
            }
        }
        isReleased = true;
    }
    if (view != null) {
        view.releaseAllResources();
    }
}