Examples with ResultSubpartitionView org.apache.flink.runtime.io.network.partition.ResultSubpartitionView used on opensource projects

Example 6 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannel method requestSubpartition.

// ------------------------------------------------------------------------
// Consume
// ------------------------------------------------------------------------
@Override
void requestSubpartition(int subpartitionIndex) throws IOException, InterruptedException {
    boolean retriggerRequest = false;
    // The lock is required to request only once in the presence of retriggered requests.
    synchronized (requestLock) {
        checkState(!isReleased, "LocalInputChannel has been released already");
        if (subpartitionView == null) {
            LOG.debug("{}: Requesting LOCAL subpartition {} of partition {}.", this, subpartitionIndex, partitionId);
            try {
                ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView(partitionId, subpartitionIndex, inputGate.getBufferProvider(), this);
                if (subpartitionView == null) {
                    throw new IOException("Error requesting subpartition.");
                }
                // make the subpartition view visible
                this.subpartitionView = subpartitionView;
                // check if the channel was released in the meantime
                if (isReleased) {
                    subpartitionView.releaseAllResources();
                    this.subpartitionView = null;
                }
            } catch (PartitionNotFoundException notFound) {
                if (increaseBackoff()) {
                    retriggerRequest = true;
                } else {
                    throw notFound;
                }
            }
        }
    }
    // input gate.
    if (retriggerRequest) {
        inputGate.retriggerPartitionRequest(partitionId.getPartitionId());
    }
}

Example 7 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannel method releaseAllResources.

/**
	 * Releases the partition reader
	 */
@Override
void releaseAllResources() throws IOException {
    if (!isReleased) {
        isReleased = true;
        ResultSubpartitionView view = subpartitionView;
        if (view != null) {
            view.releaseAllResources();
            subpartitionView = null;
        }
    }
}

Example 8 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannel method getNextBuffer.

@Override
BufferAndAvailability getNextBuffer() throws IOException, InterruptedException {
    checkError();
    ResultSubpartitionView subpartitionView = this.subpartitionView;
    if (subpartitionView == null) {
        // this can happen if the request for the partition was triggered asynchronously
        // by the time trigger
        // would be good to avoid that, by guaranteeing that the requestPartition() and
        // getNextBuffer() always come from the same thread
        // we could do that by letting the timer insert a special "requesting channel" into the input gate's queue
        subpartitionView = checkAndWaitForSubpartitionView();
    }
    Buffer next = subpartitionView.getNextBuffer();
    if (next == null) {
        if (subpartitionView.isReleased()) {
            throw new CancelTaskException("Consumed partition " + subpartitionView + " has been released.");
        } else {
            // notifications.
            throw new IllegalStateException("Consumed partition has no buffers available. " + "Number of received buffer notifications is " + numBuffersAvailable + ".");
        }
    }
    long remaining = numBuffersAvailable.decrementAndGet();
    if (remaining >= 0) {
        numBytesIn.inc(next.getSize());
        return new BufferAndAvailability(next, remaining > 0);
    } else if (subpartitionView.isReleased()) {
        throw new ProducerFailedException(subpartitionView.getFailureCause());
    } else {
        throw new IllegalStateException("No buffer available and producer partition not released.");
    }
}

Example 9 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class CancelPartitionRequestTest method testCancelPartitionRequest.

/**
	 * Verifies that requests for non-existing (failed/cancelled) input channels are properly
	 * cancelled. The receiver receives data, but there is no input channel to receive the data.
	 * This should cancel the request.
	 */
@Test
public void testCancelPartitionRequest() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferProvider.class), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[3];
                listener.notifyBuffersAvailable(Long.MAX_VALUE);
                return view;
            }
        });
        PartitionRequestProtocol protocol = new PartitionRequestProtocol(partitions, mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        ch.writeAndFlush(new PartitionRequest(pid, 0, new InputChannelID())).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() + " ms to be notified about cancelled partition.");
        }
        verify(view, times(1)).releaseAllResources();
        verify(view, times(0)).notifySubpartitionConsumed();
    } finally {
        shutdown(serverAndClient);
    }
}

Example 10 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class PartitionRequestQueueTest method testProducerFailedException.

@Test
public void testProducerFailedException() throws Exception {
    PartitionRequestQueue queue = new PartitionRequestQueue();
    ResultPartitionProvider partitionProvider = mock(ResultPartitionProvider.class);
    ResultPartitionID rpid = new ResultPartitionID();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    ResultSubpartitionView view = mock(ResultSubpartitionView.class);
    when(view.isReleased()).thenReturn(true);
    when(view.getFailureCause()).thenReturn(new RuntimeException("Expected test exception"));
    when(partitionProvider.createSubpartitionView(eq(rpid), eq(0), eq(bufferProvider), any(BufferAvailabilityListener.class))).thenReturn(view);
    EmbeddedChannel ch = new EmbeddedChannel(queue);
    SequenceNumberingViewReader seqView = new SequenceNumberingViewReader(new InputChannelID(), queue);
    seqView.requestSubpartitionView(partitionProvider, rpid, 0, bufferProvider);
    // Enqueue the erroneous view
    queue.notifyReaderNonEmpty(seqView);
    ch.runPendingTasks();
    // Read the enqueued msg
    Object msg = ch.readOutbound();
    assertEquals(msg.getClass(), NettyMessage.ErrorResponse.class);
    NettyMessage.ErrorResponse err = (NettyMessage.ErrorResponse) msg;
    assertTrue(err.cause instanceof CancelTaskException);
}