Examples with ResultPartitionID org.apache.flink.runtime.io.network.partition.ResultPartitionID used on opensource projects

Example 6 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class LocalInputChannelTest method testGetNextAfterPartitionReleased.

/**
	 * Tests that reading from a channel when after the partition has been
	 * released are handled and don't lead to NPEs.
	 */
@Test
public void testGetNextAfterPartitionReleased() throws Exception {
    ResultSubpartitionView reader = mock(ResultSubpartitionView.class);
    SingleInputGate gate = mock(SingleInputGate.class);
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class))).thenReturn(reader);
    LocalInputChannel channel = new LocalInputChannel(gate, 0, new ResultPartitionID(), partitionManager, new TaskEventDispatcher(), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    channel.requestSubpartition(0);
    // Null buffer but not released
    when(reader.getNextBuffer()).thenReturn(null);
    when(reader.isReleased()).thenReturn(false);
    try {
        channel.getNextBuffer();
        fail("Did not throw expected IllegalStateException");
    } catch (IllegalStateException ignored) {
    }
    // Null buffer and released
    when(reader.getNextBuffer()).thenReturn(null);
    when(reader.isReleased()).thenReturn(true);
    try {
        channel.getNextBuffer();
        fail("Did not throw expected CancelTaskException");
    } catch (CancelTaskException ignored) {
    }
}

Example 7 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class LocalInputChannelTest method testConcurrentReleaseAndRetriggerPartitionRequest.

/**
	 * Verifies that concurrent release via the SingleInputGate and re-triggering
	 * of a partition request works smoothly.
	 *
	 * - SingleInputGate acquires its request lock and tries to release all
	 * registered channels. When releasing a channel, it needs to acquire
	 * the channel's shared request-release lock.
	 * - If a LocalInputChannel concurrently retriggers a partition request via
	 * a Timer Thread it acquires the channel's request-release lock and calls
	 * the retrigger callback on the SingleInputGate, which again tries to
	 * acquire the gate's request lock.
	 *
	 * For certain timings this obviously leads to a deadlock. This test reliably
	 * reproduced such a timing (reported in FLINK-5228). This test is pretty much
	 * testing the buggy implementation and has not much more general value. If it
	 * becomes obsolete at some point (future greatness ;)), feel free to remove it.
	 *
	 * The fix in the end was to to not acquire the channels lock when releasing it
	 * and/or not doing any input gate callbacks while holding the channel's lock.
	 * I decided to do both.
	 */
@Test
public void testConcurrentReleaseAndRetriggerPartitionRequest() throws Exception {
    final SingleInputGate gate = new SingleInputGate("test task name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 1, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

        @Override
        public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
            // Sleep here a little to give the releaser Thread
            // time to acquire the input gate lock. We throw
            // the Exception to retrigger the request.
            Thread.sleep(100);
            throw new PartitionNotFoundException(new ResultPartitionID());
        }
    });
    final LocalInputChannel channel = new LocalInputChannel(gate, 0, new ResultPartitionID(), partitionManager, new TaskEventDispatcher(), 1, 1, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    Thread releaser = new Thread() {

        @Override
        public void run() {
            try {
                gate.releaseAllResources();
            } catch (IOException ignored) {
            }
        }
    };
    Thread requester = new Thread() {

        @Override
        public void run() {
            try {
                channel.requestSubpartition(0);
            } catch (IOException | InterruptedException ignored) {
            }
        }
    };
    requester.start();
    releaser.start();
    releaser.join();
    requester.join();
}

Example 8 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class RemoteInputChannelTest method testProducerFailedException.

@Test(expected = CancelTaskException.class)
public void testProducerFailedException() throws Exception {
    ConnectionManager connManager = mock(ConnectionManager.class);
    when(connManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(mock(PartitionRequestClient.class));
    final RemoteInputChannel ch = new RemoteInputChannel(mock(SingleInputGate.class), 0, new ResultPartitionID(), mock(ConnectionID.class), connManager, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    ch.onError(new ProducerFailedException(new RuntimeException("Expected test exception.")));
    ch.requestSubpartition(0);
    // Should throw an instance of CancelTaskException.
    ch.getNextBuffer();
}

Example 9 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class RemoteInputChannelTest method createRemoteInputChannel.

private RemoteInputChannel createRemoteInputChannel(SingleInputGate inputGate, PartitionRequestClient partitionRequestClient, Tuple2<Integer, Integer> initialAndMaxRequestBackoff) throws IOException, InterruptedException {
    final ConnectionManager connectionManager = mock(ConnectionManager.class);
    when(connectionManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(partitionRequestClient);
    return new RemoteInputChannel(inputGate, 0, new ResultPartitionID(), mock(ConnectionID.class), connectionManager, initialAndMaxRequestBackoff._1(), initialAndMaxRequestBackoff._2(), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
}

Example 10 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class SingleInputGateTest method testBackwardsEventWithUninitializedChannel.

@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
    // Setup environment
    final TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
    when(taskEventDispatcher.publish(any(ResultPartitionID.class), any(TaskEvent.class))).thenReturn(true);
    final ResultSubpartitionView iterator = mock(ResultSubpartitionView.class);
    when(iterator.getNextBuffer()).thenReturn(new Buffer(MemorySegmentFactory.allocateUnpooledSegment(1024), mock(BufferRecycler.class)));
    final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class))).thenReturn(iterator);
    // Setup reader with one local and one unknown input channel
    final IntermediateDataSetID resultId = new IntermediateDataSetID();
    final SingleInputGate inputGate = new SingleInputGate("Test Task Name", new JobID(), resultId, ResultPartitionType.PIPELINED, 0, 2, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final BufferPool bufferPool = mock(BufferPool.class);
    when(bufferPool.getNumberOfRequiredMemorySegments()).thenReturn(2);
    inputGate.setBufferPool(bufferPool);
    // Local
    ResultPartitionID localPartitionId = new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID());
    InputChannel local = new LocalInputChannel(inputGate, 0, localPartitionId, partitionManager, taskEventDispatcher, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    // Unknown
    ResultPartitionID unknownPartitionId = new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID());
    InputChannel unknown = new UnknownInputChannel(inputGate, 1, unknownPartitionId, partitionManager, taskEventDispatcher, mock(ConnectionManager.class), 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    // Set channels
    inputGate.setInputChannel(localPartitionId.getPartitionId(), local);
    inputGate.setInputChannel(unknownPartitionId.getPartitionId(), unknown);
    // Request partitions
    inputGate.requestPartitions();
    // Only the local channel can request
    verify(partitionManager, times(1)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class));
    // Send event backwards and initialize unknown channel afterwards
    final TaskEvent event = new TestTaskEvent();
    inputGate.sendTaskEvent(event);
    // Only the local channel can send out the event
    verify(taskEventDispatcher, times(1)).publish(any(ResultPartitionID.class), any(TaskEvent.class));
    // After the update, the pending event should be send to local channel
    inputGate.updateInputChannel(new InputChannelDeploymentDescriptor(new ResultPartitionID(unknownPartitionId.getPartitionId(), unknownPartitionId.getProducerId()), ResultPartitionLocation.createLocal()));
    verify(partitionManager, times(2)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class));
    verify(taskEventDispatcher, times(2)).publish(any(ResultPartitionID.class), any(TaskEvent.class));
}