Search in sources :

Example 1 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.

@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
    ResourceID localLocation = ResourceID.generate();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
    for (InputChannel channel : gate.getInputChannels().values()) {
        if (channel instanceof ChannelStateHolder) {
            ((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
        }
    }
    SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
    SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
    SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
    SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
    SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
    SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
    assertThat(gate.getInputChannels().size(), is(6));
    assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
    assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    // test setup
    gate.setup();
    assertNotNull(gate.getBufferPool());
    assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    // test request partitions
    gate.requestPartitions();
    gate.pollNext();
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    for (InputChannel inputChannel : gate.getInputChannels().values()) {
        if (inputChannel instanceof RemoteInputChannel) {
            assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
            assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
        } else if (inputChannel instanceof LocalInputChannel) {
            assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
        }
    }
    // test update channels
    gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}
Also used : TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) SubpartitionInfo(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate.SubpartitionInfo) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) ChannelStateHolder(org.apache.flink.runtime.io.network.partition.ChannelStateHolder) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) NoOpResultSubpartitionView(org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) InputChannelTestUtils.createLocalInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel) InputChannelTestUtils.createLocalInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel) SubpartitionIndexRange(org.apache.flink.runtime.deployment.SubpartitionIndexRange) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) Test(org.junit.Test)

Example 2 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class SingleInputGateTest method testQueuedBuffers.

@Test
public void testQueuedBuffers() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final BufferWritingResultPartition resultPartition = (BufferWritingResultPartition) new ResultPartitionBuilder().setResultPartitionManager(network.getResultPartitionManager()).setupBufferPoolFactoryFromNettyShuffleEnvironment(network).build();
    final SingleInputGate inputGate = createInputGate(network, 2, ResultPartitionType.PIPELINED);
    final ResultPartitionID localResultPartitionId = resultPartition.getPartitionId();
    final InputChannel[] inputChannels = new InputChannel[2];
    final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setChannelIndex(1).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
    inputChannels[0] = remoteInputChannel;
    inputChannels[1] = InputChannelBuilder.newBuilder().setChannelIndex(0).setPartitionId(localResultPartitionId).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildLocalChannel(inputGate);
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        closer.register(resultPartition::release);
        resultPartition.setup();
        setupInputGate(inputGate, inputChannels);
        remoteInputChannel.onBuffer(createBuffer(1), 0, 0);
        assertEquals(1, inputGate.getNumberOfQueuedBuffers());
        resultPartition.emitRecord(ByteBuffer.allocate(1), 0);
        assertEquals(2, inputGate.getNumberOfQueuedBuffers());
    }
}
Also used : Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) InputChannelTestUtils.createLocalInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) Test(org.junit.Test)

Example 3 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class RemoteInputChannelTest method testPartitionNotFoundExceptionWhileRetriggeringRequest.

/**
 * Tests that {@link RemoteInputChannel#retriggerSubpartitionRequest()} would throw the {@link
 * PartitionNotFoundException} if backoff is 0.
 */
@Test
public void testPartitionNotFoundExceptionWhileRetriggeringRequest() throws Exception {
    final RemoteInputChannel inputChannel = InputChannelTestUtils.createRemoteInputChannel(createSingleInputGate(1), 0, new TestingConnectionManager());
    // Request partition to initialize client to avoid illegal state after retriggering
    // partition
    inputChannel.requestSubpartition();
    // The default backoff is 0 then it would set PartitionNotFoundException on this channel
    inputChannel.retriggerSubpartitionRequest();
    try {
        inputChannel.checkError();
        fail("Should throw a PartitionNotFoundException.");
    } catch (PartitionNotFoundException notFound) {
        assertThat(inputChannel.getPartitionId(), is(notFound.getPartitionId()));
    }
}
Also used : PartitionNotFoundException(org.apache.flink.runtime.io.network.partition.PartitionNotFoundException) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) Test(org.junit.Test)

Example 4 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class UnalignedCheckpointsTest method createInputGate.

private CheckpointedInputGate createInputGate(int numberOfChannels, AbstractInvokable toNotify, boolean enableCheckpointsAfterTasksFinished) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setStateWriter(channelStateWriter).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    sequenceNumbers = new int[numberOfChannels];
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify, enableCheckpointsAfterTasksFinished);
}
Also used : SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 5 with TestingConnectionManager

use of org.apache.flink.runtime.io.network.TestingConnectionManager in project flink by apache.

the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.

private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
    ConnectionManager connectionManager = new TestingConnectionManager();
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}
Also used : Deadline(org.apache.flink.api.common.time.Deadline) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) SystemClock(org.apache.flink.util.clock.SystemClock) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) HashMap(java.util.HashMap) PartitionRequestClient(org.apache.flink.runtime.io.network.PartitionRequestClient) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) CheckedThread(org.apache.flink.core.testutils.CheckedThread) Duration(java.time.Duration) TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) BufferBuilderTestUtils.buildSomeBuffer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSomeBuffer) InputChannelInfo(org.apache.flink.runtime.checkpoint.channel.InputChannelInfo) MockChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.MockChannelStateWriter) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) Before(org.junit.Before) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) EndOfChannelStateEvent(org.apache.flink.runtime.io.network.partition.consumer.EndOfChannelStateEvent) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) EventSerializer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer) AbstractEvent(org.apache.flink.runtime.event.AbstractEvent) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) InputChannelBuilder(org.apache.flink.runtime.io.network.partition.consumer.InputChannelBuilder) AbstractInvokable(org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable) Assert.assertTrue(org.junit.Assert.assertTrue) ConnectionID(org.apache.flink.runtime.io.network.ConnectionID) Test(org.junit.Test) IOException(java.io.IOException) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) MailboxExecutorImpl(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) CountDownLatch(java.util.concurrent.CountDownLatch) RecordingChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) TestingPartitionRequestClient(org.apache.flink.runtime.io.network.TestingPartitionRequestClient) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) StreamTaskActionExecutor(org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) MailboxExecutorImpl(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Aggregations

TestingConnectionManager (org.apache.flink.runtime.io.network.TestingConnectionManager)9 NettyShuffleEnvironment (org.apache.flink.runtime.io.network.NettyShuffleEnvironment)7 Test (org.junit.Test)6 NettyShuffleEnvironmentBuilder (org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder)5 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)3 RemoteInputChannel (org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)3 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)3 SingleInputGateBuilder (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder)3 IOException (java.io.IOException)2 Optional (java.util.Optional)2 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)2 CheckpointType (org.apache.flink.runtime.checkpoint.CheckpointType)2 InputChannelInfo (org.apache.flink.runtime.checkpoint.channel.InputChannelInfo)2 CancelTaskException (org.apache.flink.runtime.execution.CancelTaskException)2 ConnectionID (org.apache.flink.runtime.io.network.ConnectionID)2 ConnectionManager (org.apache.flink.runtime.io.network.ConnectionManager)2 PartitionRequestClient (org.apache.flink.runtime.io.network.PartitionRequestClient)2 TestingPartitionRequestClient (org.apache.flink.runtime.io.network.TestingPartitionRequestClient)2 InputChannelTestUtils.createLocalInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel)2 InputChannelTestUtils.createRemoteInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel)2