Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 26 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CheckpointedInputGateTest method setupInputGate.

private CheckpointedInputGate setupInputGate(int numberOfChannels, NetworkBufferPool networkBufferPool, ConnectionManager connectionManager) throws Exception {
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    CheckpointBarrierTracker barrierHandler = new CheckpointBarrierTracker(numberOfChannels, new AbstractInvokable(new DummyEnvironment()) {

        @Override
        public void invoke() {
        }
    }, SystemClock.getInstance(), true);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}

Example 27 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class AlternatingCheckpointsTest method testHasInflightDataBeforeProcessBarrier.

@Test
public void testHasInflightDataBeforeProcessBarrier() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(inputGate);
    final long id = 1;
    barrierHandler.processBarrier(new CheckpointBarrier(id, clock.relativeTimeMillis(), new CheckpointOptions(CHECKPOINT, getDefault())), new InputChannelInfo(0, 0), false);
    assertFalse(barrierHandler.getAllBarriersReceivedFuture(id).isDone());
}

Example 28 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class AlternatingCheckpointsTest method testPreviousHandlerReset.

@Test
public void testPreviousHandlerReset() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel[] channels = { new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1) };
    inputGate.setInputChannels(channels);
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(inputGate);
    for (int i = 0; i < 4; i++) {
        int channel = i % 2;
        SnapshotType type = channel == 0 ? SavepointType.savepoint(SavepointFormatType.CANONICAL) : CHECKPOINT;
        target.setNextExpectedCheckpointId(-1);
        if (type.isSavepoint()) {
            channels[channel].setBlocked(true);
        }
        barrierHandler.processBarrier(new CheckpointBarrier(i, clock.relativeTimeMillis(), new CheckpointOptions(type, getDefault())), new InputChannelInfo(0, channel), false);
        if (type.isSavepoint()) {
            assertTrue(channels[channel].isBlocked());
            assertFalse(channels[(channel + 1) % 2].isBlocked());
        } else {
            assertFalse(channels[0].isBlocked());
            assertFalse(channels[1].isBlocked());
        }
        assertTrue(barrierHandler.isCheckpointPending());
        assertFalse(barrierHandler.getAllBarriersReceivedFuture(i).isDone());
        channels[0].setBlocked(false);
        channels[1].setBlocked(false);
    }
}

Example 29 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class AlternatingCheckpointsTest method testOutOfOrderBarrier.

@Test
public void testOutOfOrderBarrier() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel firstChannel = new TestInputChannel(inputGate, 0);
    TestInputChannel secondChannel = new TestInputChannel(inputGate, 1);
    inputGate.setInputChannels(firstChannel, secondChannel);
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(inputGate);
    long checkpointId = 10;
    long outOfOrderSavepointId = 5;
    barrierHandler.processBarrier(new CheckpointBarrier(checkpointId, clock.relativeTimeMillis(), new CheckpointOptions(CHECKPOINT, getDefault())), new InputChannelInfo(0, 0), false);
    secondChannel.setBlocked(true);
    barrierHandler.processBarrier(new CheckpointBarrier(outOfOrderSavepointId, clock.relativeTimeMillis(), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), getDefault())), new InputChannelInfo(0, 1), false);
    assertEquals(checkpointId, barrierHandler.getLatestCheckpointId());
    assertFalse(secondChannel.isBlocked());
}

Example 30 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class AlignedCheckpointsTest method createCheckpointedInputGate.

private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}