Examples with InputChannelInfo org.apache.flink.runtime.checkpoint.channel.InputChannelInfo used on opensource projects

Example 16 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class AlternatingCheckpointsTest method testOutOfOrderBarrier.

@Test
public void testOutOfOrderBarrier() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel firstChannel = new TestInputChannel(inputGate, 0);
    TestInputChannel secondChannel = new TestInputChannel(inputGate, 1);
    inputGate.setInputChannels(firstChannel, secondChannel);
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(inputGate);
    long checkpointId = 10;
    long outOfOrderSavepointId = 5;
    barrierHandler.processBarrier(new CheckpointBarrier(checkpointId, clock.relativeTimeMillis(), new CheckpointOptions(CHECKPOINT, getDefault())), new InputChannelInfo(0, 0), false);
    secondChannel.setBlocked(true);
    barrierHandler.processBarrier(new CheckpointBarrier(outOfOrderSavepointId, clock.relativeTimeMillis(), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), getDefault())), new InputChannelInfo(0, 1), false);
    assertEquals(checkpointId, barrierHandler.getLatestCheckpointId());
    assertFalse(secondChannel.isBlocked());
}

Example 17 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class InputProcessorUtilTest method testCreateCheckpointedMultipleInputGate.

@Test
public void testCreateCheckpointedMultipleInputGate() throws Exception {
    try (CloseableRegistry registry = new CloseableRegistry()) {
        MockEnvironment environment = new MockEnvironmentBuilder().build();
        MockStreamTask streamTask = new MockStreamTaskBuilder(environment).build();
        StreamConfig streamConfig = new StreamConfig(environment.getJobConfiguration());
        streamConfig.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE);
        streamConfig.setUnalignedCheckpointsEnabled(true);
        // First input gate has index larger than the second
        List<IndexedInputGate>[] inputGates = new List[] { Collections.singletonList(getGate(1, 4)), Collections.singletonList(getGate(0, 2)) };
        CheckpointBarrierHandler barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(streamTask, streamConfig, new TestSubtaskCheckpointCoordinator(new MockChannelStateWriter()), streamTask.getName(), inputGates, Collections.emptyList(), new SyncMailboxExecutor(), new TestProcessingTimeService());
        CheckpointedInputGate[] checkpointedMultipleInputGate = InputProcessorUtil.createCheckpointedMultipleInputGate(new SyncMailboxExecutor(), inputGates, environment.getMetricGroup().getIOMetricGroup(), barrierHandler, streamConfig);
        for (CheckpointedInputGate checkpointedInputGate : checkpointedMultipleInputGate) {
            registry.registerCloseable(checkpointedInputGate);
        }
        List<IndexedInputGate> allInputGates = Arrays.stream(inputGates).flatMap(gates -> gates.stream()).collect(Collectors.toList());
        for (IndexedInputGate inputGate : allInputGates) {
            for (int channelId = 0; channelId < inputGate.getNumberOfInputChannels(); channelId++) {
                barrierHandler.processBarrier(new CheckpointBarrier(1, 42, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(inputGate.getGateIndex(), channelId), false);
            }
        }
        assertTrue(barrierHandler.getAllBarriersReceivedFuture(1).isDone());
    }
}

Example 18 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class StreamTask method triggerUnfinishedChannelsCheckpoint.

private boolean triggerUnfinishedChannelsCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) throws Exception {
    Optional<CheckpointBarrierHandler> checkpointBarrierHandler = getCheckpointBarrierHandler();
    checkState(checkpointBarrierHandler.isPresent(), "CheckpointBarrier should exist for tasks with network inputs.");
    CheckpointBarrier barrier = new CheckpointBarrier(checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions);
    for (IndexedInputGate inputGate : getEnvironment().getAllInputGates()) {
        if (!inputGate.isFinished()) {
            for (InputChannelInfo channelInfo : inputGate.getUnfinishedChannels()) {
                checkpointBarrierHandler.get().processBarrier(barrier, channelInfo, true);
            }
        }
    }
    return true;
}

Example 19 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class StreamTaskNetworkInputTest method testReleasingDeserializerTimely.

@Test
public void testReleasingDeserializerTimely() throws Exception {
    int numInputChannels = 2;
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    DataOutput<Long> output = new NoOpDataOutput<>();
    Map<InputChannelInfo, TestRecordDeserializer> deserializers = createDeserializers(inputGate.getInputGate());
    Map<InputChannelInfo, TestRecordDeserializer> copiedDeserializers = new HashMap<>(deserializers);
    StreamTaskInput<Long> input = new TestStreamTaskNetworkInput(inputGate, inSerializer, numInputChannels, deserializers);
    for (InputChannelInfo channelInfo : inputGate.getInputGate().getChannelInfos()) {
        assertNotNull(deserializers.get(channelInfo));
        inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelInfo.getInputChannelIdx());
        input.emitNext(output);
        assertTrue(copiedDeserializers.get(channelInfo).isCleared());
        assertNull(deserializers.get(channelInfo));
    }
}

Example 20 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class SingleInputGateTest method testSingleInputGateInfo.

@Test
public void testSingleInputGateInfo() {
    final int numSingleInputGates = 2;
    final int numInputChannels = 3;
    for (int i = 0; i < numSingleInputGates; i++) {
        final SingleInputGate gate = new SingleInputGateBuilder().setSingleInputGateIndex(i).setNumberOfChannels(numInputChannels).build();
        int channelCounter = 0;
        for (InputChannel inputChannel : gate.getInputChannels().values()) {
            InputChannelInfo channelInfo = inputChannel.getChannelInfo();
            assertEquals(i, channelInfo.getGateIdx());
            assertEquals(channelCounter++, channelInfo.getInputChannelIdx());
        }
    }
}