Examples with TestInputChannel org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel used on opensource projects

Example 6 with TestInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testTriggeringCheckpointWithFinishedChannels.

private void testTriggeringCheckpointWithFinishedChannels(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        try (StreamTaskMailboxTestHarness<String> testHarness = createTestHarness(partitionWriters, new CompletingCheckpointResponder(), checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable())) {
            int numChannels = testHarness.inputGates[0].getInputGate().getNumberOfInputChannels();
            int[] resumedCount = new int[numChannels];
            for (int i = 0; i < numChannels; ++i) {
                TestInputChannel inputChannel = (TestInputChannel) testHarness.inputGates[0].getInputGate().getChannel(i);
                inputChannel.setActionOnResumed(() -> resumedCount[inputChannel.getChannelIndex()]++);
            }
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertArrayEquals(new int[] { 0, 0, 0 }, resumedCount);
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertArrayEquals(new int[] { 0, 0, 0 }, resumedCount);
            // Tests triggering checkpoint after received all the inputs have received
            // EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 1);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 2);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 1);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 2);
            checkpointFuture = triggerCheckpoint(testHarness, 6, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertArrayEquals(new int[] { 0, 0, 0 }, resumedCount);
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}

Example 7 with TestInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel in project flink by apache.

the class StreamTaskTest method testBufferSizeRecalculationStartSuccessfully.

@Test
public void testBufferSizeRecalculationStartSuccessfully() throws Exception {
    int expectedThroughput = 13333;
    int inputChannels = 3;
    // debloat period doesn't matter, we will schedule debloating manually
    Configuration config = new Configuration().set(BUFFER_DEBLOAT_PERIOD, Duration.ofHours(10)).set(BUFFER_DEBLOAT_TARGET, Duration.ofSeconds(1)).set(BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES, 1).set(BUFFER_DEBLOAT_ENABLED, true);
    try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(config)).addInput(STRING_TYPE_INFO, inputChannels).addInput(STRING_TYPE_INFO, inputChannels).modifyGateBuilder(gateBuilder -> gateBuilder.setThroughputCalculator(bufferDebloatConfiguration -> new ThroughputCalculator(SystemClock.getInstance()) {

        @Override
        public long calculateThroughput() {
            return expectedThroughput;
        }
    })).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build()) {
        harness.processAll();
        harness.streamTask.debloat();
        long lastBufferSize = -1;
        for (InputGate inputGate : harness.streamTask.getEnvironment().getAllInputGates()) {
            for (int i = 0; i < inputGate.getNumberOfInputChannels(); i++) {
                long currentBufferSize = ((TestInputChannel) inputGate.getChannel(i)).getCurrentBufferSize();
                assertThat(currentBufferSize, lessThan(MEMORY_SEGMENT_SIZE.defaultValue().getBytes()));
                assertThat(currentBufferSize, greaterThan(0L));
                if (lastBufferSize > 0) {
                    assertThat(lastBufferSize, is(currentBufferSize));
                }
                lastBufferSize = currentBufferSize;
            }
        }
    }
}

Example 8 with TestInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel in project flink by apache.

the class AlternatingCheckpointsTest method testBarrierHandling.

private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
    final long barrierId = 123L;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
    TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
    gate.setInputChannels(fast, slow);
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
    CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
    if (checkpointType.isSavepoint()) {
        fast.setBlocked(true);
        slow.setBlocked(true);
    }
    CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
    Buffer barrier = barrier(barrierId, 1, options);
    send(barrier.retainBuffer(), fast, checkpointedGate);
    assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
    send(barrier.retainBuffer(), slow, checkpointedGate);
    assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
    if (checkpointType.isSavepoint()) {
        for (InputChannel channel : gate.getInputChannels().values()) {
            assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
        }
    }
}

Example 9 with TestInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel in project flink by apache.

the class AlternatingCheckpointsTest method testAllChannelsUnblockedAfterAlignmentTimeout.

@Test
public void testAllChannelsUnblockedAfterAlignmentTimeout() throws Exception {
    int numberOfChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numberOfChannels, getTestBarrierHandlerFactory(target)).withTestChannels().withSyncExecutor().build();
    long alignmentTimeout = 100;
    CheckpointBarrier checkpointBarrier = new CheckpointBarrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeout));
    Buffer checkpointBarrierBuffer = toBuffer(checkpointBarrier, false);
    // we set timer on announcement and test channels do not produce announcements by themselves
    send(EventSerializer.toBuffer(new EventAnnouncement(checkpointBarrier, 0), true), 0, gate);
    // emulate blocking channels on aligned barriers
    ((TestInputChannel) gate.getChannel(0)).setBlocked(true);
    send(checkpointBarrierBuffer, 0, gate);
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    send(EventSerializer.toBuffer(new EventAnnouncement(checkpointBarrier, 0), true), 1, gate);
    // emulate blocking channels on aligned barriers
    ((TestInputChannel) gate.getChannel(1)).setBlocked(true);
    send(checkpointBarrierBuffer, 1, gate);
    assertThat(target.getTriggeredCheckpointOptions().size(), equalTo(1));
    assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
    assertFalse(((TestInputChannel) gate.getChannel(0)).isBlocked());
    assertFalse(((TestInputChannel) gate.getChannel(1)).isBlocked());
}

Example 10 with TestInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentWhenLocalBarrierFirst.

@Test
public void testTimeoutAlignmentWhenLocalBarrierFirst() throws Exception {
    // given: Gate with remote and local channels.
    int numChannels = 3;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withMixedChannels(0).withMailboxExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(1, alignmentTimeout);
    // when: Execute the first checkpoint when announcement received first.
    ((TestInputChannel) gate.getChannel(0)).read(checkpointBarrier.retainBuffer());
    (getChannel(gate, 1)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    (getChannel(gate, 2)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertAnnouncement(gate);
    assertBarrier(gate);
    assertBarrier(gate);
    assertBarrier(gate);
    // then: The checkpoint executed successfully.
    assertEquals(1, target.getTriggeredCheckpointCounter());
    // given: The time in the future.
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    checkpointBarrier = withTimeout(2, alignmentTimeout);
    // when: Execute the second checkpoint when barrier from local channel without announcement
    // received first.
    ((TestInputChannel) gate.getChannel(0)).read(checkpointBarrier.retainBuffer());
    assertBarrier(gate);
    // then: Nothing happens because the alignment timeout should only start after this barrier.
    assertEquals(1, target.getTriggeredCheckpointCounter());
    // when: Receiving the barrier from second channel(with/without) announcement after time
    // more than alignment timeout.
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    (getChannel(gate, 1)).onBuffer(checkpointBarrier.retainBuffer(), 1, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    // then: The checkpoint should started as unaligned.
    assertEquals(2, target.getTriggeredCheckpointCounter());
    List<CheckpointOptions> checkpointOptions = target.getTriggeredCheckpointOptions();
    assertEquals(AlignmentType.UNALIGNED, checkpointOptions.get(checkpointOptions.size() - 1).getAlignment());
}