Examples with RecordingChannelStateWriter org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter used on opensource projects

Example 11 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by splunk.

the class ChannelStatePersisterTest method testLateBarrier.

private void testLateBarrier(boolean startCheckpointOnLateBarrier, boolean cancelCheckpointBeforeLateBarrier) throws Exception {
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    InputChannelInfo channelInfo = new InputChannelInfo(0, 0);
    ChannelStatePersister persister = new ChannelStatePersister(channelStateWriter, channelInfo);
    long lateCheckpointId = 1L;
    long checkpointId = 2L;
    if (startCheckpointOnLateBarrier) {
        persister.startPersisting(lateCheckpointId, Collections.emptyList());
    }
    if (cancelCheckpointBeforeLateBarrier) {
        persister.stopPersisting(lateCheckpointId);
    }
    persister.checkForBarrier(barrier(lateCheckpointId));
    channelStateWriter.start(checkpointId, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault()));
    persister.startPersisting(checkpointId, Arrays.asList(buildSomeBuffer()));
    persister.maybePersist(buildSomeBuffer());
    persister.checkForBarrier(barrier(checkpointId));
    persister.maybePersist(buildSomeBuffer());
    assertTrue(persister.hasBarrierReceived());
    assertEquals(2, channelStateWriter.getAddedInput().get(channelInfo).size());
}

Example 12 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by splunk.

the class AlternatingCheckpointsTest method testChannelResetOnNewBarrier.

/**
 * Upon subsuming (or canceling) a checkpoint, channels should be notified regardless of whether
 * UC controller is currently being used or not. Otherwise, channels may not capture in-flight
 * buffers.
 */
@Test
public void testChannelResetOnNewBarrier() throws Exception {
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(2, getTestBarrierHandlerFactory(new ValidatingCheckpointHandler())).withChannelStateWriter(stateWriter).withRemoteChannels().withMailboxExecutor().build()) {
        sendBarrier(0, clock.relativeTimeMillis(), SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, // using AC because UC would require ordering in gate while polling
        0);
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(createBuffer(1024), 1, // to be captured
        0);
        send(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true), 1, gate);
        assertFalse(stateWriter.getAddedInput().isEmpty());
    }
}

Example 13 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by splunk.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnUnalignedCheckpoint.

/**
 * First we process aligned {@link CheckpointBarrier} and after that we receive an already
 * unaligned {@link CheckpointBarrier}, that has timed out on an upstream task.
 */
@Test
public void testTimeoutAlignmentOnUnalignedCheckpoint() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(3, getTestBarrierHandlerFactory(target)).withChannelStateWriter(channelStateWriter).withRemoteChannels().withMailboxExecutor().build();
    getChannel(gate, 0).onBuffer(withTimeout(Integer.MAX_VALUE).retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 1).onBuffer(dataBuffer(), 1, 0);
    getChannel(gate, 1).onBuffer(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true).retainBuffer(), 2, 0);
    assertBarrier(gate);
    assertEquals(2, channelStateWriter.getAddedInput().get(getChannel(gate, 1).getChannelInfo()).size());
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 14 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by splunk.

the class LocalInputChannelTest method testNoDataPersistedAfterReceivingAlignedBarrier.

@Test
public void testNoDataPersistedAfterReceivingAlignedBarrier() throws Exception {
    CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, CheckpointOptions.alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), 123L));
    BufferConsumer barrierHolder = EventSerializer.toBufferConsumer(barrier, false);
    BufferConsumer data = createFilledFinishedBufferConsumer(1);
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = InputChannelBuilder.newBuilder().setPartitionManager(new TestingResultPartitionManager(InputChannelTestUtils.createResultSubpartitionView(barrierHolder, data))).setStateWriter(stateWriter).buildLocalChannel(new SingleInputGateBuilder().build());
    channel.requestSubpartition();
    // pull AC barrier
    channel.getNextBuffer();
    // pretend that alignment timed out
    stateWriter.start(barrier.getId(), barrier.getCheckpointOptions());
    channel.checkpointStarted(barrier);
    // pull data
    channel.getNextBuffer();
    assertTrue("no data should be persisted after receiving a barrier", stateWriter.getAddedInput().isEmpty());
}

Example 15 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by splunk.

the class LocalInputChannelTest method testCheckpointingInflightData.

@Test
public void testCheckpointingInflightData() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().build();
    PipelinedResultPartition parent = (PipelinedResultPartition) PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED, NoOpFileChannelManager.INSTANCE);
    ResultSubpartition subpartition = parent.getAllPartitions()[0];
    ResultSubpartitionView subpartitionView = subpartition.createReadView(() -> {
    });
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    final RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = createLocalInputChannel(inputGate, partitionManager, 0, 0, b -> b.setStateWriter(stateWriter));
    inputGate.setInputChannels(channel);
    channel.requestSubpartition();
    final CheckpointStorageLocationReference location = getDefault();
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, location);
    stateWriter.start(0, options);
    final CheckpointBarrier barrier = new CheckpointBarrier(0, 123L, options);
    channel.checkpointStarted(barrier);
    // add 1 buffer before barrier and 1 buffer afterwards. Only the first buffer should be
    // written.
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(EventSerializer.toBufferConsumer(barrier, true));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertTrue(channel.getNextBuffer().isPresent());
    assertArrayEquals(stateWriter.getAddedInput().get(channel.getChannelInfo()).stream().mapToInt(Buffer::getSize).toArray(), new int[] { 1 });
}