Examples with RecordingChannelStateWriter org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter used on opensource projects

Example 6 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink-mirror by flink-ci.

the class LocalInputChannelTest method testNoDataPersistedAfterReceivingAlignedBarrier.

@Test
public void testNoDataPersistedAfterReceivingAlignedBarrier() throws Exception {
    CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, CheckpointOptions.alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), 123L));
    BufferConsumer barrierHolder = EventSerializer.toBufferConsumer(barrier, false);
    BufferConsumer data = createFilledFinishedBufferConsumer(1);
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = InputChannelBuilder.newBuilder().setPartitionManager(new TestingResultPartitionManager(InputChannelTestUtils.createResultSubpartitionView(barrierHolder, data))).setStateWriter(stateWriter).buildLocalChannel(new SingleInputGateBuilder().build());
    channel.requestSubpartition();
    // pull AC barrier
    channel.getNextBuffer();
    // pretend that alignment timed out
    stateWriter.start(barrier.getId(), barrier.getCheckpointOptions());
    channel.checkpointStarted(barrier);
    // pull data
    channel.getNextBuffer();
    assertTrue("no data should be persisted after receiving a barrier", stateWriter.getAddedInput().isEmpty());
}

Example 7 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink-mirror by flink-ci.

the class ChannelStatePersisterTest method testNewBarrierNotOverwrittenByStopPersisting.

@Test
public void testNewBarrierNotOverwrittenByStopPersisting() throws Exception {
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    InputChannelInfo channelInfo = new InputChannelInfo(0, 0);
    ChannelStatePersister persister = new ChannelStatePersister(channelStateWriter, channelInfo);
    long checkpointId = 1L;
    channelStateWriter.start(checkpointId, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault()));
    persister.checkForBarrier(barrier(checkpointId));
    persister.startPersisting(checkpointId, Arrays.asList(buildSomeBuffer()));
    assertEquals(1, channelStateWriter.getAddedInput().get(channelInfo).size());
    persister.maybePersist(buildSomeBuffer());
    assertEquals(1, channelStateWriter.getAddedInput().get(channelInfo).size());
    // meanwhile, checkpoint coordinator timed out the 1st checkpoint and started the 2nd
    // now task thread is picking up the barrier and aborts the 1st:
    persister.checkForBarrier(barrier(checkpointId + 1));
    persister.maybePersist(buildSomeBuffer());
    persister.stopPersisting(checkpointId);
    persister.maybePersist(buildSomeBuffer());
    assertEquals(1, channelStateWriter.getAddedInput().get(channelInfo).size());
    assertTrue(persister.hasBarrierReceived());
}

Example 8 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink-mirror by flink-ci.

the class ChannelStatePersisterTest method testLateBarrier.

private void testLateBarrier(boolean startCheckpointOnLateBarrier, boolean cancelCheckpointBeforeLateBarrier) throws Exception {
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    InputChannelInfo channelInfo = new InputChannelInfo(0, 0);
    ChannelStatePersister persister = new ChannelStatePersister(channelStateWriter, channelInfo);
    long lateCheckpointId = 1L;
    long checkpointId = 2L;
    if (startCheckpointOnLateBarrier) {
        persister.startPersisting(lateCheckpointId, Collections.emptyList());
    }
    if (cancelCheckpointBeforeLateBarrier) {
        persister.stopPersisting(lateCheckpointId);
    }
    persister.checkForBarrier(barrier(lateCheckpointId));
    channelStateWriter.start(checkpointId, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault()));
    persister.startPersisting(checkpointId, Arrays.asList(buildSomeBuffer()));
    persister.maybePersist(buildSomeBuffer());
    persister.checkForBarrier(barrier(checkpointId));
    persister.maybePersist(buildSomeBuffer());
    assertTrue(persister.hasBarrierReceived());
    assertEquals(2, channelStateWriter.getAddedInput().get(channelInfo).size());
}

Example 9 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink-mirror by flink-ci.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnUnalignedCheckpoint.

/**
 * First we process aligned {@link CheckpointBarrier} and after that we receive an already
 * unaligned {@link CheckpointBarrier}, that has timed out on an upstream task.
 */
@Test
public void testTimeoutAlignmentOnUnalignedCheckpoint() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(3, getTestBarrierHandlerFactory(target)).withChannelStateWriter(channelStateWriter).withRemoteChannels().withMailboxExecutor().build();
    getChannel(gate, 0).onBuffer(withTimeout(Integer.MAX_VALUE).retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 1).onBuffer(dataBuffer(), 1, 0);
    getChannel(gate, 1).onBuffer(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true).retainBuffer(), 2, 0);
    assertBarrier(gate);
    assertEquals(2, channelStateWriter.getAddedInput().get(getChannel(gate, 1).getChannelInfo()).size());
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 10 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink-mirror by flink-ci.

the class AlternatingCheckpointsTest method testChannelResetOnNewBarrier.

/**
 * Upon subsuming (or canceling) a checkpoint, channels should be notified regardless of whether
 * UC controller is currently being used or not. Otherwise, channels may not capture in-flight
 * buffers.
 */
@Test
public void testChannelResetOnNewBarrier() throws Exception {
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(2, getTestBarrierHandlerFactory(new ValidatingCheckpointHandler())).withChannelStateWriter(stateWriter).withRemoteChannels().withMailboxExecutor().build()) {
        sendBarrier(0, clock.relativeTimeMillis(), SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, // using AC because UC would require ordering in gate while polling
        0);
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(createBuffer(1024), 1, // to be captured
        0);
        send(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true), 1, gate);
        assertFalse(stateWriter.getAddedInput().isEmpty());
    }
}