Examples with RecordingChannelStateWriter org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter used on opensource projects

Example 16 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testBarrierOvertaking.

@Test
public void testBarrierOvertaking() throws Exception {
    final RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    subpartition.setChannelStateWriter(channelStateWriter);
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertEquals(0, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE, false);
    subpartition.add(eventBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(4));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[] { 0, 1, 2 }));
    channelStateWriter.start(0, options);
    BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options), true);
    subpartition.add(barrierBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    final List<Buffer> inflight = channelStateWriter.getAddedOutput().get(subpartition.getSubpartitionInfo());
    assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
    inflight.forEach(Buffer::recycleBuffer);
    assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
    assertNextBuffer(readView, 1, true, 1, false, true);
    assertNextBuffer(readView, 2, true, 0, true, true);
    assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
    assertNextBuffer(readView, 4, false, 0, false, true);
    assertNoNextBuffer(readView);
}

Example 17 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class LocalInputChannelTest method testNoDataPersistedAfterReceivingAlignedBarrier.

@Test
public void testNoDataPersistedAfterReceivingAlignedBarrier() throws Exception {
    CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, CheckpointOptions.alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), 123L));
    BufferConsumer barrierHolder = EventSerializer.toBufferConsumer(barrier, false);
    BufferConsumer data = createFilledFinishedBufferConsumer(1);
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = InputChannelBuilder.newBuilder().setPartitionManager(new TestingResultPartitionManager(InputChannelTestUtils.createResultSubpartitionView(barrierHolder, data))).setStateWriter(stateWriter).buildLocalChannel(new SingleInputGateBuilder().build());
    channel.requestSubpartition();
    // pull AC barrier
    channel.getNextBuffer();
    // pretend that alignment timed out
    stateWriter.start(barrier.getId(), barrier.getCheckpointOptions());
    channel.checkpointStarted(barrier);
    // pull data
    channel.getNextBuffer();
    assertTrue("no data should be persisted after receiving a barrier", stateWriter.getAddedInput().isEmpty());
}

Example 18 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class ChannelStatePersisterTest method testNewBarrierNotOverwrittenByStopPersisting.

@Test
public void testNewBarrierNotOverwrittenByStopPersisting() throws Exception {
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    InputChannelInfo channelInfo = new InputChannelInfo(0, 0);
    ChannelStatePersister persister = new ChannelStatePersister(channelStateWriter, channelInfo);
    long checkpointId = 1L;
    channelStateWriter.start(checkpointId, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault()));
    persister.checkForBarrier(barrier(checkpointId));
    persister.startPersisting(checkpointId, Arrays.asList(buildSomeBuffer()));
    assertEquals(1, channelStateWriter.getAddedInput().get(channelInfo).size());
    persister.maybePersist(buildSomeBuffer());
    assertEquals(1, channelStateWriter.getAddedInput().get(channelInfo).size());
    // meanwhile, checkpoint coordinator timed out the 1st checkpoint and started the 2nd
    // now task thread is picking up the barrier and aborts the 1st:
    persister.checkForBarrier(barrier(checkpointId + 1));
    persister.maybePersist(buildSomeBuffer());
    persister.stopPersisting(checkpointId);
    persister.maybePersist(buildSomeBuffer());
    assertEquals(1, channelStateWriter.getAddedInput().get(channelInfo).size());
    assertTrue(persister.hasBarrierReceived());
}

Example 19 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class AlternatingCheckpointsTest method testChannelResetOnNewBarrier.

/**
 * Upon subsuming (or canceling) a checkpoint, channels should be notified regardless of whether
 * UC controller is currently being used or not. Otherwise, channels may not capture in-flight
 * buffers.
 */
@Test
public void testChannelResetOnNewBarrier() throws Exception {
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(2, getTestBarrierHandlerFactory(new ValidatingCheckpointHandler())).withChannelStateWriter(stateWriter).withRemoteChannels().withMailboxExecutor().build()) {
        sendBarrier(0, clock.relativeTimeMillis(), SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, // using AC because UC would require ordering in gate while polling
        0);
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(createBuffer(1024), 1, // to be captured
        0);
        send(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true), 1, gate);
        assertFalse(stateWriter.getAddedInput().isEmpty());
    }
}

Example 20 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnUnalignedCheckpoint.

/**
 * First we process aligned {@link CheckpointBarrier} and after that we receive an already
 * unaligned {@link CheckpointBarrier}, that has timed out on an upstream task.
 */
@Test
public void testTimeoutAlignmentOnUnalignedCheckpoint() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(3, getTestBarrierHandlerFactory(target)).withChannelStateWriter(channelStateWriter).withRemoteChannels().withMailboxExecutor().build();
    getChannel(gate, 0).onBuffer(withTimeout(Integer.MAX_VALUE).retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 1).onBuffer(dataBuffer(), 1, 0);
    getChannel(gate, 1).onBuffer(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true).retainBuffer(), 2, 0);
    assertBarrier(gate);
    assertEquals(2, channelStateWriter.getAddedInput().get(getChannel(gate, 1).getChannelInfo()).size());
    assertEquals(1, target.getTriggeredCheckpointCounter());
}