Examples with RecordingChannelStateWriter org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter used on opensource projects

Example 1 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class ChannelStatePersisterTest method testLateBarrier.

private void testLateBarrier(boolean startCheckpointOnLateBarrier, boolean cancelCheckpointBeforeLateBarrier) throws Exception {
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    InputChannelInfo channelInfo = new InputChannelInfo(0, 0);
    ChannelStatePersister persister = new ChannelStatePersister(channelStateWriter, channelInfo);
    long lateCheckpointId = 1L;
    long checkpointId = 2L;
    if (startCheckpointOnLateBarrier) {
        persister.startPersisting(lateCheckpointId, Collections.emptyList());
    }
    if (cancelCheckpointBeforeLateBarrier) {
        persister.stopPersisting(lateCheckpointId);
    }
    persister.checkForBarrier(barrier(lateCheckpointId));
    channelStateWriter.start(checkpointId, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault()));
    persister.startPersisting(checkpointId, Arrays.asList(buildSomeBuffer()));
    persister.maybePersist(buildSomeBuffer());
    persister.checkForBarrier(barrier(checkpointId));
    persister.maybePersist(buildSomeBuffer());
    assertTrue(persister.hasBarrierReceived());
    assertEquals(2, channelStateWriter.getAddedInput().get(channelInfo).size());
}

Example 2 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.

private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
    ConnectionManager connectionManager = new TestingConnectionManager();
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}

Example 3 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class LocalInputChannelTest method testCheckpointingInflightData.

@Test
public void testCheckpointingInflightData() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().build();
    PipelinedResultPartition parent = (PipelinedResultPartition) PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED, NoOpFileChannelManager.INSTANCE);
    ResultSubpartition subpartition = parent.getAllPartitions()[0];
    ResultSubpartitionView subpartitionView = subpartition.createReadView(() -> {
    });
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    final RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = createLocalInputChannel(inputGate, partitionManager, 0, 0, b -> b.setStateWriter(stateWriter));
    inputGate.setInputChannels(channel);
    channel.requestSubpartition();
    final CheckpointStorageLocationReference location = getDefault();
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, location);
    stateWriter.start(0, options);
    final CheckpointBarrier barrier = new CheckpointBarrier(0, 123L, options);
    channel.checkpointStarted(barrier);
    // add 1 buffer before barrier and 1 buffer afterwards. Only the first buffer should be
    // written.
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(EventSerializer.toBufferConsumer(barrier, true));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertTrue(channel.getNextBuffer().isPresent());
    assertArrayEquals(stateWriter.getAddedInput().get(channel.getChannelInfo()).stream().mapToInt(Buffer::getSize).toArray(), new int[] { 1 });
}

Example 4 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testBarrierOvertaking.

@Test
public void testBarrierOvertaking() throws Exception {
    final RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    subpartition.setChannelStateWriter(channelStateWriter);
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertEquals(0, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE, false);
    subpartition.add(eventBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(4));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[] { 0, 1, 2 }));
    channelStateWriter.start(0, options);
    BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options), true);
    subpartition.add(barrierBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    final List<Buffer> inflight = channelStateWriter.getAddedOutput().get(subpartition.getSubpartitionInfo());
    assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
    inflight.forEach(Buffer::recycleBuffer);
    assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
    assertNextBuffer(readView, 1, true, 1, false, true);
    assertNextBuffer(readView, 2, true, 0, true, true);
    assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
    assertNextBuffer(readView, 4, false, 0, false, true);
    assertNoNextBuffer(readView);
}

Example 5 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by apache.

the class LocalInputChannelTest method testNoDataPersistedAfterReceivingAlignedBarrier.

@Test
public void testNoDataPersistedAfterReceivingAlignedBarrier() throws Exception {
    CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, CheckpointOptions.alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), 123L));
    BufferConsumer barrierHolder = EventSerializer.toBufferConsumer(barrier, false);
    BufferConsumer data = createFilledFinishedBufferConsumer(1);
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = InputChannelBuilder.newBuilder().setPartitionManager(new TestingResultPartitionManager(InputChannelTestUtils.createResultSubpartitionView(barrierHolder, data))).setStateWriter(stateWriter).buildLocalChannel(new SingleInputGateBuilder().build());
    channel.requestSubpartition();
    // pull AC barrier
    channel.getNextBuffer();
    // pretend that alignment timed out
    stateWriter.start(barrier.getId(), barrier.getCheckpointOptions());
    channel.checkpointStarted(barrier);
    // pull data
    channel.getNextBuffer();
    assertTrue("no data should be persisted after receiving a barrier", stateWriter.getAddedInput().isEmpty());
}