Examples with CheckpointOptions org.apache.flink.runtime.checkpoint.CheckpointOptions used on opensource projects

Example 31 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class RemoteInputChannelTest method testNotifyOnPriority.

@Test
public void testNotifyOnPriority() throws IOException {
    SingleInputGate inputGate = new SingleInputGateBuilder().build();
    RemoteInputChannel channel = InputChannelTestUtils.createRemoteInputChannel(inputGate, 0);
    CheckpointOptions options = new CheckpointOptions(CHECKPOINT, getDefault());
    assertPriorityAvailability(inputGate, false, false, () -> assertAvailability(inputGate, false, true, () -> {
        channel.onBuffer(toBuffer(new CheckpointBarrier(1L, 123L, options), false), 0, 0);
    }));
    assertPriorityAvailability(inputGate, false, true, () -> assertAvailability(inputGate, true, true, () -> {
        channel.onBuffer(toBuffer(new CheckpointBarrier(2L, 123L, options), true), 1, 0);
    }));
}

Example 32 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testBarrierOvertaking.

@Test
public void testBarrierOvertaking() throws Exception {
    final RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    subpartition.setChannelStateWriter(channelStateWriter);
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertEquals(0, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE, false);
    subpartition.add(eventBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(4));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[] { 0, 1, 2 }));
    channelStateWriter.start(0, options);
    BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options), true);
    subpartition.add(barrierBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    final List<Buffer> inflight = channelStateWriter.getAddedOutput().get(subpartition.getSubpartitionInfo());
    assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
    inflight.forEach(Buffer::recycleBuffer);
    assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
    assertNextBuffer(readView, 1, true, 1, false, true);
    assertNextBuffer(readView, 2, true, 0, true, true);
    assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
    assertNextBuffer(readView, 4, false, 0, false, true);
    assertNoNextBuffer(readView);
}

Example 33 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testAvailabilityAfterPriority.

@Test
public void testAvailabilityAfterPriority() throws Exception {
    subpartition.setChannelStateWriter(ChannelStateWriter.NO_OP);
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[] { 0, 1, 2 }));
    BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options), true);
    subpartition.add(barrierBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertEquals(2, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertEquals(2, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 1, false, true);
    assertNextBuffer(readView, 1, false, 0, false, true);
    assertNextBuffer(readView, 2, false, 0, false, true);
    assertNoNextBuffer(readView);
}

Example 34 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class TaskTest method assertCheckpointDeclined.

private void assertCheckpointDeclined(Task task, TestCheckpointResponder testCheckpointResponder, long checkpointId, CheckpointFailureReason failureReason) {
    CheckpointOptions checkpointOptions = CheckpointOptions.alignedNoTimeout(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault());
    task.triggerCheckpointBarrier(checkpointId, 1, checkpointOptions);
    assertEquals(1, testCheckpointResponder.getDeclineReports().size());
    assertEquals(checkpointId, testCheckpointResponder.getDeclineReports().get(0).getCheckpointId());
    assertEquals(failureReason, testCheckpointResponder.getDeclineReports().get(0).getCause().getCheckpointFailureReason());
    testCheckpointResponder.clear();
}

Example 35 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class ChannelPersistenceITCase method write.

private ChannelStateWriteResult write(long checkpointId, Map<InputChannelInfo, byte[]> icMap, Map<ResultSubpartitionInfo, byte[]> rsMap) throws Exception {
    int maxStateSize = sizeOfBytes(icMap) + sizeOfBytes(rsMap) + Long.BYTES * 2;
    Map<InputChannelInfo, Buffer> icBuffers = wrapWithBuffers(icMap);
    Map<ResultSubpartitionInfo, Buffer> rsBuffers = wrapWithBuffers(rsMap);
    try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl("test", 0, getStreamFactoryFactory(maxStateSize))) {
        writer.open();
        writer.start(checkpointId, new CheckpointOptions(CHECKPOINT, new CheckpointStorageLocationReference("poly".getBytes())));
        for (Map.Entry<InputChannelInfo, Buffer> e : icBuffers.entrySet()) {
            writer.addInputData(checkpointId, e.getKey(), SEQUENCE_NUMBER_UNKNOWN, ofElements(Buffer::recycleBuffer, e.getValue()));
        }
        writer.finishInput(checkpointId);
        for (Map.Entry<ResultSubpartitionInfo, Buffer> e : rsBuffers.entrySet()) {
            writer.addOutputData(checkpointId, e.getKey(), SEQUENCE_NUMBER_UNKNOWN, e.getValue());
        }
        writer.finishOutput(checkpointId);
        ChannelStateWriteResult result = writer.getAndRemoveWriteResult(checkpointId);
        // prevent abnormal complete in close
        result.getResultSubpartitionStateHandles().join();
        return result;
    }
}