Examples with InputChannelInfo org.apache.flink.runtime.checkpoint.channel.InputChannelInfo used on opensource projects

Example 6 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class RemoteInputChannelTest method testExceptionOnPersisting.

@Test
public void testExceptionOnPersisting() throws Exception {
    // Setup
    final SingleInputGate inputGate = createSingleInputGate(1);
    final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().setStateWriter(new ChannelStateWriter.NoOpChannelStateWriter() {

        @Override
        public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator<Buffer> data) {
            try {
                data.close();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
            throw new ExpectedTestException();
        }
    }).buildRemoteChannel(inputGate);
    inputChannel.checkpointStarted(new CheckpointBarrier(42, System.currentTimeMillis(), CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault())));
    final Buffer buffer = createBuffer(TestBufferFactory.BUFFER_SIZE);
    assertFalse(buffer.isRecycled());
    try {
        inputChannel.onBuffer(buffer, 0, -1);
        fail("This should have failed");
    } catch (ExpectedTestException ex) {
    // ignore
    }
    // This check is not strictly speaking necessary. Generally speaking if exception happens
    // during persisting, there are two potentially correct outcomes:
    // 1. buffer is recycled only once, in #onBuffer call when handling exception
    // 2. buffer is stored inside RemoteInputChannel and recycled on releaseAllResources.
    // What's not acceptable is that it would be released twice, in both places. Without this
    // check below, we would be just relaying on Buffer throwing IllegalReferenceCountException.
    // I've added this check just to be sure. It's freezing the current implementation that's
    // unlikely to change, on the other hand, thanks to it we don't need to relay on
    // IllegalReferenceCountException being thrown from the Buffer.
    // 
    // In other words, if you end up reading this after refactoring RemoteInputChannel, it might
    // be safe to remove this assertion. Just make sure double recycling of the same buffer is
    // still throwing IllegalReferenceCountException.
    assertFalse(buffer.isRecycled());
    inputChannel.releaseAllResources();
    assertTrue(buffer.isRecycled());
}

Example 7 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class ChannelStatePersisterTest method testLateBarrier.

private void testLateBarrier(boolean startCheckpointOnLateBarrier, boolean cancelCheckpointBeforeLateBarrier) throws Exception {
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    InputChannelInfo channelInfo = new InputChannelInfo(0, 0);
    ChannelStatePersister persister = new ChannelStatePersister(channelStateWriter, channelInfo);
    long lateCheckpointId = 1L;
    long checkpointId = 2L;
    if (startCheckpointOnLateBarrier) {
        persister.startPersisting(lateCheckpointId, Collections.emptyList());
    }
    if (cancelCheckpointBeforeLateBarrier) {
        persister.stopPersisting(lateCheckpointId);
    }
    persister.checkForBarrier(barrier(lateCheckpointId));
    channelStateWriter.start(checkpointId, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault()));
    persister.startPersisting(checkpointId, Arrays.asList(buildSomeBuffer()));
    persister.maybePersist(buildSomeBuffer());
    persister.checkForBarrier(barrier(checkpointId));
    persister.maybePersist(buildSomeBuffer());
    assertTrue(persister.hasBarrierReceived());
    assertEquals(2, channelStateWriter.getAddedInput().get(channelInfo).size());
}

Example 8 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class ChannelStatePersisterTest method testNewBarrierNotOverwrittenByCheckForBarrier.

@Test
public void testNewBarrierNotOverwrittenByCheckForBarrier() throws Exception {
    ChannelStatePersister persister = new ChannelStatePersister(ChannelStateWriter.NO_OP, new InputChannelInfo(0, 0));
    persister.startPersisting(1L, Collections.emptyList());
    persister.startPersisting(2L, Collections.emptyList());
    assertFalse(persister.checkForBarrier(barrier(1L)).isPresent());
    assertFalse(persister.hasBarrierReceived());
}

Example 9 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class UnalignedCheckpointsTest method testEndOfStreamWithPendingCheckpoint.

@Test
public void testEndOfStreamWithPendingCheckpoint() throws Exception {
    final int numberOfChannels = 2;
    final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildLocalChannel).setNumberOfChannels(numberOfChannels).build();
    final SingleCheckpointBarrierHandler handler = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), false, inputGate);
    // should trigger respective checkpoint
    handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0), false);
    assertTrue(handler.isCheckpointPending());
    assertEquals(DEFAULT_CHECKPOINT_ID, handler.getLatestCheckpointId());
    assertEquals(numberOfChannels, handler.getNumOpenChannels());
    // should abort current checkpoint while processing eof
    handler.processEndOfPartition(new InputChannelInfo(0, 0));
    assertFalse(handler.isCheckpointPending());
    assertEquals(DEFAULT_CHECKPOINT_ID, handler.getLatestCheckpointId());
    assertEquals(numberOfChannels - 1, handler.getNumOpenChannels());
    assertEquals(DEFAULT_CHECKPOINT_ID, invokable.getAbortedCheckpointId());
}

Example 10 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class UnalignedCheckpointsTest method testProcessCancellationBarrier.

private void testProcessCancellationBarrier(SingleCheckpointBarrierHandler handler, ValidatingCheckpointInvokable invokable) throws Exception {
    final long cancelledCheckpointId = new Random().nextBoolean() ? DEFAULT_CHECKPOINT_ID : DEFAULT_CHECKPOINT_ID + 1L;
    // should abort current checkpoint while processing CancelCheckpointMarker
    handler.processCancellationBarrier(new CancelCheckpointMarker(cancelledCheckpointId), new InputChannelInfo(0, 0));
    verifyTriggeredCheckpoint(handler, invokable, cancelledCheckpointId);
    final long nextCancelledCheckpointId = cancelledCheckpointId + 1L;
    // should update current checkpoint id and abort notification while processing
    // CancelCheckpointMarker
    handler.processCancellationBarrier(new CancelCheckpointMarker(nextCancelledCheckpointId), new InputChannelInfo(0, 0));
    verifyTriggeredCheckpoint(handler, invokable, nextCancelledCheckpointId);
}