Examples with RecordingChannelStateWriter org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter used on opensource projects

Example 26 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink by splunk.

the class CheckpointedInputGateTest method testPersisting.

/**
 * This tests a scenario where an older triggered checkpoint, was cancelled and a newer
 * checkpoint was triggered very quickly after the cancellation. It can happen that a task can
 * receive first the more recent checkpoint barrier and later the obsoleted one. This can happen
 * for many reasons (for example Source tasks not running, or just a race condition with
 * notifyCheckpointAborted RPCs) and Task should be able to handle this properly. In FLINK-21104
 * the problem was that this obsoleted checkpoint barrier was causing a checkState to fail.
 */
public void testPersisting(boolean drainGate) throws Exception {
    int numberOfChannels = 3;
    NetworkBufferPool bufferPool = new NetworkBufferPool(numberOfChannels * 3, 1024);
    try {
        long checkpointId = 2L;
        long obsoleteCheckpointId = 1L;
        ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(checkpointId);
        RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
        CheckpointedInputGate gate = setupInputGateWithAlternatingController(numberOfChannels, bufferPool, validatingHandler, stateWriter);
        // enqueue first checkpointId before obsoleteCheckpointId, so that we never trigger
        // and also never cancel the obsoleteCheckpointId
        enqueue(gate, 0, buildSomeBuffer());
        enqueue(gate, 0, barrier(checkpointId));
        enqueue(gate, 0, buildSomeBuffer());
        enqueue(gate, 1, buildSomeBuffer());
        enqueue(gate, 1, barrier(obsoleteCheckpointId));
        enqueue(gate, 1, buildSomeBuffer());
        enqueue(gate, 2, buildSomeBuffer());
        assertEquals(0, validatingHandler.getTriggeredCheckpointCounter());
        // trigger checkpoint
        gate.pollNext();
        assertEquals(1, validatingHandler.getTriggeredCheckpointCounter());
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 2);
        assertAddedInputSize(stateWriter, 2, 1);
        enqueue(gate, 0, buildSomeBuffer());
        enqueue(gate, 1, buildSomeBuffer());
        enqueue(gate, 2, buildSomeBuffer());
        while (drainGate && gate.pollNext().isPresent()) {
        }
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 3);
        assertAddedInputSize(stateWriter, 2, 2);
        enqueue(gate, 1, barrier(checkpointId));
        enqueue(gate, 1, buildSomeBuffer());
        // Another obsoleted barrier that should be ignored
        enqueue(gate, 2, barrier(obsoleteCheckpointId));
        enqueue(gate, 2, buildSomeBuffer());
        while (drainGate && gate.pollNext().isPresent()) {
        }
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 3);
        assertAddedInputSize(stateWriter, 2, 3);
        enqueue(gate, 2, barrier(checkpointId));
        enqueue(gate, 2, buildSomeBuffer());
        while (drainGate && gate.pollNext().isPresent()) {
        }
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 3);
        assertAddedInputSize(stateWriter, 2, 3);
    } finally {
        bufferPool.destroy();
    }
}

Example 27 with RecordingChannelStateWriter

use of org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter in project flink-mirror by flink-ci.

the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.

private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
    ConnectionManager connectionManager = new TestingConnectionManager();
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}