Examples with SequentialChannelStateReader org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReader used on opensource projects

Example 1 with SequentialChannelStateReader

use of org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReader in project flink by apache.

the class StreamTask method restoreGates.

private CompletableFuture<Void> restoreGates() throws Exception {
    SequentialChannelStateReader reader = getEnvironment().getTaskStateManager().getSequentialChannelStateReader();
    reader.readOutputData(getEnvironment().getAllWriters(), !configuration.isGraphContainingLoops());
    operatorChain.initializeStateAndOpenOperators(createStreamTaskStateInitializer());
    IndexedInputGate[] inputGates = getEnvironment().getAllInputGates();
    channelIOExecutor.execute(() -> {
        try {
            reader.readInputData(inputGates);
        } catch (Exception e) {
            asyncExceptionHandler.handleAsyncException("Unable to read channel state", e);
        }
    });
    // We wait for all input channel state to recover before we go into RUNNING state, and thus
    // start checkpointing. If we implement incremental checkpointing of input channel state
    // we must make sure it supports CheckpointType#FULL_CHECKPOINT
    List<CompletableFuture<?>> recoveredFutures = new ArrayList<>(inputGates.length);
    for (InputGate inputGate : inputGates) {
        recoveredFutures.add(inputGate.getStateConsumedFuture());
        inputGate.getStateConsumedFuture().thenRun(() -> mainMailboxExecutor.execute(inputGate::requestPartitions, "Input gate request partitions"));
    }
    return CompletableFuture.allOf(recoveredFutures.toArray(new CompletableFuture[0])).thenRun(mailboxProcessor::suspend);
}

Example 2 with SequentialChannelStateReader

use of org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReader in project flink by apache.

the class ChannelPersistenceITCase method testReadWritten.

@Test
public void testReadWritten() throws Exception {
    byte[] inputChannelInfoData = randomBytes(1024);
    byte[] resultSubpartitionInfoData = randomBytes(1024);
    int partitionIndex = 0;
    SequentialChannelStateReader reader = new SequentialChannelStateReaderImpl(toTaskStateSnapshot(write(1L, singletonMap(new InputChannelInfo(0, 0), inputChannelInfoData), singletonMap(new ResultSubpartitionInfo(partitionIndex, 0), resultSubpartitionInfoData))));
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 1024);
    try {
        int numChannels = 1;
        InputGate gate = buildGate(networkBufferPool, numChannels);
        reader.readInputData(new InputGate[] { gate });
        assertArrayEquals(inputChannelInfoData, collectBytes(gate::pollNext, BufferOrEvent::getBuffer));
        BufferWritingResultPartition resultPartition = buildResultPartition(networkBufferPool, ResultPartitionType.PIPELINED, partitionIndex, numChannels);
        reader.readOutputData(new BufferWritingResultPartition[] { resultPartition }, false);
        ResultSubpartitionView view = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        assertArrayEquals(resultSubpartitionInfoData, collectBytes(() -> Optional.ofNullable(view.getNextBuffer()), BufferAndBacklog::buffer));
    } finally {
        networkBufferPool.destroy();
    }
}