Search in sources :

Example 1 with SequentialChannelStateReaderImpl

use of org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReaderImpl in project flink by apache.

the class ChannelPersistenceITCase method upstreamBlocksAfterRecoveringState.

private void upstreamBlocksAfterRecoveringState(ResultPartitionType type) throws Exception {
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 1024);
    byte[] dataAfterRecovery = randomBytes(1024);
    try {
        BufferWritingResultPartition resultPartition = buildResultPartition(networkBufferPool, type, 0, 1);
        new SequentialChannelStateReaderImpl(new TaskStateSnapshot()).readOutputData(new BufferWritingResultPartition[] { resultPartition }, true);
        resultPartition.emitRecord(ByteBuffer.wrap(dataAfterRecovery), 0);
        ResultSubpartitionView view = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        if (type != ResultPartitionType.PIPELINED_APPROXIMATE) {
            assertEquals(RECOVERY_COMPLETION, view.getNextBuffer().buffer().getDataType());
            assertNull(view.getNextBuffer());
            view.resumeConsumption();
        }
        assertArrayEquals(dataAfterRecovery, collectBytes(view.getNextBuffer().buffer()));
    } finally {
        networkBufferPool.destroy();
    }
}
Also used : TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) SequentialChannelStateReaderImpl(org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReaderImpl) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)

Example 2 with SequentialChannelStateReaderImpl

use of org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReaderImpl in project flink by apache.

the class ChannelPersistenceITCase method testReadWritten.

@Test
public void testReadWritten() throws Exception {
    byte[] inputChannelInfoData = randomBytes(1024);
    byte[] resultSubpartitionInfoData = randomBytes(1024);
    int partitionIndex = 0;
    SequentialChannelStateReader reader = new SequentialChannelStateReaderImpl(toTaskStateSnapshot(write(1L, singletonMap(new InputChannelInfo(0, 0), inputChannelInfoData), singletonMap(new ResultSubpartitionInfo(partitionIndex, 0), resultSubpartitionInfoData))));
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 1024);
    try {
        int numChannels = 1;
        InputGate gate = buildGate(networkBufferPool, numChannels);
        reader.readInputData(new InputGate[] { gate });
        assertArrayEquals(inputChannelInfoData, collectBytes(gate::pollNext, BufferOrEvent::getBuffer));
        BufferWritingResultPartition resultPartition = buildResultPartition(networkBufferPool, ResultPartitionType.PIPELINED, partitionIndex, numChannels);
        reader.readOutputData(new BufferWritingResultPartition[] { resultPartition }, false);
        ResultSubpartitionView view = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        assertArrayEquals(resultSubpartitionInfoData, collectBytes(() -> Optional.ofNullable(view.getNextBuffer()), BufferAndBacklog::buffer));
    } finally {
        networkBufferPool.destroy();
    }
}
Also used : InputChannelInfo(org.apache.flink.runtime.checkpoint.channel.InputChannelInfo) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) SequentialChannelStateReaderImpl(org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReaderImpl) ResultSubpartitionInfo(org.apache.flink.runtime.checkpoint.channel.ResultSubpartitionInfo) SequentialChannelStateReader(org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReader) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) Test(org.junit.Test)

Aggregations

SequentialChannelStateReaderImpl (org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReaderImpl)2 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)2 BufferWritingResultPartition (org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition)2 NoOpBufferAvailablityListener (org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener)2 ResultSubpartitionView (org.apache.flink.runtime.io.network.partition.ResultSubpartitionView)2 TaskStateSnapshot (org.apache.flink.runtime.checkpoint.TaskStateSnapshot)1 InputChannelInfo (org.apache.flink.runtime.checkpoint.channel.InputChannelInfo)1 ResultSubpartitionInfo (org.apache.flink.runtime.checkpoint.channel.ResultSubpartitionInfo)1 SequentialChannelStateReader (org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReader)1 InputGate (org.apache.flink.runtime.io.network.partition.consumer.InputGate)1 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)1 Test (org.junit.Test)1