Search in sources :

Example 1 with EXCLUSIVE

use of org.apache.flink.runtime.state.CheckpointedStateScope.EXCLUSIVE in project flink by apache.

the class ChannelStateCheckpointWriterTest method testFileHandleSize.

@Test
public void testFileHandleSize() throws Exception {
    int numChannels = 3;
    int numWritesPerChannel = 4;
    int numBytesPerWrite = 5;
    ChannelStateWriteResult result = new ChannelStateWriteResult();
    ChannelStateCheckpointWriter writer = createWriter(result, new FsCheckpointStreamFactory(getSharedInstance(), fromLocalFile(temporaryFolder.newFolder("checkpointsDir")), fromLocalFile(temporaryFolder.newFolder("sharedStateDir")), numBytesPerWrite - 1, numBytesPerWrite - 1).createCheckpointStateOutputStream(EXCLUSIVE));
    InputChannelInfo[] channels = IntStream.range(0, numChannels).mapToObj(i -> new InputChannelInfo(0, i)).toArray(InputChannelInfo[]::new);
    for (int call = 0; call < numWritesPerChannel; call++) {
        for (int channel = 0; channel < numChannels; channel++) {
            write(writer, channels[channel], getData(numBytesPerWrite));
        }
    }
    writer.completeInput();
    writer.completeOutput();
    for (InputChannelStateHandle handle : result.inputChannelStateHandles.get()) {
        assertEquals((Integer.BYTES + numBytesPerWrite) * numWritesPerChannel, handle.getStateSize());
    }
}
Also used : IntStream(java.util.stream.IntStream) MemorySegmentFactory.wrap(org.apache.flink.core.memory.MemorySegmentFactory.wrap) ByteArrayOutputStream(java.io.ByteArrayOutputStream) EXCLUSIVE(org.apache.flink.runtime.state.CheckpointedStateScope.EXCLUSIVE) HashMap(java.util.HashMap) Random(java.util.Random) RunnableWithException(org.apache.flink.util.function.RunnableWithException) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) Collections.singletonList(java.util.Collections.singletonList) DataOutputStream(java.io.DataOutputStream) MemoryCheckpointOutputStream(org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory.MemoryCheckpointOutputStream) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) MemorySegment(org.apache.flink.core.memory.MemorySegment) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) ChannelStateWriteResult(org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter.ChannelStateWriteResult) StreamStateHandle(org.apache.flink.runtime.state.StreamStateHandle) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) File(java.io.File) Rule(org.junit.Rule) FsCheckpointStreamFactory(org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory) Assert.assertFalse(org.junit.Assert.assertFalse) TemporaryFolder(org.junit.rules.TemporaryFolder) LocalFileSystem.getSharedInstance(org.apache.flink.core.fs.local.LocalFileSystem.getSharedInstance) Assert.assertEquals(org.junit.Assert.assertEquals) CheckpointStateOutputStream(org.apache.flink.runtime.state.CheckpointStateOutputStream) Path.fromLocalFile(org.apache.flink.core.fs.Path.fromLocalFile) FsCheckpointStreamFactory(org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory) ChannelStateWriteResult(org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter.ChannelStateWriteResult) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Test(org.junit.Test)

Aggregations

ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 DataOutputStream (java.io.DataOutputStream)1 File (java.io.File)1 IOException (java.io.IOException)1 Collections.singletonList (java.util.Collections.singletonList)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 Random (java.util.Random)1 IntStream (java.util.stream.IntStream)1 Path.fromLocalFile (org.apache.flink.core.fs.Path.fromLocalFile)1 LocalFileSystem.getSharedInstance (org.apache.flink.core.fs.local.LocalFileSystem.getSharedInstance)1 MemorySegment (org.apache.flink.core.memory.MemorySegment)1 MemorySegmentFactory (org.apache.flink.core.memory.MemorySegmentFactory)1 MemorySegmentFactory.wrap (org.apache.flink.core.memory.MemorySegmentFactory.wrap)1 ChannelStateWriteResult (org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter.ChannelStateWriteResult)1 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)1 FreeingBufferRecycler (org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler)1 NetworkBuffer (org.apache.flink.runtime.io.network.buffer.NetworkBuffer)1 CheckpointStateOutputStream (org.apache.flink.runtime.state.CheckpointStateOutputStream)1 EXCLUSIVE (org.apache.flink.runtime.state.CheckpointedStateScope.EXCLUSIVE)1