Search in sources :

Example 56 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class FileChannelStreamsTest method testCloseAndDeleteInputView.

@Test
public void testCloseAndDeleteInputView() {
    try (IOManager ioManager = new IOManagerAsync()) {
        MemoryManager memMan = MemoryManagerBuilder.newBuilder().build();
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        // add some test data
        try (FileWriter wrt = new FileWriter(channel.getPath())) {
            wrt.write("test data");
        }
        BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
        // read just something
        in.readInt();
        // close for the first time, make sure all memory returns
        in.close();
        assertTrue(memMan.verifyEmpty());
        // close again, should not cause an exception
        in.close();
        // delete, make sure file is removed
        in.closeAndDelete();
        assertFalse(new File(channel.getPath()).exists());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
Also used : IOManager(org.apache.flink.runtime.io.disk.iomanager.IOManager) FileWriter(java.io.FileWriter) ArrayList(java.util.ArrayList) FileIOChannel(org.apache.flink.runtime.io.disk.iomanager.FileIOChannel) MemoryManager(org.apache.flink.runtime.memory.MemoryManager) MemorySegment(org.apache.flink.core.memory.MemorySegment) IOManagerAsync(org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync) DummyInvokable(org.apache.flink.runtime.operators.testutils.DummyInvokable) File(java.io.File) Test(org.junit.Test)

Example 57 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class SeekableFileChannelInputViewTest method testSeek.

@Test
public void testSeek() {
    final int PAGE_SIZE = 16 * 1024;
    final int NUM_RECORDS = 120000;
    try (IOManager ioManager = new IOManagerAsync()) {
        MemoryManager memMan = MemoryManagerBuilder.newBuilder().setMemorySize(4 * PAGE_SIZE).setPageSize(PAGE_SIZE).build();
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
        FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
        // bytes)
        for (int i = 0; i < NUM_RECORDS; i += 4) {
            out.writeInt(i);
        }
        // close for the first time, make sure all memory returns
        out.close();
        assertTrue(memMan.verifyEmpty());
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        SeekableFileChannelInputView in = new SeekableFileChannelInputView(ioManager, channel, memMan, memory, out.getBytesInLatestSegment());
        // read first, complete
        for (int i = 0; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the middle of the 3rd page
        int i = 2 * PAGE_SIZE + PAGE_SIZE / 4;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the end
        i = 120000 - 4;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the beginning
        i = 0;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to after a page
        i = PAGE_SIZE;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to after a page
        i = 3 * PAGE_SIZE;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the end
        i = NUM_RECORDS;
        in.seek(i);
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek out of bounds
        try {
            in.seek(-10);
            fail("should throw an exception");
        } catch (IllegalArgumentException ignored) {
        }
        try {
            in.seek(NUM_RECORDS + 1);
            fail("should throw an exception");
        } catch (IllegalArgumentException ignored) {
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
Also used : IOManager(org.apache.flink.runtime.io.disk.iomanager.IOManager) ArrayList(java.util.ArrayList) FileIOChannel(org.apache.flink.runtime.io.disk.iomanager.FileIOChannel) MemoryManager(org.apache.flink.runtime.memory.MemoryManager) MemorySegment(org.apache.flink.core.memory.MemorySegment) EOFException(java.io.EOFException) IOManagerAsync(org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync) EOFException(java.io.EOFException) DummyInvokable(org.apache.flink.runtime.operators.testutils.DummyInvokable) Test(org.junit.Test)

Example 58 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BufferReaderWriterUtilTest method readPrematureEndOfFile1.

@Test
public void readPrematureEndOfFile1() throws Exception {
    final FileChannel fc = tmpFileChannel();
    final Buffer buffer = createTestBuffer();
    final MemorySegment readBuffer = MemorySegmentFactory.allocateUnpooledOffHeapMemory(buffer.getSize(), null);
    BufferReaderWriterUtil.writeToByteChannel(fc, buffer, BufferReaderWriterUtil.allocatedWriteBufferArray());
    fc.truncate(fc.position() - 1);
    fc.position(0);
    try {
        BufferReaderWriterUtil.readFromByteChannel(fc, BufferReaderWriterUtil.allocatedHeaderBuffer(), readBuffer, FreeingBufferRecycler.INSTANCE);
        fail();
    } catch (IOException e) {
    // expected
    }
}
Also used : Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) ByteBuffer(java.nio.ByteBuffer) FileChannel(java.nio.channels.FileChannel) IOException(java.io.IOException) MemorySegment(org.apache.flink.core.memory.MemorySegment) Test(org.junit.Test)

Example 59 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BufferReaderWriterUtilTest method readPrematureEndOfFile2.

@Test
public void readPrematureEndOfFile2() throws Exception {
    final FileChannel fc = tmpFileChannel();
    final Buffer buffer = createTestBuffer();
    final MemorySegment readBuffer = MemorySegmentFactory.allocateUnpooledOffHeapMemory(buffer.getSize(), null);
    BufferReaderWriterUtil.writeToByteChannel(fc, buffer, BufferReaderWriterUtil.allocatedWriteBufferArray());
    // less than a header size
    fc.truncate(2);
    fc.position(0);
    try {
        BufferReaderWriterUtil.readFromByteChannel(fc, BufferReaderWriterUtil.allocatedHeaderBuffer(), readBuffer, FreeingBufferRecycler.INSTANCE);
        fail();
    } catch (IOException e) {
    // expected
    }
}
Also used : Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) ByteBuffer(java.nio.ByteBuffer) FileChannel(java.nio.channels.FileChannel) IOException(java.io.IOException) MemorySegment(org.apache.flink.core.memory.MemorySegment) Test(org.junit.Test)

Example 60 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class DataBufferTest method checkReadResult.

private void checkReadResult(DataBuffer dataBuffer, ByteBuffer expectedBuffer, int expectedChannel, int bufferSize) {
    MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(bufferSize);
    BufferWithChannel bufferWithChannel = dataBuffer.getNextBuffer(segment);
    assertEquals(expectedChannel, bufferWithChannel.getChannelIndex());
    assertEquals(expectedBuffer, bufferWithChannel.getBuffer().getNioBufferReadable());
}
Also used : MemorySegment(org.apache.flink.core.memory.MemorySegment)

Aggregations

MemorySegment (org.apache.flink.core.memory.MemorySegment)375 Test (org.junit.Test)136 ArrayList (java.util.ArrayList)52 DummyInvokable (org.apache.flink.runtime.operators.testutils.DummyInvokable)44 IOException (java.io.IOException)37 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)29 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)26 NetworkBuffer (org.apache.flink.runtime.io.network.buffer.NetworkBuffer)25 MemoryAllocationException (org.apache.flink.runtime.memory.MemoryAllocationException)24 IntPair (org.apache.flink.runtime.operators.testutils.types.IntPair)24 FileIOChannel (org.apache.flink.runtime.io.disk.iomanager.FileIOChannel)20 EOFException (java.io.EOFException)18 ByteBuffer (java.nio.ByteBuffer)18 AbstractInvokable (org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable)18 TestData (org.apache.flink.runtime.operators.testutils.TestData)18 Random (java.util.Random)16 UniformIntPairGenerator (org.apache.flink.runtime.operators.testutils.UniformIntPairGenerator)16 Chunk (org.apache.flink.runtime.state.heap.space.Chunk)15 BinaryRowData (org.apache.flink.table.data.binary.BinaryRowData)15 IOManagerAsync (org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync)14