Examples with DummyInvokable org.apache.flink.runtime.operators.testutils.DummyInvokable used on opensource projects

Example 6 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class FileChannelStreamsITCase method testWriteAndReadLongRecords.

@Test
public void testWriteAndReadLongRecords() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_LONG; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        Pair readPair = new Pair();
        for (int i = 0; i < NUM_PAIRS_LONG; i++) {
            generator.next(pair);
            readPair.read(inView);
            assertEquals("The re-generated and the read record do not match.", pair, readPair);
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class FileChannelStreamsITCase method testWriteReadOneBufferOnly.

@Test
public void testWriteReadOneBufferOnly() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), 1);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), 1);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        Pair readPair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            readPair.read(inView);
            assertEquals("The re-generated and the read record do not match.", pair, readPair);
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 8 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class FileChannelStreamsTest method testCloseAndDeleteInputView.

@Test
public void testCloseAndDeleteInputView() {
    final IOManager ioManager = new IOManagerAsync();
    try {
        MemoryManager memMan = new MemoryManager(4 * 16 * 1024, 1, 16 * 1024, MemoryType.HEAP, true);
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        // add some test data
        try (FileWriter wrt = new FileWriter(channel.getPath())) {
            wrt.write("test data");
        }
        BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
        // read just something
        in.readInt();
        // close for the first time, make sure all memory returns
        in.close();
        assertTrue(memMan.verifyEmpty());
        // close again, should not cause an exception
        in.close();
        // delete, make sure file is removed
        in.closeAndDelete();
        assertFalse(new File(channel.getPath()).exists());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioManager.shutdown();
    }
}

Example 9 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class SeekableFileChannelInputViewTest method testSeek.

@Test
public void testSeek() {
    final IOManager ioManager = new IOManagerAsync();
    final int PAGE_SIZE = 16 * 1024;
    final int NUM_RECORDS = 120000;
    try {
        MemoryManager memMan = new MemoryManager(4 * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
        FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
        // write some integers across 7.5 pages (7 pages = 114.688 bytes, 8 pages = 131.072 bytes)
        for (int i = 0; i < NUM_RECORDS; i += 4) {
            out.writeInt(i);
        }
        // close for the first time, make sure all memory returns
        out.close();
        assertTrue(memMan.verifyEmpty());
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        SeekableFileChannelInputView in = new SeekableFileChannelInputView(ioManager, channel, memMan, memory, out.getBytesInLatestSegment());
        // read first, complete
        for (int i = 0; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the middle of the 3rd page
        int i = 2 * PAGE_SIZE + PAGE_SIZE / 4;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the end
        i = 120000 - 4;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the beginning
        i = 0;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to after a page
        i = PAGE_SIZE;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to after a page
        i = 3 * PAGE_SIZE;
        in.seek(i);
        for (; i < NUM_RECORDS; i += 4) {
            assertEquals(i, in.readInt());
        }
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek to the end
        i = NUM_RECORDS;
        in.seek(i);
        try {
            in.readInt();
            fail("should throw EOF exception");
        } catch (EOFException ignored) {
        }
        // seek out of bounds
        try {
            in.seek(-10);
            fail("should throw an exception");
        } catch (IllegalArgumentException ignored) {
        }
        try {
            in.seek(NUM_RECORDS + 1);
            fail("should throw an exception");
        } catch (IllegalArgumentException ignored) {
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioManager.shutdown();
    }
}

Example 10 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MassiveStringSorting method testStringSorting.

public void testStringSorting() {
    File input = null;
    File sorted = null;
    try {
        // the source file
        input = generateFileWithStrings(300000, "http://some-uri.com/that/is/a/common/prefix/to/all");
        // the sorted file
        sorted = File.createTempFile("sorted_strings", "txt");
        String[] command = { "/bin/bash", "-c", "export LC_ALL=\"C\" && cat \"" + input.getAbsolutePath() + "\" | sort > \"" + sorted.getAbsolutePath() + "\"" };
        Process p = null;
        try {
            p = Runtime.getRuntime().exec(command);
            int retCode = p.waitFor();
            if (retCode != 0) {
                throw new Exception("Command failed with return code " + retCode);
            }
            p = null;
        } finally {
            if (p != null) {
                p.destroy();
            }
        }
        // sort the data
        UnilateralSortMerger<String> sorter = null;
        BufferedReader reader = null;
        BufferedReader verifyReader = null;
        try {
            MemoryManager mm = new MemoryManager(1024 * 1024, 1);
            IOManager ioMan = new IOManagerAsync();
            TypeSerializer<String> serializer = StringSerializer.INSTANCE;
            TypeComparator<String> comparator = new StringComparator(true);
            reader = new BufferedReader(new FileReader(input));
            MutableObjectIterator<String> inputIterator = new StringReaderMutableObjectIterator(reader);
            sorter = new UnilateralSortMerger<String>(mm, ioMan, inputIterator, new DummyInvokable(), new RuntimeSerializerFactory<String>(serializer, String.class), comparator, 1.0, 4, 0.8f, true, /* use large record handler */
            false);
            MutableObjectIterator<String> sortedData = sorter.getIterator();
            reader.close();
            // verify
            verifyReader = new BufferedReader(new FileReader(sorted));
            String next;
            while ((next = verifyReader.readLine()) != null) {
                String nextFromStratoSort = sortedData.next("");
                Assert.assertNotNull(nextFromStratoSort);
                Assert.assertEquals(next, nextFromStratoSort);
            }
        } finally {
            if (reader != null) {
                reader.close();
            }
            if (verifyReader != null) {
                verifyReader.close();
            }
            if (sorter != null) {
                sorter.close();
            }
        }
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        if (input != null) {
            input.delete();
        }
        if (sorted != null) {
            sorted.delete();
        }
    }
}