Examples with PairGenerator org.apache.flink.runtime.operators.testutils.PairGenerator used on opensource projects

Example 1 with PairGenerator

use of org.apache.flink.runtime.operators.testutils.PairGenerator in project flink by apache.

the class FileChannelStreamsITCase method testReadTooMany.

@Test
public void testReadTooMany() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        try {
            Pair readPair = new Pair();
            for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
                generator.next(pair);
                readPair.read(inView);
                assertEquals("The re-generated and the read record do not match.", pair, readPair);
            }
            fail("Expected an EOFException which did not occur.");
        } catch (EOFException eofex) {
        // expected
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with PairGenerator

use of org.apache.flink.runtime.operators.testutils.PairGenerator in project flink by apache.

the class FileChannelStreamsITCase method testWriteAndReadLongRecords.

@Test
public void testWriteAndReadLongRecords() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_LONG; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        Pair readPair = new Pair();
        for (int i = 0; i < NUM_PAIRS_LONG; i++) {
            generator.next(pair);
            readPair.read(inView);
            assertEquals("The re-generated and the read record do not match.", pair, readPair);
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with PairGenerator

use of org.apache.flink.runtime.operators.testutils.PairGenerator in project flink by apache.

the class FileChannelStreamsITCase method testWriteReadOneBufferOnly.

@Test
public void testWriteReadOneBufferOnly() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), 1);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), 1);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        Pair readPair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            readPair.read(inView);
            assertEquals("The re-generated and the read record do not match.", pair, readPair);
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with PairGenerator

use of org.apache.flink.runtime.operators.testutils.PairGenerator in project flink by apache.

the class FileChannelStreamsITCase method testWriteReadSmallRecords.

// --------------------------------------------------------------------------------------------
@Test
public void testWriteReadSmallRecords() {
    try {
        List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        Pair readPair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            readPair.read(inView);
            assertEquals("The re-generated and the read record do not match.", pair, readPair);
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 5 with PairGenerator

use of org.apache.flink.runtime.operators.testutils.PairGenerator in project flink by apache.

the class FileChannelStreamsITCase method testWriteReadNotAll.

@Test
public void testWriteReadNotAll() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        Pair readPair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {
            generator.next(pair);
            readPair.read(inView);
            assertEquals("The re-generated and the read record do not match.", pair, readPair);
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}