Examples with DummyInvokable org.apache.flink.runtime.operators.testutils.DummyInvokable used on opensource projects

Example 46 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class ReusingBlockResettableIteratorTest method testDoubleBufferedBlockResettableIterator.

@Test
public void testDoubleBufferedBlockResettableIterator() throws Exception {
    final AbstractInvokable memOwner = new DummyInvokable();
    // create the resettable Iterator
    final ReusingBlockResettableIterator<Record> iterator = new ReusingBlockResettableIterator<Record>(this.memman, this.reader, this.serializer, 2, memOwner);
    // open the iterator
    iterator.open();
    // now test walking through the iterator
    int lower = 0;
    int upper = 0;
    do {
        lower = upper;
        upper = lower;
        // find the upper bound
        while (iterator.hasNext()) {
            Record target = iterator.next();
            int val = target.getField(0, IntValue.class).getValue();
            Assert.assertEquals(upper++, val);
        }
        // now reset the buffer a few times
        for (int i = 0; i < 5; ++i) {
            iterator.reset();
            int count = 0;
            while (iterator.hasNext()) {
                Record target = iterator.next();
                int val = target.getField(0, IntValue.class).getValue();
                Assert.assertEquals(lower + (count++), val);
            }
            Assert.assertEquals(upper - lower, count);
        }
    } while (iterator.nextBlock());
    Assert.assertEquals(NUM_VALUES, upper);
    // close the iterator
    iterator.close();
}

Example 47 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class SpillingResettableMutableObjectIteratorTest method testResettableIterator.

/**
	 * Tests the resettable iterator with too little memory, so that the data
	 * has to be written to disk.
	 */
@Test
public void testResettableIterator() {
    try {
        final AbstractInvokable memOwner = new DummyInvokable();
        // create the resettable Iterator
        SpillingResettableMutableObjectIterator<Record> iterator = new SpillingResettableMutableObjectIterator<Record>(this.reader, this.serializer, this.memman, this.ioman, 2, memOwner);
        // open the iterator
        iterator.open();
        // now test walking through the iterator
        int count = 0;
        Record target = new Record();
        while ((target = iterator.next(target)) != null) {
            Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
        }
        Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
        // test resetting the iterator a few times
        for (int j = 0; j < 10; ++j) {
            count = 0;
            iterator.reset();
            target = new Record();
            // now we should get the same results
            while ((target = iterator.next(target)) != null) {
                Assert.assertEquals("After reset nr. " + j + 1 + " element " + count + " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
            }
            Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS, count);
        }
        // close the iterator
        iterator.close();
    } catch (Exception ex) {
        ex.printStackTrace();
        Assert.fail("Test encountered an exception.");
    }
}

Example 48 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class ReusingBlockResettableIteratorTest method testSerialBlockResettableIterator.

@Test
public void testSerialBlockResettableIterator() throws Exception {
    final AbstractInvokable memOwner = new DummyInvokable();
    // create the resettable Iterator
    final ReusingBlockResettableIterator<Record> iterator = new ReusingBlockResettableIterator<Record>(this.memman, this.reader, this.serializer, 1, memOwner);
    // open the iterator
    iterator.open();
    // now test walking through the iterator
    int lower = 0;
    int upper = 0;
    do {
        lower = upper;
        upper = lower;
        // find the upper bound
        while (iterator.hasNext()) {
            Record target = iterator.next();
            int val = target.getField(0, IntValue.class).getValue();
            Assert.assertEquals(upper++, val);
        }
        // now reset the buffer a few times
        for (int i = 0; i < 5; ++i) {
            iterator.reset();
            int count = 0;
            while (iterator.hasNext()) {
                Record target = iterator.next();
                int val = target.getField(0, IntValue.class).getValue();
                Assert.assertEquals(lower + (count++), val);
            }
            Assert.assertEquals(upper - lower, count);
        }
    } while (iterator.nextBlock());
    Assert.assertEquals(NUM_VALUES, upper);
    // close the iterator
    iterator.close();
}

Example 49 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class FixedLengthRecordSorterTest method testFlushFullMemoryPage.

@Test
public void testFlushFullMemoryPage() throws Exception {
    // Insert IntPair which would fill 2 memory pages.
    final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);
    // write the records
    IntPair record = new IntPair();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record) && num < NUM_RECORDS);
    FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
    BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
    final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
    ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());
    sorter.writeToOutput(outputView, 0, NUM_RECORDS);
    this.memoryManager.release(outputView.close());
    BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
    final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
    ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
    final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
    ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);
    record = iterator.next(record);
    int i = 0;
    while (record != null) {
        Assert.assertEquals(i, record.getKey());
        record = iterator.next(record);
        i++;
    }
    Assert.assertEquals(NUM_RECORDS, i);
    this.memoryManager.release(dataBuffer);
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 50 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class FixedLengthRecordSorterTest method testSort.

@Test
public void testSort() throws Exception {
    final int NUM_RECORDS = 559273;
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
    // write the records
    IntPair record = new IntPair();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record) && num < NUM_RECORDS);
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
    MutableObjectIterator<IntPair> iter = sorter.getIterator();
    IntPair readTarget = new IntPair();
    int current;
    int last;
    iter.next(readTarget);
    last = readTarget.getKey();
    while ((readTarget = iter.next(readTarget)) != null) {
        current = readTarget.getKey();
        final int cmp = last - current;
        if (cmp > 0) {
            Assert.fail("Next key is not larger or equal to previous key.");
        }
        last = current;
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}