Examples with DummyInvokable org.apache.flink.runtime.operators.testutils.DummyInvokable used on opensource projects

Example 41 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class NormalizedKeySorterTest method testSort.

@Test
public void testSort() throws Exception {
    final int NUM_RECORDS = 559273;
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
    TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    // write the records
    Tuple2<Integer, String> record = new Tuple2<>();
    int num = 0;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record) && num < NUM_RECORDS);
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
    MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
    Tuple2<Integer, String> readTarget = new Tuple2<>();
    iter.next(readTarget);
    int last = readTarget.f0;
    while ((readTarget = iter.next(readTarget)) != null) {
        int current = readTarget.f0;
        final int cmp = last - current;
        if (cmp > 0) {
            Assert.fail("Next key is not larger or equal to previous key.");
        }
        last = current;
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 42 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class NormalizedKeySorterTest method testCompare.

/**
	 * The compare test creates a sorted stream, writes it to the buffer and
	 * compares random elements. It expects that earlier elements are lower than later
	 * ones.
	 */
@Test
public void testCompare() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
    TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED, ValueMode.RANDOM_LENGTH);
    // write the records
    Tuple2<Integer, String> record = new Tuple2<>();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record));
    // compare random elements
    Random rnd = new Random(SEED << 1);
    for (int i = 0; i < 2 * num; i++) {
        int pos1 = rnd.nextInt(num);
        int pos2 = rnd.nextInt(num);
        int cmp = sorter.compare(pos1, pos2);
        if (pos1 < pos2) {
            Assert.assertTrue(cmp <= 0);
        } else {
            Assert.assertTrue(cmp >= 0);
        }
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 43 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class NormalizedKeySorterTest method testReset.

@Test
public void testReset() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
    TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
    // write the buffer full with the first set of records
    Tuple2<Integer, String> record = new Tuple2<>();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record));
    sorter.reset();
    // write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
    generator = new TestData.TupleGenerator(SEED2, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
    // write the buffer full with the first set of records
    int num2 = -1;
    do {
        generator.next(record);
        num2++;
    } while (sorter.write(record));
    Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
    // re-read the records
    generator.reset();
    Tuple2<Integer, String> readTarget = new Tuple2<>();
    int i = 0;
    while (i < num) {
        generator.next(record);
        readTarget = sorter.getRecord(readTarget, i++);
        int rk = readTarget.f0;
        int gk = record.f0;
        String rv = readTarget.f1;
        String gv = record.f1;
        Assert.assertEquals("The re-read key is wrong", gk, rk);
        Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 44 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class NormalizedKeySorterTest method testSortLongStringKeys.

@Test
public void testSortLongStringKeys() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    @SuppressWarnings("unchecked") TypeComparator<Tuple2<Integer, String>> accessors = TestData.getIntStringTupleTypeInfo().createComparator(new int[] { 1 }, new boolean[] { true }, 0, null);
    NormalizedKeySorter<Tuple2<Integer, String>> sorter = new NormalizedKeySorter<>(TestData.getIntStringTupleSerializer(), accessors, memory);
    TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
    // write the records
    Tuple2<Integer, String> record = new Tuple2<>();
    do {
        generator.next(record);
    } while (sorter.write(record));
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
    MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
    Tuple2<Integer, String> readTarget = new Tuple2<>();
    iter.next(readTarget);
    String last = readTarget.f1;
    while ((readTarget = iter.next(readTarget)) != null) {
        String current = readTarget.f1;
        final int cmp = last.compareTo(current);
        if (cmp > 0) {
            Assert.fail("Next value is not larger or equal to previous value.");
        }
        last = current;
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 45 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class NormalizedKeySorterTest method testWriteAndRead.

@Test
public void testWriteAndRead() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
    TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    // write the records
    Tuple2<Integer, String> record = new Tuple2<>();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record));
    // re-read the records
    generator.reset();
    Tuple2<Integer, String> readTarget = new Tuple2<>();
    int i = 0;
    while (i < num) {
        generator.next(record);
        readTarget = sorter.getRecord(readTarget, i++);
        int rk = readTarget.f0;
        int gk = record.f0;
        String rv = readTarget.f1;
        String gv = record.f1;
        Assert.assertEquals("The re-read key is wrong", gk, rk);
        Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}