Examples with RandomIntPairGenerator org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator used on opensource projects

Example 1 with RandomIntPairGenerator

use of org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator in project flink by apache.

the class FixedLengthRecordSorterTest method testReset.

@Test
public void testReset() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
    // write the buffer full with the first set of records
    IntPair record = new IntPair();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record) && num < 3354624);
    sorter.reset();
    // write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
    generator.reset();
    // write the buffer full with the first set of records
    int num2 = -1;
    do {
        generator.next(record);
        num2++;
    } while (sorter.write(record) && num2 < 3354624);
    Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
    // re-read the records
    generator.reset();
    IntPair readTarget = new IntPair();
    int i = 0;
    while (i < num) {
        generator.next(record);
        readTarget = sorter.getRecord(readTarget, i++);
        int rk = readTarget.getKey();
        int gk = record.getKey();
        int rv = readTarget.getValue();
        int gv = record.getValue();
        Assert.assertEquals("The re-read key is wrong", gk, rk);
        Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 2 with RandomIntPairGenerator

use of org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator in project flink by apache.

the class FixedLengthRecordSorterTest method testSwap.

/**
	 * The swap test fills the sort buffer and swaps all elements such that they are
	 * backwards. It then resets the generator, goes backwards through the buffer
	 * and compares for equality.
	 */
@Test
public void testSwap() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
    // write the records
    IntPair record = new IntPair();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record) && num < 3354624);
    // swap the records
    int start = 0, end = num - 1;
    while (start < end) {
        sorter.swap(start++, end--);
    }
    // re-read the records
    generator.reset();
    IntPair readTarget = new IntPair();
    int i = num - 1;
    while (i >= 0) {
        generator.next(record);
        readTarget = sorter.getRecord(readTarget, i--);
        int rk = readTarget.getKey();
        int gk = record.getKey();
        int rv = readTarget.getValue();
        int gv = record.getValue();
        Assert.assertEquals("The re-read key is wrong", gk, rk);
        Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 3 with RandomIntPairGenerator

use of org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator in project flink by apache.

the class FixedLengthRecordSorterTest method testWriteAndRead.

@Test
public void testWriteAndRead() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
    //		long startTime = System.currentTimeMillis();
    // write the records
    IntPair record = new IntPair();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record) && num < 3354624);
    //		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
    // re-read the records
    generator.reset();
    IntPair readTarget = new IntPair();
    //		startTime = System.currentTimeMillis();
    int i = 0;
    while (i < num) {
        generator.next(record);
        readTarget = sorter.getRecord(readTarget, i++);
        int rk = readTarget.getKey();
        int gk = record.getKey();
        int rv = readTarget.getValue();
        int gv = record.getValue();
        if (gk != rk) {
            Assert.fail("The re-read key is wrong " + i);
        }
        if (gv != rv) {
            Assert.fail("The re-read value is wrong");
        }
    }
    //		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
    //		System.out.println("RECORDS " + num);
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 4 with RandomIntPairGenerator

use of org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator in project flink by apache.

the class FixedLengthRecordSorterTest method testWriteAndIterator.

@Test
public void testWriteAndIterator() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
    RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
    // write the records
    IntPair record = new IntPair();
    int num = -1;
    do {
        generator.next(record);
        num++;
    } while (sorter.write(record));
    // re-read the records
    generator.reset();
    MutableObjectIterator<IntPair> iter = sorter.getIterator();
    IntPair readTarget = new IntPair();
    int count = 0;
    while ((readTarget = iter.next(readTarget)) != null) {
        count++;
        generator.next(record);
        int rk = readTarget.getKey();
        int gk = record.getKey();
        int rv = readTarget.getValue();
        int gv = record.getValue();
        Assert.assertEquals("The re-read key is wrong", gk, rk);
        Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
    Assert.assertEquals("Incorrect number of records", num, count);
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}

Example 5 with RandomIntPairGenerator

use of org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator in project flink by apache.

the class ExternalSortITCase method testSpillingSortWithIntermediateMergeIntPair.

@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
    try {
        // amount of pairs
        final int PAIRS = 50000000;
        // comparator
        final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
        final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
        final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
        // merge iterator
        LOG.debug("Initializing sortmerger...");
        Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, generator, this.parentTask, serializerFactory, comparator, (double) 64 / 78, 4, 0.7f, true, /*use large record handler*/
        true);
        // emit data
        LOG.debug("Emitting data...");
        // check order
        MutableObjectIterator<IntPair> iterator = merger.getIterator();
        LOG.debug("Checking results...");
        int pairsRead = 1;
        int nextStep = PAIRS / 20;
        IntPair rec1 = new IntPair();
        IntPair rec2 = new IntPair();
        Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
        while ((rec2 = iterator.next(rec2)) != null) {
            final int k1 = rec1.getKey();
            final int k2 = rec2.getKey();
            pairsRead++;
            Assert.assertTrue(k1 - k2 <= 0);
            IntPair tmp = rec1;
            rec1 = rec2;
            rec2 = tmp;
            // log
            if (pairsRead == nextStep) {
                nextStep += PAIRS / 20;
            }
        }
        Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
        merger.close();
        testSuccess = true;
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}