Examples with IntPair org.apache.flink.runtime.operators.testutils.types.IntPair used on opensource projects

Example 26 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class MutableHashTableTestBase method getRandomizedIntPairs.

protected static IntPair[] getRandomizedIntPairs(int num, Random rnd) {
    IntPair[] pairs = new IntPair[num];
    // create all the pairs, dense
    for (int i = 0; i < num; i++) {
        pairs[i] = new IntPair(i, i + KEY_VALUE_DIFF);
    }
    // randomly swap them
    for (int i = 0; i < 2 * num; i++) {
        int pos1 = rnd.nextInt(num);
        int pos2 = rnd.nextInt(num);
        IntPair tmp = pairs[pos1];
        pairs[pos1] = pairs[pos2];
        pairs[pos2] = tmp;
    }
    return pairs;
}

Example 27 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class HashTableITCase method testHashWithBuildSideOuterJoin2.

@Test
public void testHashWithBuildSideOuterJoin2() throws Exception {
    final int NUM_KEYS = 40000;
    final int BUILD_VALS_PER_KEY = 2;
    final int PROBE_VALS_PER_KEY = 1;
    // The keys of probe and build sides are overlapped, so there would be none unmatched build elements
    // after probe phase, make sure build side outer join works well in this case.
    // create a build input that gives 80000 pairs with 2 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    // create a probe input that gives 40000 pairs with 1 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        // 33 is minimum number of pages required to perform hash join this inputs
        memSegments = this.memManager.allocatePages(MEM_OWNER, 33);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, probeInput, true);
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
    while (join.nextRecord()) {
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        IntPair next = buildSide.next(recordReuse);
        if (next == null && join.getCurrentProbeRecord() == null) {
            fail("Should not return join result that both probe and build element are null.");
        }
        while (next != null) {
            numRecordsInJoinResult++;
            next = buildSide.next(recordReuse);
        }
    }
    Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);
    join.close();
    this.memManager.release(join.getFreedMemory());
}

Example 28 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class HashTableITCase method testInMemoryReOpenWithSmallMemory.

/*
	 * This test is same as `testInMemoryReOpen()` but only number of keys and pages are different. This test
	 * validates a bug fix MutableHashTable memory leakage with small memory segments.
	 */
@Test
public void testInMemoryReOpenWithSmallMemory() throws Exception {
    final int NUM_KEYS = 10000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 30000 pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    // create a probe input that gives 100000 pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        // 33 is minimum number of pages required to perform hash join this inputs
        memSegments = this.memManager.allocatePages(MEM_OWNER, 33);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
    while (join.nextRecord()) {
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        while (buildSide.next(recordReuse) != null) {
            numRecordsInJoinResult++;
        }
    }
    Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);
    join.close();
    // ----------------------------------------------------------------------------------------
    // recreate the inputs
    // create a build input that gives 30000 pairs with 3 values sharing the same key
    buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    // create a probe input that gives 100000 pairs with 10 values sharing a key
    probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    join.open(buildInput, probeInput);
    numRecordsInJoinResult = 0;
    while (join.nextRecord()) {
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        while (buildSide.next(recordReuse) != null) {
            numRecordsInJoinResult++;
        }
    }
    Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);
    join.close();
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}

Example 29 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class HashTableITCase method testSparseProbeSpillingIntPair.

/*
	 * Spills build records, so that probe records are also spilled. But only so
	 * few probe records are used that some partitions remain empty.
	 */
@Test
public void testSparseProbeSpillingIntPair() throws IOException, MemoryAllocationException {
    final int NUM_BUILD_KEYS = 1000000;
    final int NUM_BUILD_VALS = 1;
    final int NUM_PROBE_KEYS = 20;
    final int NUM_PROBE_VALS = 1;
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_BUILD_KEYS, NUM_BUILD_VALS, false);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 128);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, new UniformIntPairGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));
    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS) * NUM_PROBE_VALS;
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
    while (join.nextRecord()) {
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        while (buildSide.next(recordReuse) != null) {
            numRecordsInJoinResult++;
        }
    }
    Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);
    join.close();
    this.memManager.release(join.getFreedMemory());
}

Example 30 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class HashTableITCase method testHashWithBuildSideOuterJoin1.

@Test
public void testHashWithBuildSideOuterJoin1() throws Exception {
    final int NUM_KEYS = 20000;
    final int BUILD_VALS_PER_KEY = 1;
    final int PROBE_VALS_PER_KEY = 1;
    // create a build input that gives 40000 pairs with 1 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(2 * NUM_KEYS, BUILD_VALS_PER_KEY, false);
    // create a probe input that gives 20000 pairs with 1 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        // 33 is minimum number of pages required to perform hash join this inputs
        memSegments = this.memManager.allocatePages(MEM_OWNER, 33);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, probeInput, true);
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
    while (join.nextRecord()) {
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        while (buildSide.next(recordReuse) != null) {
            numRecordsInJoinResult++;
        }
    }
    Assert.assertEquals("Wrong number of records in join result.", 2 * NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);
    join.close();
    this.memManager.release(join.getFreedMemory());
}