Examples with IntPair org.apache.flink.runtime.operators.testutils.types.IntPair used on opensource projects

Example 1 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class MutableHashTableTestBase method testBuildAndRetrieve.

@Test
public void testBuildAndRetrieve() throws Exception {
    final int NUM_MEM_PAGES = 32 * NUM_PAIRS / PAGE_SIZE;
    AbstractMutableHashTable<IntPair> table = getHashTable(intPairSerializer, intPairComparator, getMemory(NUM_MEM_PAGES));
    final Random rnd = new Random(RANDOM_SEED);
    final IntPair[] pairs = getRandomizedIntPairs(NUM_PAIRS, rnd);
    table.open();
    for (int i = 0; i < NUM_PAIRS; i++) {
        table.insert(pairs[i]);
    }
    AbstractHashTableProber<IntPair, IntPair> prober = table.getProber(intPairComparator, pairComparator);
    IntPair target = new IntPair();
    for (int i = 0; i < NUM_PAIRS; i++) {
        assertNotNull(prober.getMatchFor(pairs[i], target));
        assertEquals(pairs[i].getValue(), target.getValue());
    }
    table.close();
    assertEquals("Memory lost", NUM_MEM_PAGES, table.getFreeMemory().size());
}

Example 2 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class NonReusingHashJoinIteratorITCase method collectIntPairData.

public static Map<Integer, Collection<Integer>> collectIntPairData(MutableObjectIterator<IntPair> iter) throws Exception {
    Map<Integer, Collection<Integer>> map = new HashMap<>();
    IntPair pair = new IntPair();
    while ((pair = iter.next(pair)) != null) {
        final int key = pair.getKey();
        final int value = pair.getValue();
        if (!map.containsKey(key)) {
            map.put(key, new ArrayList<Integer>());
        }
        Collection<Integer> values = map.get(key);
        values.add(value);
    }
    return map;
}

Example 3 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class NonReusingHashJoinIteratorITCase method testBuildFirstWithMixedDataTypes.

@Test
public void testBuildFirstWithMixedDataTypes() {
    try {
        MutableObjectIterator<IntPair> input1 = new UniformIntPairGenerator(500, 40, false);
        final TupleGenerator generator2 = new TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleIntPairMatch>> expectedMatchesMap = joinIntPairs(collectIntPairData(input1), collectTupleData(input2));
        final FlatJoinFunction<IntPair, Tuple2<Integer, String>, Tuple2<Integer, String>> matcher = new TupleIntPairMatchRemovingMatcher(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        input1 = new UniformIntPairGenerator(500, 40, false);
        generator2.reset();
        input2.reset();
        // compare with iterator values
        NonReusingBuildSecondHashJoinIterator<IntPair, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new NonReusingBuildSecondHashJoinIterator<>(input1, input2, this.pairSerializer, this.pairComparator, this.recordSerializer, this.record2Comparator, this.pairRecordPairComparator, this.memoryManager, this.ioManager, this.parentTask, 1.0, false, false, true);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleIntPairMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 4 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class ReusingHashJoinIteratorITCase method testBuildFirstWithMixedDataTypes.

@Test
public void testBuildFirstWithMixedDataTypes() {
    try {
        MutableObjectIterator<IntPair> input1 = new UniformIntPairGenerator(500, 40, false);
        final TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
        // collect expected data
        final Map<Integer, Collection<TupleIntPairMatch>> expectedMatchesMap = joinIntPairs(collectIntPairData(input1), collectTupleData(input2));
        final FlatJoinFunction<IntPair, Tuple2<Integer, String>, Tuple2<Integer, String>> matcher = new TupleIntPairMatchRemovingMatcher(expectedMatchesMap);
        final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
        // reset the generators
        input1 = new UniformIntPairGenerator(500, 40, false);
        generator2.reset();
        input2.reset();
        // compare with iterator values
        ReusingBuildSecondHashJoinIterator<IntPair, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator = new ReusingBuildSecondHashJoinIterator<>(input1, input2, this.pairSerializer, this.pairComparator, this.recordSerializer, this.record2Comparator, this.pairRecordPairComparator, this.memoryManager, this.ioManager, this.parentTask, 1.0, false, false, true);
        iterator.open();
        while (iterator.callWithNextKey(matcher, collector)) ;
        iterator.close();
        // assert that each expected match was seen
        for (Entry<Integer, Collection<TupleIntPairMatch>> entry : expectedMatchesMap.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                Assert.fail("Collection for key " + entry.getKey() + " is not empty");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
}

Example 5 with IntPair

use of org.apache.flink.runtime.operators.testutils.types.IntPair in project flink by apache.

the class HashTablePerformanceComparison method testCompactingHashMapPerformance.

@Test
public void testCompactingHashMapPerformance() {
    try {
        final int NUM_MEM_PAGES = SIZE * NUM_PAIRS / PAGE_SIZE;
        MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
        MutableObjectIterator<IntPair> probeTester = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
        MutableObjectIterator<IntPair> updater = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
        MutableObjectIterator<IntPair> updateTester = new UniformIntPairGenerator(NUM_PAIRS, 1, false);
        long start;
        long end;
        long first = System.currentTimeMillis();
        System.out.println("Creating and filling CompactingHashMap...");
        start = System.currentTimeMillis();
        AbstractMutableHashTable<IntPair> table = new CompactingHashTable<IntPair>(serializer, comparator, getMemory(NUM_MEM_PAGES, PAGE_SIZE));
        table.open();
        IntPair target = new IntPair();
        while (buildInput.next(target) != null) {
            table.insert(target);
        }
        end = System.currentTimeMillis();
        System.out.println("HashMap ready. Time: " + (end - start) + " ms");
        System.out.println("Starting first probing run...");
        start = System.currentTimeMillis();
        AbstractHashTableProber<IntPair, IntPair> prober = table.getProber(comparator, pairComparator);
        IntPair temp = new IntPair();
        while (probeTester.next(target) != null) {
            assertNotNull(prober.getMatchFor(target, temp));
            assertEquals(temp.getValue(), target.getValue());
        }
        end = System.currentTimeMillis();
        System.out.println("Probing done. Time: " + (end - start) + " ms");
        System.out.println("Starting update...");
        start = System.currentTimeMillis();
        while (updater.next(target) != null) {
            target.setValue(target.getValue() + 1);
            table.insertOrReplaceRecord(target);
        }
        end = System.currentTimeMillis();
        System.out.println("Update done. Time: " + (end - start) + " ms");
        System.out.println("Starting second probing run...");
        start = System.currentTimeMillis();
        while (updateTester.next(target) != null) {
            assertNotNull(prober.getMatchFor(target, temp));
            assertEquals(target.getValue() + 1, temp.getValue());
        }
        end = System.currentTimeMillis();
        System.out.println("Probing done. Time: " + (end - start) + " ms");
        table.close();
        end = System.currentTimeMillis();
        System.out.println("Overall time: " + (end - first) + " ms");
        assertEquals("Memory lost", NUM_MEM_PAGES, table.getFreeMemory().size());
    } catch (Exception e) {
        e.printStackTrace();
        fail("Error: " + e.getMessage());
    }
}