Examples with IntPairComparator org.apache.flink.runtime.operators.testutils.types.IntPairComparator used on opensource projects

Example 1 with IntPairComparator

use of org.apache.flink.runtime.operators.testutils.types.IntPairComparator in project flink by apache.

the class FixedLengthRecordSorterTest method beforeTest.

@Before
public void beforeTest() {
    this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).setPageSize(MEMORY_PAGE_SIZE).build();
    this.ioManager = new IOManagerAsync();
    this.serializer = new IntPairSerializer();
    this.comparator = new IntPairComparator();
}

Example 2 with IntPairComparator

use of org.apache.flink.runtime.operators.testutils.types.IntPairComparator in project flink by apache.

the class InPlaceMutableHashTableTest method testWithIntPair.

@Test
public void testWithIntPair() throws Exception {
    Random rnd = new Random(RANDOM_SEED);
    // varying the keyRange between 1000 and 1000000 can make a 5x speed difference
    // (because of cache misses (also in the segment arrays))
    final int keyRange = 1000000;
    final int valueRange = 10;
    final int numRecords = 1000000;
    final IntPairSerializer serializer = new IntPairSerializer();
    final TypeComparator<IntPair> comparator = new IntPairComparator();
    final ReduceFunction<IntPair> reducer = new SumReducer();
    // Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
    List<IntPair> expectedOutput = new ArrayList<>();
    InPlaceMutableHashTableWithJavaHashMap<IntPair, Integer> reference = new InPlaceMutableHashTableWithJavaHashMap<>(serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));
    // Create the InPlaceMutableHashTable to test
    final int numMemPages = keyRange * 32 / // memory use is proportional to the number of different keys
    PAGE_SIZE;
    List<IntPair> actualOutput = new ArrayList<>();
    InPlaceMutableHashTable<IntPair> table = new InPlaceMutableHashTable<>(serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
    InPlaceMutableHashTable<IntPair>.ReduceFacade reduceFacade = table.new ReduceFacade(reducer, new CopyingListCollector<>(actualOutput, serializer), true);
    table.open();
    // Generate some input
    final List<IntPair> input = new ArrayList<>();
    for (int i = 0; i < numRecords; i++) {
        input.add(new IntPair(rnd.nextInt(keyRange), rnd.nextInt(valueRange)));
    }
    // System.out.println("start");
    // long start = System.currentTimeMillis();
    // Process the generated input
    final int numIntermingledEmits = 5;
    for (IntPair record : input) {
        reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
        reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
        if (rnd.nextDouble() < 1.0 / ((double) numRecords / numIntermingledEmits)) {
            // this will fire approx. numIntermingledEmits times
            reference.emitAndReset();
            reduceFacade.emitAndReset();
        }
    }
    reference.emitAndReset();
    reduceFacade.emit();
    table.close();
    // long end = System.currentTimeMillis();
    // System.out.println("stop, time: " + (end - start));
    // Check results
    assertEquals(expectedOutput.size(), actualOutput.size());
    Integer[] expectedValues = new Integer[expectedOutput.size()];
    for (int i = 0; i < expectedOutput.size(); i++) {
        expectedValues[i] = expectedOutput.get(i).getValue();
    }
    Integer[] actualValues = new Integer[actualOutput.size()];
    for (int i = 0; i < actualOutput.size(); i++) {
        actualValues[i] = actualOutput.get(i).getValue();
    }
    Arrays.sort(expectedValues, Ordering.<Integer>natural());
    Arrays.sort(actualValues, Ordering.<Integer>natural());
    assertArrayEquals(expectedValues, actualValues);
}

Example 3 with IntPairComparator

use of org.apache.flink.runtime.operators.testutils.types.IntPairComparator in project flink by apache.

the class HashTableITCase method setup.

@Before
public void setup() {
    final int[] keyPos = new int[] { 0 };
    @SuppressWarnings("unchecked") final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
    this.recordBuildSideAccesssor = RecordSerializer.get();
    this.recordProbeSideAccesssor = RecordSerializer.get();
    this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
    this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
    this.pactRecordComparator = new RecordPairComparatorFirstInt();
    this.pairBuildSideAccesssor = new IntPairSerializer();
    this.pairProbeSideAccesssor = new IntPairSerializer();
    this.pairBuildSideComparator = new IntPairComparator();
    this.pairProbeSideComparator = new IntPairComparator();
    this.pairComparator = new IntPairPairComparator();
    this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(32 * 1024 * 1024).build();
    this.ioManager = new IOManagerAsync();
}