Examples with IntList org.apache.flink.runtime.operators.testutils.types.IntList used on opensource projects

Example 1 with IntList

use of org.apache.flink.runtime.operators.testutils.types.IntList in project flink by apache.

the class MutableHashTableTestBase method getRandomizedIntLists.

protected static IntList[] getRandomizedIntLists(int num, Random rnd) {
    IntList[] lists = new IntList[num];
    for (int i = 0; i < num; i++) {
        int[] value = new int[rnd.nextInt(MAX_LIST_SIZE) + 1];
        //int[] value = new int[MAX_LIST_SIZE-1];
        for (int j = 0; j < value.length; j++) {
            value[j] = -rnd.nextInt(Integer.MAX_VALUE);
        }
        lists[i] = new IntList(i, value);
    }
    return lists;
}

Example 2 with IntList

use of org.apache.flink.runtime.operators.testutils.types.IntList in project flink by apache.

the class MutableHashTableTestBase method testEntryIterator.

@Test
public void testEntryIterator() throws Exception {
    final int NUM_MEM_PAGES = SIZE * NUM_LISTS / PAGE_SIZE;
    AbstractMutableHashTable<IntList> table = getHashTable(serializerV, comparatorV, getMemory(NUM_MEM_PAGES));
    final Random rnd = new Random(RANDOM_SEED);
    final IntList[] lists = getRandomizedIntLists(NUM_LISTS, rnd);
    table.open();
    int result = 0;
    for (int i = 0; i < NUM_LISTS; i++) {
        table.insert(lists[i]);
        result += lists[i].getKey();
    }
    MutableObjectIterator<IntList> iter = table.getEntryIterator();
    IntList target = new IntList();
    int sum = 0;
    while ((target = iter.next(target)) != null) {
        sum += target.getKey();
    }
    table.close();
    assertTrue(sum == result);
    assertEquals("Memory lost", NUM_MEM_PAGES, table.getFreeMemory().size());
}

Example 3 with IntList

use of org.apache.flink.runtime.operators.testutils.types.IntList in project flink by apache.

the class MutableHashTableTestBase method testVariableLengthBuildAndRetrieveMajorityUpdated.

@Test
public void testVariableLengthBuildAndRetrieveMajorityUpdated() throws Exception {
    final int NUM_MEM_PAGES = SIZE * NUM_LISTS / PAGE_SIZE;
    AbstractMutableHashTable<IntList> table = getHashTable(serializerV, comparatorV, getMemory(NUM_MEM_PAGES));
    final Random rnd = new Random(RANDOM_SEED);
    final IntList[] lists = getRandomizedIntLists(NUM_LISTS, rnd);
    table.open();
    for (int i = 0; i < NUM_LISTS; i++) {
        table.insert(lists[i]);
    }
    AbstractHashTableProber<IntList, IntList> prober = table.getProber(comparatorV, pairComparatorV);
    IntList target = new IntList();
    for (int i = 0; i < NUM_LISTS; i++) {
        assertNotNull(prober.getMatchFor(lists[i], target));
        assertArrayEquals(lists[i].getValue(), target.getValue());
    }
    final IntList[] overwriteLists = getRandomizedIntLists(NUM_LISTS, rnd);
    // test replacing
    for (int i = 0; i < NUM_LISTS; i++) {
        if (i % 100 != 0) {
            table.insertOrReplaceRecord(overwriteLists[i]);
            lists[i] = overwriteLists[i];
        }
    }
    for (int i = 0; i < NUM_LISTS; i++) {
        assertNotNull("" + i, prober.getMatchFor(lists[i], target));
        assertArrayEquals(lists[i].getValue(), target.getValue());
    }
    table.close();
    assertEquals("Memory lost", NUM_MEM_PAGES, table.getFreeMemory().size());
}

Example 4 with IntList

use of org.apache.flink.runtime.operators.testutils.types.IntList in project flink by apache.

the class MutableHashTableTestBase method testVariableLengthBuildAndRetrieve.

@Test
public void testVariableLengthBuildAndRetrieve() throws Exception {
    final int NUM_MEM_PAGES = SIZE * NUM_LISTS / PAGE_SIZE;
    AbstractMutableHashTable<IntList> table = getHashTable(serializerV, comparatorV, getMemory(NUM_MEM_PAGES));
    final Random rnd = new Random(RANDOM_SEED);
    final IntList[] lists = getRandomizedIntLists(NUM_LISTS, rnd);
    table.open();
    for (int i = 0; i < NUM_LISTS; i++) {
        try {
            table.insert(lists[i]);
        } catch (Exception e) {
            throw e;
        }
    }
    AbstractHashTableProber<IntList, IntList> prober = table.getProber(comparatorV, pairComparatorV);
    IntList target = new IntList();
    for (int i = 0; i < NUM_LISTS; i++) {
        assertNotNull(prober.getMatchFor(lists[i], target));
        assertArrayEquals(lists[i].getValue(), target.getValue());
    }
    final IntList[] overwriteLists = getRandomizedIntLists(NUM_LISTS, rnd);
    // test replacing
    for (int i = 0; i < NUM_LISTS; i++) {
        table.insertOrReplaceRecord(overwriteLists[i]);
    }
    for (int i = 0; i < NUM_LISTS; i++) {
        assertNotNull("" + i, prober.getMatchFor(overwriteLists[i], target));
        assertArrayEquals(overwriteLists[i].getValue(), target.getValue());
    }
    table.close();
    assertEquals("Memory lost", NUM_MEM_PAGES, table.getFreeMemory().size());
}

Example 5 with IntList

use of org.apache.flink.runtime.operators.testutils.types.IntList in project flink by apache.

the class MutableHashTableTestBase method testVariableLengthBuildAndRetrieveMinorityUpdated.

@Test
public void testVariableLengthBuildAndRetrieveMinorityUpdated() throws Exception {
    final int NUM_LISTS = 20000;
    final int NUM_MEM_PAGES = SIZE * NUM_LISTS / PAGE_SIZE;
    AbstractMutableHashTable<IntList> table = getHashTable(serializerV, comparatorV, getMemory(NUM_MEM_PAGES));
    final int STEP_SIZE = 100;
    final Random rnd = new Random(RANDOM_SEED);
    final IntList[] lists = getRandomizedIntLists(NUM_LISTS, rnd);
    table.open();
    for (int i = 0; i < NUM_LISTS; i++) {
        table.insert(lists[i]);
    }
    AbstractHashTableProber<IntList, IntList> prober = table.getProber(comparatorV, pairComparatorV);
    IntList target = new IntList();
    for (int i = 0; i < NUM_LISTS; i++) {
        assertNotNull(prober.getMatchFor(lists[i], target));
        assertArrayEquals(lists[i].getValue(), target.getValue());
    }
    final IntList[] overwriteLists = getRandomizedIntLists(NUM_LISTS / STEP_SIZE, rnd);
    // test replacing
    for (int i = 0; i < NUM_LISTS; i += STEP_SIZE) {
        overwriteLists[i / STEP_SIZE].setKey(overwriteLists[i / STEP_SIZE].getKey() * STEP_SIZE);
        table.insertOrReplaceRecord(overwriteLists[i / STEP_SIZE]);
        lists[i] = overwriteLists[i / STEP_SIZE];
    }
    for (int i = 0; i < NUM_LISTS; i++) {
        assertNotNull(prober.getMatchFor(lists[i], target));
        assertArrayEquals(lists[i].getValue(), target.getValue());
    }
    table.close();
    assertEquals("Memory lost", NUM_MEM_PAGES, table.getFreeMemory().size());
}