Examples with UnionIterator org.apache.flink.runtime.operators.testutils.UnionIterator used on opensource projects

Example 6 with UnionIterator

use of org.apache.flink.runtime.operators.testutils.UnionIterator in project flink by apache.

the class HashTableITCase method testSpillingHashJoinWithTwoRecursionsIntPair.

/*
	 * This test is basically identical to the "testSpillingHashJoinWithMassiveCollisions" test, only that the number
	 * of repeated values (causing bucket collisions) are large enough to make sure that their target partition no longer
	 * fits into memory by itself and needs to be repartitioned in the recursion again.
	 */
@Test
public void testSpillingHashJoinWithTwoRecursionsIntPair() throws IOException {
    // the following two values are known to have a hash-code collision on the first recursion level.
    // we use them to make sure one partition grows over-proportionally large
    final int REPEATED_VALUE_1 = 40559;
    final int REPEATED_VALUE_2 = 92882;
    final int REPEATED_VALUE_COUNT_BUILD = 200000;
    final int REPEATED_VALUE_COUNT_PROBE = 5;
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<IntPair> probeInput = new UnionIterator<IntPair>(probes);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // create the map for validating the results
    HashMap<Integer, Long> map = new HashMap<Integer, Long>(NUM_KEYS);
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    IntPair record;
    final IntPair recordReuse = new IntPair();
    while (join.nextRecord()) {
        int numBuildValues = 0;
        final IntPair probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getKey();
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues = 1;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getKey());
        } else {
            fail("No build side values found for a probe key.");
        }
        while ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues++;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getKey());
        }
        Long contained = map.get(key);
        if (contained == null) {
            contained = Long.valueOf(numBuildValues);
        } else {
            contained = Long.valueOf(contained.longValue() + numBuildValues);
        }
        map.put(key, contained);
    }
    join.close();
    Assert.assertEquals("Wrong number of keys", NUM_KEYS, map.size());
    for (Map.Entry<Integer, Long> entry : map.entrySet()) {
        long val = entry.getValue();
        int key = entry.getKey();
        Assert.assertEquals("Wrong number of values in per-key cross product for key " + key, (key == REPEATED_VALUE_1 || key == REPEATED_VALUE_2) ? (PROBE_VALS_PER_KEY + REPEATED_VALUE_COUNT_PROBE) * (BUILD_VALS_PER_KEY + REPEATED_VALUE_COUNT_BUILD) : PROBE_VALS_PER_KEY * BUILD_VALS_PER_KEY, val);
    }
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}

Example 7 with UnionIterator

use of org.apache.flink.runtime.operators.testutils.UnionIterator in project flink by apache.

the class HashTableITCase method testSpillingHashJoinWithTwoRecursions.

/*
	 * This test is basically identical to the "testSpillingHashJoinWithMassiveCollisions" test, only that the number
	 * of repeated values (causing bucket collisions) are large enough to make sure that their target partition no longer
	 * fits into memory by itself and needs to be repartitioned in the recursion again.
	 */
@Test
public void testSpillingHashJoinWithTwoRecursions() throws IOException {
    // the following two values are known to have a hash-code collision on the first recursion level.
    // we use them to make sure one partition grows over-proportionally large
    final int REPEATED_VALUE_1 = 40559;
    final int REPEATED_VALUE_2 = 92882;
    final int REPEATED_VALUE_COUNT_BUILD = 200000;
    final int REPEATED_VALUE_COUNT_PROBE = 5;
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<Record> buildInput = new UnionIterator<Record>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<Record> probeInput = new UnionIterator<Record>(probes);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // create the map for validating the results
    HashMap<Integer, Long> map = new HashMap<Integer, Long>(NUM_KEYS);
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<Record, Record> join = new MutableHashTable<Record, Record>(this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    Record record;
    final Record recordReuse = new Record();
    while (join.nextRecord()) {
        int numBuildValues = 0;
        final Record probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getField(0, IntValue.class).getValue();
        MutableObjectIterator<Record> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues = 1;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());
        } else {
            fail("No build side values found for a probe key.");
        }
        while ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues++;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());
        }
        Long contained = map.get(key);
        if (contained == null) {
            contained = Long.valueOf(numBuildValues);
        } else {
            contained = Long.valueOf(contained.longValue() + numBuildValues);
        }
        map.put(key, contained);
    }
    join.close();
    Assert.assertEquals("Wrong number of keys", NUM_KEYS, map.size());
    for (Map.Entry<Integer, Long> entry : map.entrySet()) {
        long val = entry.getValue();
        int key = entry.getKey();
        Assert.assertEquals("Wrong number of values in per-key cross product for key " + key, (key == REPEATED_VALUE_1 || key == REPEATED_VALUE_2) ? (PROBE_VALS_PER_KEY + REPEATED_VALUE_COUNT_PROBE) * (BUILD_VALS_PER_KEY + REPEATED_VALUE_COUNT_BUILD) : PROBE_VALS_PER_KEY * BUILD_VALS_PER_KEY, val);
    }
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}

Example 8 with UnionIterator

use of org.apache.flink.runtime.operators.testutils.UnionIterator in project flink by apache.

the class HashTableITCase method testSpillingHashJoinWithMassiveCollisions.

@Test
public void testSpillingHashJoinWithMassiveCollisions() throws IOException {
    // the following two values are known to have a hash-code collision on the initial level.
    // we use them to make sure one partition grows over-proportionally large
    final int REPEATED_VALUE_1 = 40559;
    final int REPEATED_VALUE_2 = 92882;
    final int REPEATED_VALUE_COUNT_BUILD = 200000;
    final int REPEATED_VALUE_COUNT_PROBE = 5;
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<Record> buildInput = new UnionIterator<Record>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<Record> probeInput = new UnionIterator<Record>(probes);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // create the map for validating the results
    HashMap<Integer, Long> map = new HashMap<Integer, Long>(NUM_KEYS);
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<Record, Record> join = new MutableHashTable<Record, Record>(this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    Record record;
    final Record recordReuse = new Record();
    while (join.nextRecord()) {
        int numBuildValues = 0;
        final Record probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getField(0, IntValue.class).getValue();
        MutableObjectIterator<Record> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues = 1;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());
        } else {
            fail("No build side values found for a probe key.");
        }
        while ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues++;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());
        }
        Long contained = map.get(key);
        if (contained == null) {
            contained = Long.valueOf(numBuildValues);
        } else {
            contained = Long.valueOf(contained.longValue() + numBuildValues);
        }
        map.put(key, contained);
    }
    join.close();
    Assert.assertEquals("Wrong number of keys", NUM_KEYS, map.size());
    for (Map.Entry<Integer, Long> entry : map.entrySet()) {
        long val = entry.getValue();
        int key = entry.getKey();
        Assert.assertEquals("Wrong number of values in per-key cross product for key " + key, (key == REPEATED_VALUE_1 || key == REPEATED_VALUE_2) ? (PROBE_VALS_PER_KEY + REPEATED_VALUE_COUNT_PROBE) * (BUILD_VALS_PER_KEY + REPEATED_VALUE_COUNT_BUILD) : PROBE_VALS_PER_KEY * BUILD_VALS_PER_KEY, val);
    }
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}

Example 9 with UnionIterator

use of org.apache.flink.runtime.operators.testutils.UnionIterator in project flink by apache.

the class HashTableITCase method testFailingHashJoinTooManyRecursions.

/*
	 * This test is basically identical to the "testSpillingHashJoinWithMassiveCollisions" test, only that the number
	 * of repeated values (causing bucket collisions) are large enough to make sure that their target partition no longer
	 * fits into memory by itself and needs to be repartitioned in the recursion again.
	 */
@Test
public void testFailingHashJoinTooManyRecursions() throws IOException {
    // the following two values are known to have a hash-code collision on the first recursion level.
    // we use them to make sure one partition grows over-proportionally large
    final int REPEATED_VALUE_1 = 40559;
    final int REPEATED_VALUE_2 = 92882;
    final int REPEATED_VALUE_COUNT = 3000000;
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<Record> build1 = new UniformRecordGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<Record> build2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<Record> build3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<Record>> builds = new ArrayList<MutableObjectIterator<Record>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<Record> buildInput = new UnionIterator<Record>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<Record> probe1 = new UniformRecordGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<Record> probe2 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT);
    MutableObjectIterator<Record> probe3 = new ConstantsKeyValuePairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT);
    List<MutableObjectIterator<Record>> probes = new ArrayList<MutableObjectIterator<Record>>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<Record> probeInput = new UnionIterator<Record>(probes);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<Record, Record> join = new MutableHashTable<Record, Record>(this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    final Record recordReuse = new Record();
    try {
        while (join.nextRecord()) {
            MutableObjectIterator<Record> buildSide = join.getBuildSideIterator();
            if (buildSide.next(recordReuse) == null) {
                fail("No build side values found for a probe key.");
            }
            while (buildSide.next(recordReuse) != null) ;
        }
        fail("Hash Join must have failed due to too many recursions.");
    } catch (Exception ex) {
    // expected
    }
    join.close();
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}

Example 10 with UnionIterator

use of org.apache.flink.runtime.operators.testutils.UnionIterator in project flink by apache.

the class CombinerOversizedRecordsTest method testOversizedRecordCombineTask.

@Test
public void testOversizedRecordCombineTask() {
    try {
        final int keyCnt = 100;
        final int valCnt = 20;
        // create a long heavy string payload
        StringBuilder bld = new StringBuilder(10 * 1024 * 1024);
        Random rnd = new Random();
        for (int i = 0; i < 10000000; i++) {
            bld.append((char) (rnd.nextInt(26) + 'a'));
        }
        String longString = bld.toString();
        bld = null;
        // construct the input as a union of
        // 1) long string
        // 2) some random values
        // 3) long string
        // 4) random values
        // 5) long string
        // random values 1
        MutableObjectIterator<Tuple2<Integer, Integer>> gen1 = new UniformIntTupleGenerator(keyCnt, valCnt, false);
        // random values 2
        MutableObjectIterator<Tuple2<Integer, Integer>> gen2 = new UniformIntTupleGenerator(keyCnt, valCnt, false);
        @SuppressWarnings("unchecked") MutableObjectIterator<Tuple3<Integer, Integer, String>> input = new UnionIterator<Tuple3<Integer, Integer, String>>(new SingleValueIterator<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(-1, -1, longString)), new StringIteratorDecorator(gen1), new SingleValueIterator<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(-1, -1, longString)), new StringIteratorDecorator(gen2), new SingleValueIterator<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(-1, -1, longString)));
        setInput(input, serializer);
        addDriverComparator(this.comparator);
        addDriverComparator(this.comparator);
        setOutput(this.outList, this.outSerializer);
        getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
        getTaskConfig().setRelativeMemoryDriver(combine_frac);
        getTaskConfig().setFilehandlesDriver(2);
        GroupReduceCombineDriver<Tuple3<Integer, Integer, String>, Tuple3<Integer, Double, String>> testTask = new GroupReduceCombineDriver<Tuple3<Integer, Integer, String>, Tuple3<Integer, Double, String>>();
        testDriver(testTask, TestCombiner.class);
        assertEquals(3, testTask.getOversizedRecordCount());
        assertTrue(keyCnt + 3 == outList.size() || 2 * keyCnt + 3 == outList.size());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}