Examples with UniformIntTupleGenerator org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator used on opensource projects

Example 1 with UniformIntTupleGenerator

use of org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator in project flink by apache.

the class AbstractOuterJoinTaskExternalITCase method testExternalSortOuterJoinTask.

@Test
public void testExternalSortOuterJoinTask() throws Exception {
    final int keyCnt1 = 16384 * 4;
    final int valCnt1 = 2;
    final int keyCnt2 = 8192;
    final int valCnt2 = 4 * 2;
    final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(this.getSortStrategy());
    getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
    setNumFileHandlesForSort(4);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
    addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
    testDriver(testTask, MockJoinStub.class);
    Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}

Example 2 with UniformIntTupleGenerator

use of org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator in project flink by apache.

the class AbstractOuterJoinTaskTest method testSortFirstOuterJoinTask.

@Test
public void testSortFirstOuterJoinTask() throws Exception {
    int keyCnt1 = 20;
    int valCnt1 = 20;
    int keyCnt2 = 20;
    int valCnt2 = 20;
    setOutput(this.outList, this.serializer);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
    getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
    setNumFileHandlesForSort(4);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
    addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
    testDriver(testTask, MockJoinStub.class);
    final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
    Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
    this.outList.clear();
}

Example 3 with UniformIntTupleGenerator

use of org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator in project flink by apache.

the class AbstractOuterJoinTaskTest method testSortBothOuterJoinTask.

private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
    setOutput(this.outList, this.serializer);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
    getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
    setNumFileHandlesForSort(4);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
    addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
    testDriver(testTask, MockJoinStub.class);
    final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
    Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
    this.outList.clear();
}

Example 4 with UniformIntTupleGenerator

use of org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator in project flink by apache.

the class JoinCancelingITCase method executeTaskWithGenerator.

private void executeTaskWithGenerator(JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner, int keys, int vals, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
    UniformIntTupleGenerator g = new UniformIntTupleGenerator(keys, vals, false);
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
    DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
    input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE).where(0).equalTo(0).with(joiner).output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
    env.setParallelism(parallelism);
    runAndCancelJob(env.createProgramPlan(), msecsTillCanceling, maxTimeTillCanceled);
}

Example 5 with UniformIntTupleGenerator

use of org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator in project flink by apache.

the class RightOuterJoinTaskExternalITCase method testExternalHashRightOuterJoinTask.

@Test
public void testExternalHashRightOuterJoinTask() throws Exception {
    final int keyCnt1 = 32768;
    final int valCnt1 = 8;
    final int keyCnt2 = 65536;
    final int valCnt2 = 8;
    final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
    getTaskConfig().setRelativeMemoryDriver(hash_frac);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
    addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
    testDriver(testTask, MockJoinStub.class);
    Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}