Search in sources :

Example 6 with RuntimePairComparatorFactory

use of org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory in project flink by apache.

the class RightOuterJoinTaskTest method testCancelRightOuterJoinTaskWhileBuilding.

@Test
public void testCancelRightOuterJoinTaskWhileBuilding() throws Exception {
    setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
    getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
    addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockJoinStub.class);
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    taskRunner.start();
    Thread.sleep(1000);
    cancel();
    taskRunner.join(60000);
    assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
    final Throwable taskError = error.get();
    if (taskError != null) {
        fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
    }
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) UniformIntTupleGenerator(org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator) Tuple2(org.apache.flink.api.java.tuple.Tuple2) AtomicReference(java.util.concurrent.atomic.AtomicReference) InfiniteIntTupleIterator(org.apache.flink.runtime.operators.testutils.InfiniteIntTupleIterator) Test(org.junit.Test)

Example 7 with RuntimePairComparatorFactory

use of org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory in project flink by apache.

the class LeftOuterJoinTaskExternalITCase method testExternalHashLeftOuterJoinTask.

@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {
    final int keyCnt1 = 65536;
    final int valCnt1 = 8;
    final int keyCnt2 = 32768;
    final int valCnt2 = 8;
    final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(hash_frac);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
    addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
    testDriver(testTask, MockJoinStub.class);
    Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) UniformIntTupleGenerator(org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Test(org.junit.Test)

Example 8 with RuntimePairComparatorFactory

use of org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory in project flink by apache.

the class LeftOuterJoinTaskTest method testCancelLeftOuterJoinTaskWhileProbing.

@Test
public void testCancelLeftOuterJoinTaskWhileProbing() throws Exception {
    setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
    addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockJoinStub.class);
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    taskRunner.start();
    Thread.sleep(1000);
    cancel();
    taskRunner.join(60000);
    assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
    final Throwable taskError = error.get();
    if (taskError != null) {
        fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
    }
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) UniformIntTupleGenerator(org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator) Tuple2(org.apache.flink.api.java.tuple.Tuple2) AtomicReference(java.util.concurrent.atomic.AtomicReference) InfiniteIntTupleIterator(org.apache.flink.runtime.operators.testutils.InfiniteIntTupleIterator) Test(org.junit.Test)

Example 9 with RuntimePairComparatorFactory

use of org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory in project flink by apache.

the class AbstractOuterJoinTaskTest method testMergeOuterJoinTask.

@Test
public void testMergeOuterJoinTask() throws Exception {
    int keyCnt1 = 20;
    int valCnt1 = 20;
    int keyCnt2 = 20;
    int valCnt2 = 20;
    setOutput(this.outList, this.serializer);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
    getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
    setNumFileHandlesForSort(4);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
    addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
    testDriver(testTask, MockJoinStub.class);
    final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
    Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
    this.outList.clear();
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) UniformIntTupleGenerator(org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Test(org.junit.Test)

Example 10 with RuntimePairComparatorFactory

use of org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory in project flink by apache.

the class AbstractOuterJoinTaskTest method testCancelOuterJoinTaskWhileRunning.

@Test
public void testCancelOuterJoinTaskWhileRunning() throws Exception {
    setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
    getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
    getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
    setNumFileHandlesForSort(4);
    final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
    addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
    addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileRunning()") {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockJoinStub.class);
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    taskRunner.start();
    Thread.sleep(1000);
    cancel();
    taskRunner.interrupt();
    taskRunner.join(60000);
    assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
    final Throwable taskError = error.get();
    if (taskError != null) {
        fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
    }
}
Also used : RuntimePairComparatorFactory(org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory) Tuple2(org.apache.flink.api.java.tuple.Tuple2) AtomicReference(java.util.concurrent.atomic.AtomicReference) InfiniteIntTupleIterator(org.apache.flink.runtime.operators.testutils.InfiniteIntTupleIterator) Test(org.junit.Test)

Aggregations

Tuple2 (org.apache.flink.api.java.tuple.Tuple2)19 RuntimePairComparatorFactory (org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory)19 UniformIntTupleGenerator (org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator)16 Test (org.junit.Test)16 AtomicReference (java.util.concurrent.atomic.AtomicReference)7 InfiniteIntTupleIterator (org.apache.flink.runtime.operators.testutils.InfiniteIntTupleIterator)7