Examples with UniformRecordGenerator org.apache.flink.runtime.operators.testutils.UniformRecordGenerator used on opensource projects

Example 86 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testCancelMatchTaskWhileMatching.

@Test
public void testCancelMatchTaskWhileMatching() {
    final int keyCnt = 20;
    final int valCnt = 20;
    try {
        setOutput(new NirvanaOutputList());
        addDriverComparator(this.comparator1);
        addDriverComparator(this.comparator2);
        getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
        getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
        getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
        setNumFileHandlesForSort(4);
        final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
        addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
        addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
        final AtomicReference<Throwable> error = new AtomicReference<>();
        Thread taskRunner = new Thread("Task runner for testCancelMatchTaskWhileMatching()") {

            @Override
            public void run() {
                try {
                    testDriver(testTask, MockDelayingMatchStub.class);
                } catch (Throwable t) {
                    error.set(t);
                }
            }
        };
        taskRunner.start();
        Thread.sleep(1000);
        cancel();
        taskRunner.interrupt();
        taskRunner.join(60000);
        assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
        Throwable taskError = error.get();
        if (taskError != null) {
            taskError.printStackTrace();
            fail("Error in task while canceling: " + taskError.getMessage());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 87 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testHash2MatchTask.

@Test
public void testHash2MatchTask() {
    int keyCnt1 = 20;
    int valCnt1 = 1;
    int keyCnt2 = 20;
    int valCnt2 = 1;
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(hash_frac);
    JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
    try {
        testDriver(testTask, MockMatchStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test caused an exception.");
    }
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2);
    Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
    this.outList.clear();
}

Example 88 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testCancelMatchTaskWhileSort2.

@Test
public void testCancelMatchTaskWhileSort2() {
    final int keyCnt = 20;
    final int valCnt = 20;
    try {
        setOutput(new NirvanaOutputList());
        addDriverComparator(this.comparator1);
        addDriverComparator(this.comparator2);
        getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
        getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
        getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
        setNumFileHandlesForSort(4);
        final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
        try {
            addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
            addInputSorted(new DelayingInfinitiveInputIterator(100), this.comparator1.duplicate());
        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail("The test caused an exception.");
        }
        final AtomicReference<Throwable> error = new AtomicReference<>();
        Thread taskRunner = new Thread("Task runner for testCancelMatchTaskWhileSort2()") {

            @Override
            public void run() {
                try {
                    testDriver(testTask, MockMatchStub.class);
                } catch (Throwable t) {
                    error.set(t);
                }
            }
        };
        taskRunner.start();
        Thread.sleep(1000);
        cancel();
        taskRunner.interrupt();
        taskRunner.join(60000);
        assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
        Throwable taskError = error.get();
        if (taskError != null) {
            taskError.printStackTrace();
            fail("Error in task while canceling: " + taskError.getMessage());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 89 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testHashSecondCancelMatchTaskWhileMatching.

@Test
public void testHashSecondCancelMatchTaskWhileMatching() {
    int keyCnt = 20;
    int valCnt = 20;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(new NirvanaOutputList());
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(hash_frac);
    final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
    final AtomicBoolean success = new AtomicBoolean(false);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockMatchStub.class);
                success.set(true);
            } catch (Exception ie) {
                ie.printStackTrace();
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(1, taskRunner, this);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
    Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}

Example 90 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testCancelMatchTaskWhileSort1.

@Test
public void testCancelMatchTaskWhileSort1() {
    final int keyCnt = 20;
    final int valCnt = 20;
    try {
        setOutput(new NirvanaOutputList());
        addDriverComparator(this.comparator1);
        addDriverComparator(this.comparator2);
        getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
        getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
        getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
        setNumFileHandlesForSort(4);
        final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
        try {
            addInputSorted(new DelayingInfinitiveInputIterator(100), this.comparator1.duplicate());
            addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail("The test caused an exception.");
        }
        final AtomicReference<Throwable> error = new AtomicReference<>();
        Thread taskRunner = new Thread("Task runner for testCancelMatchTaskWhileSort1()") {

            @Override
            public void run() {
                try {
                    testDriver(testTask, MockMatchStub.class);
                } catch (Throwable t) {
                    error.set(t);
                }
            }
        };
        taskRunner.start();
        Thread.sleep(1000);
        cancel();
        taskRunner.interrupt();
        taskRunner.join(60000);
        assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());
        Throwable taskError = error.get();
        if (taskError != null) {
            taskError.printStackTrace();
            fail("Error in task while canceling: " + taskError.getMessage());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}