Examples with UniformRecordGenerator org.apache.flink.runtime.operators.testutils.UniformRecordGenerator used on opensource projects

Example 56 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CachedMatchTaskTest method testHash4MatchTask.

@Test
public void testHash4MatchTask() {
    int keyCnt1 = 20;
    int valCnt1 = 20;
    int keyCnt2 = 20;
    int valCnt2 = 1;
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
    getTaskConfig().setRelativeMemoryDriver(1.0f);
    BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
    try {
        testResettableDriver(testTask, MockMatchStub.class, 3);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test caused an exception.");
    }
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2);
    Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
    this.outList.clear();
}

Example 57 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CoGroupTaskTest method testSortBoth1CoGroupTask.

@Test
public void testSortBoth1CoGroupTask() {
    int keyCnt1 = 100;
    int valCnt1 = 2;
    int keyCnt2 = 200;
    int valCnt2 = 1;
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2) + (keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
    final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
    try {
        addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
        addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
        testDriver(testTask, MockCoGroupStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test caused an exception.");
    }
    Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}

Example 58 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CoGroupTaskTest method testSortSecondCoGroupTask.

@Test
public void testSortSecondCoGroupTask() {
    int keyCnt1 = 200;
    int valCnt1 = 2;
    int keyCnt2 = 200;
    int valCnt2 = 4;
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2) + (keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
    final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
    try {
        addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
        addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
        testDriver(testTask, MockCoGroupStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test caused an exception.");
    }
    Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}

Example 59 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CoGroupTaskTest method testCancelCoGroupTaskWhileSorting2.

@Test
public void testCancelCoGroupTaskWhileSorting2() {
    int keyCnt = 10;
    int valCnt = 2;
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
    final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
    try {
        addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
        addInputSorted(new DelayingInfinitiveInputIterator(1000), this.comparator2.duplicate());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test caused an exception.");
    }
    final AtomicBoolean success = new AtomicBoolean(false);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockCoGroupStub.class);
                success.set(true);
            } catch (Exception ie) {
                ie.printStackTrace();
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(1, taskRunner, this);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
    Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}

Example 60 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CoGroupTaskTest method testCancelCoGroupTaskWhileSorting1.

@Test
public void testCancelCoGroupTaskWhileSorting1() {
    int keyCnt = 10;
    int valCnt = 2;
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
    final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
    try {
        addInputSorted(new DelayingInfinitiveInputIterator(1000), this.comparator1.duplicate());
        addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test caused an exception.");
    }
    final AtomicBoolean success = new AtomicBoolean(false);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockCoGroupStub.class);
                success.set(true);
            } catch (Exception ie) {
                ie.printStackTrace();
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(1, taskRunner, this);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
    Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}