Examples with UniformRecordGenerator org.apache.flink.runtime.operators.testutils.UniformRecordGenerator used on opensource projects

Example 66 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class ReduceTaskExternalITCase method testMultiLevelMergeCombiningReduceTask.

@Test
public void testMultiLevelMergeCombiningReduceTask() throws IOException {
    int keyCnt = 32768;
    int valCnt = 8;
    addDriverComparator(this.comparator);
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
    Sorter<Record> sorter = null;
    try {
        sorter = ExternalSorter.newBuilder(getMemoryManager(), getContainingTask(), RecordSerializerFactory.get().getSerializer(), this.comparator.duplicate()).maxNumFileHandles(2).withCombiner(new MockCombiningReduceStub()).enableSpilling(getIOManager(), 0.8f).memoryFraction(this.perSortFractionMem).objectReuse(false).largeRecords(true).build(new UniformRecordGenerator(keyCnt, valCnt, false));
        addInput(sorter.getIterator());
        GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
        testDriver(testTask, MockCombiningReduceStub.class);
    } catch (Exception e) {
        LOG.info("Exception while running the test task.", e);
        Assert.fail("Invoke method caused exception: " + e.getMessage());
    } finally {
        if (sorter != null) {
            sorter.close();
        }
    }
    int expSum = 0;
    for (int i = 1; i < valCnt; i++) {
        expSum += i;
    }
    Assert.assertTrue("Resultset size was " + this.outList.size() + ". Expected was " + keyCnt, this.outList.size() == keyCnt);
    for (Record record : this.outList) {
        Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum - record.getField(0, IntValue.class).getValue());
    }
    this.outList.clear();
}

Example 67 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CachedMatchTaskTest method testHashSecondCancelMatchTaskWhileMatching.

@Test
public void testHashSecondCancelMatchTaskWhileMatching() {
    int keyCnt = 20;
    int valCnt = 20;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(new NirvanaOutputList());
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(1.0f);
    final BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
    final AtomicBoolean success = new AtomicBoolean(false);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockMatchStub.class);
                success.set(true);
            } catch (Exception ie) {
                ie.printStackTrace();
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(1, taskRunner, this);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
    Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}

Example 68 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CachedMatchTaskTest method testHashFirstCancelMatchTaskWhileMatching.

@Test
public void testHashFirstCancelMatchTaskWhileMatching() {
    int keyCnt = 20;
    int valCnt = 20;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(new NirvanaOutputList());
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
    getTaskConfig().setRelativeMemoryDriver(1.0f);
    final BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
    final AtomicBoolean success = new AtomicBoolean(false);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockMatchStub.class);
                success.set(true);
            } catch (Exception ie) {
                ie.printStackTrace();
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(1, taskRunner, this);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
    Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}

Example 69 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class ReduceTaskExternalITCase method testSingleLevelMergeReduceTask.

@Test
public void testSingleLevelMergeReduceTask() {
    final int keyCnt = 8192;
    final int valCnt = 8;
    setNumFileHandlesForSort(2);
    addDriverComparator(this.comparator);
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
    try {
        addInputSorted(new UniformRecordGenerator(keyCnt, valCnt, false), this.comparator.duplicate());
        GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
        testDriver(testTask, MockReduceStub.class);
    } catch (Exception e) {
        LOG.info("Exception while running the test task.", e);
        Assert.fail("Exception in Test: " + e.getMessage());
    }
    Assert.assertTrue("Resultset size was " + this.outList.size() + ". Expected was " + keyCnt, this.outList.size() == keyCnt);
    for (Record record : this.outList) {
        Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == valCnt - record.getField(0, IntValue.class).getValue());
    }
    this.outList.clear();
}

Example 70 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CrossTaskTest method testFailingBlockCrossTask.

@Test
public void testFailingBlockCrossTask() {
    int keyCnt1 = 10;
    int valCnt1 = 1;
    int keyCnt2 = 100;
    int valCnt2 = 4;
    setOutput(this.output);
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_FIRST);
    getTaskConfig().setRelativeMemoryDriver(cross_frac);
    final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
    try {
        testDriver(testTask, MockFailingCrossStub.class);
        Assert.fail("Exception not forwarded.");
    } catch (ExpectedTestException etex) {
    // good!
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test failed due to an exception.");
    }
}