Examples with UniformRecordGenerator org.apache.flink.runtime.operators.testutils.UniformRecordGenerator used on opensource projects

Example 61 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CoGroupTaskTest method testSortFirstCoGroupTask.

@Test
public void testSortFirstCoGroupTask() {
    int keyCnt1 = 200;
    int valCnt1 = 2;
    int keyCnt2 = 200;
    int valCnt2 = 4;
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2) + (keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
    setOutput(this.output);
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
    final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
    try {
        addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
        addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
        testDriver(testTask, MockCoGroupStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test caused an exception.");
    }
    Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}

Example 62 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class CrossTaskExternalITCase method testExternalBlockCrossTask.

@Test
public void testExternalBlockCrossTask() {
    int keyCnt1 = 2;
    int valCnt1 = 1;
    // 43690 fit into memory, 43691 do not!
    int keyCnt2 = 43700;
    int valCnt2 = 1;
    final int expCnt = keyCnt1 * valCnt1 * keyCnt2 * valCnt2;
    setOutput(this.output);
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_FIRST);
    getTaskConfig().setRelativeMemoryDriver(cross_frac);
    final CrossDriver<Record, Record, Record> testTask = new CrossDriver<Record, Record, Record>();
    try {
        testDriver(testTask, MockCrossStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test failed due to an exception.");
    }
    Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}

Example 63 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class DataSinkTaskTest method testFailingSortingDataSinkTask.

@Test
@SuppressWarnings("unchecked")
public void testFailingSortingDataSinkTask() {
    int keyCnt = 100;
    int valCnt = 20;
    double memoryFraction = 1.0;
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    super.addInput(new UniformRecordGenerator(keyCnt, valCnt, true), 0);
    DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
    Configuration stubParams = new Configuration();
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(new RecordComparatorFactory(new int[] { 1 }, (new Class[] { IntValue.class })), 0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
    File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
    super.registerFileOutputTask(MockFailingOutputFormat.class, tempTestFile.toURI().toString(), stubParams);
    boolean stubFailed = false;
    try {
        testTask.invoke();
    } catch (Exception e) {
        stubFailed = true;
    }
    Assert.assertTrue("Function exception was not forwarded.", stubFailed);
    // assert that temp file was removed
    Assert.assertFalse("Temp output file has not been removed", tempTestFile.exists());
}

Example 64 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class DataSinkTaskTest method testFailingDataSinkTask.

@Test
public void testFailingDataSinkTask() {
    int keyCnt = 100;
    int valCnt = 20;
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    super.addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
    DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
    Configuration stubParams = new Configuration();
    File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
    super.registerFileOutputTask(MockFailingOutputFormat.class, tempTestFile.toURI().toString(), stubParams);
    boolean stubFailed = false;
    try {
        testTask.invoke();
    } catch (Exception e) {
        stubFailed = true;
    }
    Assert.assertTrue("Function exception was not forwarded.", stubFailed);
    // assert that temp file was removed
    Assert.assertFalse("Temp output file has not been removed", tempTestFile.exists());
}

Example 65 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class FlatMapTaskTest method testFailingMapTask.

@Test
public void testFailingMapTask() {
    final int keyCnt = 100;
    final int valCnt = 20;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
    setOutput(new DiscardingOutputCollector<Record>());
    final FlatMapDriver<Record, Record> testTask = new FlatMapDriver<>();
    try {
        testDriver(testTask, MockFailingMapStub.class);
        Assert.fail("Function exception was not forwarded.");
    } catch (ExpectedTestException e) {
    // good!
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Exception in test.");
    }
}