Examples with UniformRecordGenerator org.apache.flink.runtime.operators.testutils.UniformRecordGenerator used on opensource projects

Example 91 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testHash4MatchTask.

@Test
public void testHash4MatchTask() {
    int keyCnt1 = 20;
    int valCnt1 = 20;
    int keyCnt2 = 20;
    int valCnt2 = 1;
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    getTaskConfig().setRelativeMemoryDriver(hash_frac);
    JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
    try {
        testDriver(testTask, MockMatchStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test caused an exception.");
    }
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2);
    Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
    this.outList.clear();
}

Example 92 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class JoinTaskTest method testHash1MatchTask.

@Test
public void testHash1MatchTask() {
    int keyCnt1 = 20;
    int valCnt1 = 1;
    int keyCnt2 = 10;
    int valCnt2 = 2;
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    addDriverComparator(this.comparator1);
    addDriverComparator(this.comparator2);
    getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
    getTaskConfig().setRelativeMemoryDriver(hash_frac);
    JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
    try {
        testDriver(testTask, MockMatchStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test caused an exception.");
    }
    final int expCnt = valCnt1 * valCnt2 * Math.min(keyCnt1, keyCnt2);
    Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
    this.outList.clear();
}

Example 93 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class ReduceTaskTest method testCancelReduceTaskWhileReducing.

@Test
public void testCancelReduceTaskWhileReducing() {
    final int keyCnt = 1000;
    final int valCnt = 2;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
    addDriverComparator(this.comparator);
    setOutput(new NirvanaOutputList());
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
    final GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
    final AtomicBoolean success = new AtomicBoolean(false);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testDriver(testTask, MockDelayingReduceStub.class);
                success.set(true);
            } catch (Exception ie) {
                ie.printStackTrace();
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(2, taskRunner, this);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
}

Example 94 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class ReduceTaskTest method testReduceTaskWithSortingInput.

@Test
public void testReduceTaskWithSortingInput() {
    final int keyCnt = 100;
    final int valCnt = 20;
    addDriverComparator(this.comparator);
    setOutput(this.outList);
    getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
    try {
        addInputSorted(new UniformRecordGenerator(keyCnt, valCnt, false), this.comparator.duplicate());
        GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
        testDriver(testTask, MockReduceStub.class);
    } catch (Exception e) {
        LOG.info("Exception while running the test task.", e);
        Assert.fail("Exception in Test: " + e.getMessage());
    }
    Assert.assertTrue("Resultset size was " + this.outList.size() + ". Expected was " + keyCnt, this.outList.size() == keyCnt);
    for (Record record : this.outList) {
        Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == valCnt - record.getField(0, IntValue.class).getValue());
    }
    this.outList.clear();
}

Example 95 with UniformRecordGenerator

use of org.apache.flink.runtime.operators.testutils.UniformRecordGenerator in project flink by apache.

the class ChainTaskTest method testBatchTaskOutputInCloseMethod.

@Test
public void testBatchTaskOutputInCloseMethod() {
    final int numChainedTasks = 10;
    final int keyCnt = 100;
    final int valCnt = 10;
    try {
        initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
        addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
        addOutput(outList);
        registerTask(FlatMapDriver.class, MockMapStub.class);
        for (int i = 0; i < numChainedTasks; i++) {
            final TaskConfig taskConfig = new TaskConfig(new Configuration());
            taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
            taskConfig.setOutputSerializer(serFact);
            taskConfig.setStubWrapper(new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class));
            getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
        }
        final BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);
        testTask.invoke();
        Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}