Examples with IdentityMapper org.apache.flink.optimizer.testfunctions.IdentityMapper used on opensource projects

Example 11 with IdentityMapper

use of org.apache.flink.optimizer.testfunctions.IdentityMapper in project flink by apache.

the class SortPartialReuseTest method testCustomPartitioningNotReused.

@Test
public void testCustomPartitioningNotReused() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
        input.partitionCustom(new Partitioner<Long>() {

            @Override
            public int partition(Long key, int numPartitions) {
                return 0;
            }
        }, 0).map(new IdentityMapper<Tuple3<Long, Long, Long>>()).withForwardedFields("0", "1", "2").groupBy(0, 1).reduceGroup(new IdentityGroupReducerCombinable<Tuple3<Long, Long, Long>>()).withForwardedFields("0", "1", "2").groupBy(1).reduceGroup(new IdentityGroupReducerCombinable<Tuple3<Long, Long, Long>>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        SingleInputPlanNode reducer2 = (SingleInputPlanNode) sink.getInput().getSource();
        SingleInputPlanNode combiner = (SingleInputPlanNode) reducer2.getInput().getSource();
        SingleInputPlanNode reducer1 = (SingleInputPlanNode) combiner.getInput().getSource();
        assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
        // should be locally forwarding, reusing sort and partitioning
        assertEquals(ShipStrategyType.PARTITION_HASH, reducer2.getInput().getShipStrategy());
        assertEquals(LocalStrategy.COMBININGSORT, reducer2.getInput().getLocalStrategy());
        assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
        assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
        assertEquals(ShipStrategyType.FORWARD, reducer1.getInput().getShipStrategy());
        assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with IdentityMapper

use of org.apache.flink.optimizer.testfunctions.IdentityMapper in project flink by apache.

the class JobGraphGeneratorTest method testGeneratingJobGraphWithUnconsumedResultPartition.

@Test
public void testGeneratingJobGraphWithUnconsumedResultPartition() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<>(1L, 2L)).setParallelism(1);
    DataSet<Tuple2<Long, Long>> ds = input.map(new IdentityMapper<>()).setParallelism(3);
    AbstractID intermediateDataSetID = new AbstractID();
    // this output branch will be excluded.
    ds.output(BlockingShuffleOutputFormat.createOutputFormat(intermediateDataSetID)).setParallelism(1);
    // this is the normal output branch.
    ds.output(new DiscardingOutputFormat<>()).setParallelism(1);
    JobGraph jobGraph = compileJob(env);
    Assert.assertEquals(3, jobGraph.getVerticesSortedTopologicallyFromSources().size());
    JobVertex mapVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Assert.assertThat(mapVertex, Matchers.instanceOf(JobVertex.class));
    // there are 2 output result with one of them is ResultPartitionType.BLOCKING_PERSISTENT
    Assert.assertEquals(2, mapVertex.getProducedDataSets().size());
    Assert.assertTrue(mapVertex.getProducedDataSets().stream().anyMatch(dataSet -> dataSet.getId().equals(new IntermediateDataSetID(intermediateDataSetID)) && dataSet.getResultType() == ResultPartitionType.BLOCKING_PERSISTENT));
}

Example 13 with IdentityMapper

use of org.apache.flink.optimizer.testfunctions.IdentityMapper in project flink by apache.

the class DistinctAndGroupingOptimizerTest method testDistinctDestroysPartitioningOfNonDistinctFields.

@Test
public void testDistinctDestroysPartitioningOfNonDistinctFields() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L), new Tuple2<Long, Long>(1L, 1L)).map(new IdentityMapper<Tuple2<Long, Long>>()).setParallelism(4);
        data.distinct(1).groupBy(0).sum(1).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
        SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
        SingleInputPlanNode distinctReducer = (SingleInputPlanNode) combiner.getInput().getSource();
        assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
        // reducer must repartition, because it works on a different field
        assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
        // distinct reducer is partitioned
        assertEquals(ShipStrategyType.PARTITION_HASH, distinctReducer.getInput().getShipStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with IdentityMapper

use of org.apache.flink.optimizer.testfunctions.IdentityMapper in project flink by apache.

the class IterationCompilerTest method testWorksetIterationWithUnionRoot.

@Test
public void testWorksetIterationWithUnionRoot() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(43);
        DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20).map(new MapFunction<Long, Tuple2<Long, Long>>() {

            @Override
            public Tuple2<Long, Long> map(Long value) {
                return null;
            }
        });
        DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
        iter.closeWith(iter.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>()).union(iter.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>())), iter.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>()).union(iter.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>()))).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) sink.getInput().getSource();
        // make sure that the root is part of the dynamic path
        // the "NoOp"a that come after the union.
        SingleInputPlanNode nextWorksetNoop = (SingleInputPlanNode) iterNode.getNextWorkSetPlanNode();
        SingleInputPlanNode solutionDeltaNoop = (SingleInputPlanNode) iterNode.getSolutionSetDeltaPlanNode();
        NAryUnionPlanNode nextWorksetUnion = (NAryUnionPlanNode) nextWorksetNoop.getInput().getSource();
        NAryUnionPlanNode solutionDeltaUnion = (NAryUnionPlanNode) solutionDeltaNoop.getInput().getSource();
        assertTrue(nextWorksetNoop.isOnDynamicPath());
        assertTrue(nextWorksetNoop.getCostWeight() >= 1);
        assertTrue(solutionDeltaNoop.isOnDynamicPath());
        assertTrue(solutionDeltaNoop.getCostWeight() >= 1);
        assertTrue(nextWorksetUnion.isOnDynamicPath());
        assertTrue(nextWorksetUnion.getCostWeight() >= 1);
        assertTrue(solutionDeltaUnion.isOnDynamicPath());
        assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 15 with IdentityMapper

use of org.apache.flink.optimizer.testfunctions.IdentityMapper in project flink by apache.

the class IterationCompilerTest method testIterationWithUnionRoot.

@Test
public void testIterationWithUnionRoot() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(43);
        IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);
        iteration.closeWith(iteration.map(new IdentityMapper<Long>()).union(iteration.map(new IdentityMapper<Long>()))).output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        BulkIterationPlanNode iterNode = (BulkIterationPlanNode) sink.getInput().getSource();
        // make sure that the root is part of the dynamic path
        // the "NoOp" that comes after the union.
        SingleInputPlanNode noop = (SingleInputPlanNode) iterNode.getRootOfStepFunction();
        NAryUnionPlanNode union = (NAryUnionPlanNode) noop.getInput().getSource();
        assertTrue(noop.isOnDynamicPath());
        assertTrue(noop.getCostWeight() >= 1);
        assertTrue(union.isOnDynamicPath());
        assertTrue(union.getCostWeight() >= 1);
        // see that the jobgraph generator can translate this
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}