Examples with FieldList org.apache.flink.api.common.operators.util.FieldList used on opensource projects

Example 56 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class ReduceCompilationTest method testGroupedReduceWithFieldPositionKey.

@Test
public void testGroupedReduceWithFieldPositionKey() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(8);
        DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class).name("source").setParallelism(6);
        data.groupBy(1).reduce(new RichReduceFunction<Tuple2<String, Double>>() {

            @Override
            public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2) {
                return null;
            }
        }).name("reducer").output(new DiscardingOutputFormat<Tuple2<String, Double>>()).name("sink");
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
        // get the original nodes
        SourcePlanNode sourceNode = resolver.getNode("source");
        SingleInputPlanNode reduceNode = resolver.getNode("reducer");
        SinkPlanNode sinkNode = resolver.getNode("sink");
        // get the combiner
        SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
        // check wiring
        assertEquals(sourceNode, combineNode.getInput().getSource());
        assertEquals(reduceNode, sinkNode.getInput().getSource());
        // check the strategies
        assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
        assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
        // check the keys
        assertEquals(new FieldList(1), reduceNode.getKeys(0));
        assertEquals(new FieldList(1), combineNode.getKeys(0));
        assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
        // check parallelism
        assertEquals(6, sourceNode.getParallelism());
        assertEquals(6, combineNode.getParallelism());
        assertEquals(8, reduceNode.getParallelism());
        assertEquals(8, sinkNode.getParallelism());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
    }
}

Example 57 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class ReduceCompilationTest method testGroupedReduceWithHint.

@Test
public void testGroupedReduceWithHint() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(8);
        DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class).name("source").setParallelism(6);
        data.groupBy(new KeySelector<Tuple2<String, Double>, String>() {

            public String getKey(Tuple2<String, Double> value) {
                return value.f0;
            }
        }).reduce(new RichReduceFunction<Tuple2<String, Double>>() {

            @Override
            public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2) {
                return null;
            }
        }).setCombineHint(CombineHint.HASH).name("reducer").output(new DiscardingOutputFormat<Tuple2<String, Double>>()).name("sink");
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
        // get the original nodes
        SourcePlanNode sourceNode = resolver.getNode("source");
        SingleInputPlanNode reduceNode = resolver.getNode("reducer");
        SinkPlanNode sinkNode = resolver.getNode("sink");
        // get the combiner
        SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
        // get the key extractors and projectors
        SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
        SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
        // check wiring
        assertEquals(sourceNode, keyExtractor.getInput().getSource());
        assertEquals(keyProjector, sinkNode.getInput().getSource());
        // check the strategies
        assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
        assertEquals(DriverStrategy.HASHED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
        // check the keys
        assertEquals(new FieldList(0), reduceNode.getKeys(0));
        assertEquals(new FieldList(0), combineNode.getKeys(0));
        assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
        // check parallelism
        assertEquals(6, sourceNode.getParallelism());
        assertEquals(6, keyExtractor.getParallelism());
        assertEquals(6, combineNode.getParallelism());
        assertEquals(8, reduceNode.getParallelism());
        assertEquals(8, keyProjector.getParallelism());
        assertEquals(8, sinkNode.getParallelism());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
    }
}

Example 58 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class ReduceCompilationTest method testGroupedReduceWithoutCombiner.

/**
 * Test program compilation when the Reduce's combiner has been excluded by setting {@code
 * CombineHint.NONE}.
 */
@Test
public void testGroupedReduceWithoutCombiner() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(8);
    DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class).name("source").setParallelism(6);
    data.groupBy(0).reduce(new RichReduceFunction<Tuple2<String, Double>>() {

        @Override
        public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2) {
            return null;
        }
    }).setCombineHint(CombineHint.NONE).name("reducer").output(new DiscardingOutputFormat<Tuple2<String, Double>>()).name("sink");
    Plan p = env.createProgramPlan();
    OptimizedPlan op = compileNoStats(p);
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
    // get the original nodes
    SourcePlanNode sourceNode = resolver.getNode("source");
    SingleInputPlanNode reduceNode = resolver.getNode("reducer");
    SinkPlanNode sinkNode = resolver.getNode("sink");
    // check wiring
    assertEquals(sourceNode, reduceNode.getInput().getSource());
    // check the strategies
    assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
    // check the keys
    assertEquals(new FieldList(0), reduceNode.getKeys(0));
    assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
    // check parallelism
    assertEquals(6, sourceNode.getParallelism());
    assertEquals(8, reduceNode.getParallelism());
    assertEquals(8, sinkNode.getParallelism());
}

Example 59 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class CoGroupGlobalPropertiesCompatibilityTest method checkInompatiblePartitionings.

@Test
public void checkInompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1);
        final FieldList keysRight = new FieldList(3);
        final Partitioner<Object> part = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        final Partitioner<Object> part2 = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
        // test incompatible hash with custom partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible custom partitionings
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test incompatible range partitioning with different key size
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible range partitioning with different ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.DESCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist3 = new TestDistribution(1);
        TestDistribution dist4 = new TestDistribution(2);
        // test incompatible range partitioning with different distribution
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist3);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist4);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist3);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist4);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 60 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class GlobalPropertiesFilteringTest method testHashPartitioningErased.

@Test
public void testHashPartitioningErased() {
    SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "0;1" }, null, null, tupleInfo, tupleInfo);
    GlobalProperties gprops = new GlobalProperties();
    gprops.setHashPartitioned(new FieldList(0, 1, 4));
    GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
    assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
    assertNull(result.getPartitioningFields());
}