Examples with FieldList org.apache.flink.api.common.operators.util.FieldList used on opensource projects

Example 31 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class GlobalPropertiesFilteringTest method testAllErased2.

@Test
public void testAllErased2() {
    SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(semProps, new String[] { "2" }, null, null, tupleInfo, tupleInfo);
    GlobalProperties gprops = new GlobalProperties();
    gprops.setHashPartitioned(new FieldList(0, 1));
    gprops.addUniqueFieldCombination(new FieldSet(3, 4));
    gprops.addUniqueFieldCombination(new FieldSet(5, 6));
    GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);
    assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
    assertNull(result.getPartitioningFields());
    assertNull(result.getPartitioningOrdering());
    assertNull(result.getUniqueFieldCombination());
}

Example 32 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class GlobalPropertiesFilteringTest method testAllErased1.

@Test
public void testAllErased1() {
    SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
    GlobalProperties gprops = new GlobalProperties();
    gprops.setHashPartitioned(new FieldList(0, 1));
    gprops.addUniqueFieldCombination(new FieldSet(3, 4));
    gprops.addUniqueFieldCombination(new FieldSet(5, 6));
    GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);
    assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
    assertNull(result.getPartitioningFields());
    assertNull(result.getPartitioningOrdering());
    assertNull(result.getUniqueFieldCombination());
}

Example 33 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class GlobalPropertiesFilteringTest method testAnyPartitioningPreserved2.

@Test
public void testAnyPartitioningPreserved2() {
    SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "0->1; 1->2; 4->3" }, null, null, tupleInfo, tupleInfo);
    GlobalProperties gprops = new GlobalProperties();
    gprops.setAnyPartitioning(new FieldList(0, 1, 4));
    GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
    assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
    FieldList pFields = result.getPartitioningFields();
    assertEquals(3, pFields.size());
    assertTrue(pFields.contains(1));
    assertTrue(pFields.contains(2));
    assertTrue(pFields.contains(3));
}

Example 34 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class GlobalPropertiesFilteringTest method testRangePartitioningPreserved1.

@Test
public void testRangePartitioningPreserved1() {
    SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "1;2;5" }, null, null, tupleInfo, tupleInfo);
    Ordering o = new Ordering();
    o.appendOrdering(1, IntValue.class, Order.ASCENDING);
    o.appendOrdering(5, LongValue.class, Order.DESCENDING);
    o.appendOrdering(2, StringValue.class, Order.ASCENDING);
    GlobalProperties gprops = new GlobalProperties();
    gprops.setRangePartitioned(o);
    GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
    assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
    FieldList pFields = result.getPartitioningFields();
    assertEquals(3, pFields.size());
    assertEquals(1, pFields.get(0).intValue());
    assertEquals(5, pFields.get(1).intValue());
    assertEquals(2, pFields.get(2).intValue());
    Ordering pOrder = result.getPartitioningOrdering();
    assertEquals(3, pOrder.getNumberOfFields());
    assertEquals(1, pOrder.getFieldNumber(0).intValue());
    assertEquals(5, pOrder.getFieldNumber(1).intValue());
    assertEquals(2, pOrder.getFieldNumber(2).intValue());
    assertEquals(Order.ASCENDING, pOrder.getOrder(0));
    assertEquals(Order.DESCENDING, pOrder.getOrder(1));
    assertEquals(Order.ASCENDING, pOrder.getOrder(2));
    assertEquals(IntValue.class, pOrder.getType(0));
    assertEquals(LongValue.class, pOrder.getType(1));
    assertEquals(StringValue.class, pOrder.getType(2));
}

Example 35 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class ReduceCompilationTest method testGroupedReduceWithSelectorFunctionKey.

@Test
public void testGroupedReduceWithSelectorFunctionKey() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(8);
        DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class).name("source").setParallelism(6);
        data.groupBy(new KeySelector<Tuple2<String, Double>, String>() {

            public String getKey(Tuple2<String, Double> value) {
                return value.f0;
            }
        }).reduce(new RichReduceFunction<Tuple2<String, Double>>() {

            @Override
            public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2) {
                return null;
            }
        }).name("reducer").output(new DiscardingOutputFormat<Tuple2<String, Double>>()).name("sink");
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
        // get the original nodes
        SourcePlanNode sourceNode = resolver.getNode("source");
        SingleInputPlanNode reduceNode = resolver.getNode("reducer");
        SinkPlanNode sinkNode = resolver.getNode("sink");
        // get the combiner
        SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
        // get the key extractors and projectors
        SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
        SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
        // check wiring
        assertEquals(sourceNode, keyExtractor.getInput().getSource());
        assertEquals(keyProjector, sinkNode.getInput().getSource());
        // check the strategies
        assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
        assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
        // check the keys
        assertEquals(new FieldList(0), reduceNode.getKeys(0));
        assertEquals(new FieldList(0), combineNode.getKeys(0));
        assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
        // check parallelism
        assertEquals(6, sourceNode.getParallelism());
        assertEquals(6, keyExtractor.getParallelism());
        assertEquals(6, combineNode.getParallelism());
        assertEquals(8, reduceNode.getParallelism());
        assertEquals(8, keyProjector.getParallelism());
        assertEquals(8, sinkNode.getParallelism());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
    }
}