Examples with FieldList org.apache.flink.api.common.operators.util.FieldList used on opensource projects

Example 51 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class DataSourceNode method setDataPropertiesFromSplitProperties.

private void setDataPropertiesFromSplitProperties(SplitDataProperties splitProps) {
    // set global properties
    int[] partitionKeys = splitProps.getSplitPartitionKeys();
    Partitioner<?> partitioner = splitProps.getSplitPartitioner();
    if (partitionKeys != null && partitioner != null) {
        this.gprops.setCustomPartitioned(new FieldList(partitionKeys), partitioner);
    } else if (partitionKeys != null) {
        this.gprops.setAnyPartitioning(new FieldList(partitionKeys));
    }
    // set local properties
    int[] groupingKeys = splitProps.getSplitGroupKeys();
    Ordering ordering = splitProps.getSplitOrder();
    // adapt split grouping and sorting
    if (ordering != null) {
        // sorting falls back to grouping because a source can read multiple,
        // randomly assigned splits
        groupingKeys = ordering.getFieldPositions();
    }
    if (groupingKeys != null && partitionKeys != null) {
        // check if grouping is also valid across splits, i.e., whether grouping keys are
        // valid superset of partition keys
        boolean allFieldsIncluded = true;
        for (int i : partitionKeys) {
            boolean fieldIncluded = false;
            for (int j : groupingKeys) {
                if (i == j) {
                    fieldIncluded = true;
                    break;
                }
            }
            if (!fieldIncluded) {
                allFieldsIncluded = false;
                break;
            }
        }
        if (allFieldsIncluded) {
            this.lprops = LocalProperties.forGrouping(new FieldList(groupingKeys));
        } else {
            this.lprops = new LocalProperties();
        }
    } else {
        this.lprops = new LocalProperties();
    }
}

Example 52 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class WorksetIterationsJavaApiCompilerTest method testJavaApiWithDeferredSoltionSetUpdateWithMapper.

@Test
public void testJavaApiWithDeferredSoltionSetUpdateWithMapper() {
    try {
        Plan plan = getJavaTestPlan(false, true);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
        DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
        SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
        SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
        // iteration preserves partitioning in reducer, so the first partitioning is out of the
        // loop,
        // the in-loop partitioning is before the final reducer
        // verify joinWithInvariant
        assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
        assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
        assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
        // verify joinWithSolutionSet
        assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
        assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
        // verify reducer
        assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
        assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
        // currently, the system may partition before or after the mapper
        ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
        ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
        assertTrue((ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) || (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH));
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

Example 53 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class WorksetIterationsJavaApiCompilerTest method testJavaApiWithDirectSoltionSetUpdate.

@Test
public void testJavaApiWithDirectSoltionSetUpdate() {
    try {
        Plan plan = getJavaTestPlan(true, false);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
        DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
        SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
        // iteration preserves partitioning in reducer, so the first partitioning is out of the
        // loop,
        // the in-loop partitioning is before the final reducer
        // verify joinWithInvariant
        assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
        assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
        assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
        // verify joinWithSolutionSet
        assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
        assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
        // verify reducer
        assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
        assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
        // verify solution delta
        assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
        assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

Example 54 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class JoinGlobalPropertiesCompatibilityTest method checkInompatiblePartitionings.

@Test
public void checkInompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1);
        final FieldList keysRight = new FieldList(3);
        final Partitioner<Object> part = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        final Partitioner<Object> part2 = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        SortMergeInnerJoinDescriptor descr = new SortMergeInnerJoinDescriptor(keysLeft, keysRight);
        // test incompatible hash with custom partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible custom partitionings
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test incompatible range partitioning with different key size
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible range partitioning with different ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.DESCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist3 = new TestDistribution(1);
        TestDistribution dist4 = new TestDistribution(2);
        // test incompatible range partitioning with different distribution
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist3);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist4);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist3);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist4);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 55 with FieldList

use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.

the class GroupReduceCompilationTest method testGroupedReduceWithFieldPositionKeyNonCombinable.

@Test
public void testGroupedReduceWithFieldPositionKeyNonCombinable() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(8);
        DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class).name("source").setParallelism(6);
        data.groupBy(1).reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {

            public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {
            }
        }).name("reducer").output(new DiscardingOutputFormat<Tuple2<String, Double>>()).name("sink");
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
        // get the original nodes
        SourcePlanNode sourceNode = resolver.getNode("source");
        SingleInputPlanNode reduceNode = resolver.getNode("reducer");
        SinkPlanNode sinkNode = resolver.getNode("sink");
        // check wiring
        assertEquals(sourceNode, reduceNode.getInput().getSource());
        assertEquals(reduceNode, sinkNode.getInput().getSource());
        // check that both reduce and combiner have the same strategy
        assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
        // check the keys
        assertEquals(new FieldList(1), reduceNode.getKeys(0));
        assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
        // check parallelism
        assertEquals(6, sourceNode.getParallelism());
        assertEquals(8, reduceNode.getParallelism());
        assertEquals(8, sinkNode.getParallelism());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
    }
}