Examples with Costs org.apache.flink.optimizer.costs.Costs used on opensource projects

Example 6 with Costs

use of org.apache.flink.optimizer.costs.Costs in project flink by apache.

the class AllGroupWithPartialPreGroupProperties method instantiate.

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
        // locally connected, directly instantiate
        return new SingleInputPlanNode(node, "GroupReduce (" + node.getOperator().getName() + ")", in, DriverStrategy.ALL_GROUP_REDUCE);
    } else {
        // non forward case.plug in a combiner
        Channel toCombiner = new Channel(in.getSource());
        toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        // create an input node for combine with same parallelism as input node
        GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
        combinerNode.setParallelism(in.getSource().getParallelism());
        SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine (" + node.getOperator().getName() + ")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
        combiner.setCosts(new Costs(0, 0));
        combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
        Channel toReducer = new Channel(combiner);
        toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode());
        toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
        return new SingleInputPlanNode(node, "GroupReduce (" + node.getOperator().getName() + ")", toReducer, DriverStrategy.ALL_GROUP_REDUCE);
    }
}

Example 7 with Costs

use of org.apache.flink.optimizer.costs.Costs in project flink by apache.

the class ReduceProperties method instantiate.

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD || (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) {
        if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
            LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
        }
        return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", in, DriverStrategy.SORTED_REDUCE, this.keyList);
    } else {
        // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
        Channel toCombiner = new Channel(in.getSource());
        toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        // create an input node for combine with same parallelism as input node
        ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
        combinerNode.setParallelism(in.getSource().getParallelism());
        SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine (" + node.getOperator().getName() + ")", toCombiner, this.combinerStrategy, this.keyList);
        combiner.setCosts(new Costs(0, 0));
        combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
        Channel toReducer = new Channel(combiner);
        toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode());
        toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
        return new SingleInputPlanNode(node, "Reduce(" + node.getOperator().getName() + ")", toReducer, DriverStrategy.SORTED_REDUCE, this.keyList);
    }
}

Example 8 with Costs

use of org.apache.flink.optimizer.costs.Costs in project flink by apache.

the class GroupReduceWithCombineProperties method instantiate.

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
        if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
            LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
        }
        // adjust a sort (changes grouping, so it must be for this driver to combining sort
        if (in.getLocalStrategy() == LocalStrategy.SORT) {
            if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
                throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
            }
            in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
        }
        return new SingleInputPlanNode(node, "Reduce(" + node.getOperator().getName() + ")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
    } else {
        // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
        Channel toCombiner = new Channel(in.getSource());
        toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        // create an input node for combine with same parallelism as input node
        GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
        combinerNode.setParallelism(in.getSource().getParallelism());
        SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine(" + node.getOperator().getName() + ")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
        combiner.setCosts(new Costs(0, 0));
        combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
        // set sorting comparator key info
        combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
        // set grouping comparator key info
        combiner.setDriverKeyInfo(this.keyList, 1);
        Channel toReducer = new Channel(combiner);
        toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode());
        if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
            toReducer.setDataDistribution(in.getDataDistribution());
        }
        toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
        return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
    }
}