Examples with Ordering org.apache.flink.api.common.operators.Ordering used on opensource projects

Example 46 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class CoGroupOperator method translateToDataFlow.

@Override
@Internal
protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
        keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
    }
    final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;
    if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
        po = translateSelectorFunctionCoGroup(selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);
        po.setParallelism(getParallelism());
        po.setCustomPartitioner(customPartitioner);
    } else if (keys2 instanceof SelectorFunctionKeys) {
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
        po = translateSelectorFunctionCoGroupRight(logicalKeyPositions1, selectorKeys2, function, getInput1Type(), getResultType(), name, input1, input2);
        po.setParallelism(getParallelism());
        po.setCustomPartitioner(customPartitioner);
    } else if (keys1 instanceof SelectorFunctionKeys) {
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        po = translateSelectorFunctionCoGroupLeft(selectorKeys1, logicalKeyPositions2, function, getInput2Type(), getResultType(), name, input1, input2);
    } else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
        try {
            keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException e) {
            throw new InvalidProgramException("The types of the key fields do not match.", e);
        }
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op = new CoGroupOperatorBase<>(function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name);
        op.setFirstInput(input1);
        op.setSecondInput(input2);
        po = op;
    } else {
        throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }
    // configure shared characteristics
    po.setParallelism(getParallelism());
    po.setCustomPartitioner(customPartitioner);
    if (groupSortKeyOrderFirst.size() > 0) {
        Ordering o = new Ordering();
        for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
            o.appendOrdering(entry.getLeft(), null, entry.getRight());
        }
        po.setGroupOrderForInputOne(o);
    }
    if (groupSortKeyOrderSecond.size() > 0) {
        Ordering o = new Ordering();
        for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
            o.appendOrdering(entry.getLeft(), null, entry.getRight());
        }
        po.setGroupOrderForInputTwo(o);
    }
    return po;
}

Example 47 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class PartitionOperator method computeOrdering.

private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) {
    Ordering ordering = new Ordering();
    final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions();
    if (orders == null) {
        for (int key : logicalKeyPositions) {
            ordering.appendOrdering(key, null, Order.ASCENDING);
        }
    } else {
        final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes();
        int index = 0;
        for (int i = 0; i < originalKeyFieldTypes.length; i++) {
            final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields();
            for (int j = index; j < index + typeTotalFields; j++) {
                ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]);
            }
            index += typeTotalFields;
        }
    }
    return ordering;
}

Example 48 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class SortPartitionOperator method translateToDataFlowWithKeyExtractor.

private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
    TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
    Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);
    Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);
    int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
    Ordering partitionOrdering = new Ordering();
    for (int keyPosition : sortKeyPositions) {
        partitionOrdering.appendOrdering(keyPosition, null, order);
    }
    // distinguish between partition types
    UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
    SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
    noop.setInput(keyedInput);
    if (this.getParallelism() < 0) {
        // use parallelism of input if not explicitly specified
        noop.setParallelism(input.getParallelism());
    } else {
        // use explicitly specified parallelism
        noop.setParallelism(this.getParallelism());
    }
    return KeyFunctions.appendKeyRemover(noop, keys);
}

Example 49 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class SortPartitionOperator method translateToDataFlow.

// --------------------------------------------------------------------------------------------
//  Translation
// --------------------------------------------------------------------------------------------
protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {
    String name = "Sort at " + sortLocationName;
    if (useKeySelector) {
        return translateToDataFlowWithKeyExtractor(input, (Keys.SelectorFunctionKeys<T, ?>) keys.get(0), orders.get(0), name);
    }
    // flatten sort key positions
    List<Integer> allKeyPositions = new ArrayList<>();
    List<Order> allOrders = new ArrayList<>();
    for (int i = 0, length = keys.size(); i < length; i++) {
        int[] sortKeyPositions = keys.get(i).computeLogicalKeyPositions();
        Order order = orders.get(i);
        for (int sortKeyPosition : sortKeyPositions) {
            allKeyPositions.add(sortKeyPosition);
            allOrders.add(order);
        }
    }
    Ordering partitionOrdering = new Ordering();
    for (int i = 0, length = allKeyPositions.size(); i < length; i++) {
        partitionOrdering.appendOrdering(allKeyPositions.get(i), null, allOrders.get(i));
    }
    // distinguish between partition types
    UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<>(getType(), getType());
    SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
    noop.setInput(input);
    if (this.getParallelism() < 0) {
        // use parallelism of input if not explicitly specified
        noop.setParallelism(input.getParallelism());
    } else {
        // use explicitly specified parallelism
        noop.setParallelism(this.getParallelism());
    }
    return noop;
}

Example 50 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class DataSink method translateToDataFlow.

// --------------------------------------------------------------------------------------------
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) {
    // select the name (or create a default one)
    String name = this.name != null ? this.name : this.format.toString();
    GenericDataSinkBase<T> sink = new GenericDataSinkBase<>(this.format, new UnaryOperatorInformation<>(this.type, new NothingTypeInfo()), name);
    // set input
    sink.setInput(input);
    // set parameters
    if (this.parameters != null) {
        sink.getParameters().addAll(this.parameters);
    }
    // set parallelism
    if (this.parallelism > 0) {
        // use specified parallelism
        sink.setParallelism(this.parallelism);
    } else {
        // if no parallelism has been specified, use parallelism of input operator to enable chaining
        sink.setParallelism(input.getParallelism());
    }
    if (this.sortKeyPositions != null) {
        // configure output sorting
        Ordering ordering = new Ordering();
        for (int i = 0; i < this.sortKeyPositions.length; i++) {
            ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]);
        }
        sink.setLocalOrder(ordering);
    }
    return sink;
}