Examples with SelectorFunctionKeys org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys used on opensource projects

Example 1 with SelectorFunctionKeys

use of org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys in project flink by apache.

the class CoGroupOperator method getSemanticProperties.

@Override
public DualInputSemanticProperties getSemanticProperties() {
    DualInputSemanticProperties props = super.getSemanticProperties();
    // offset semantic information by extracted key fields
    if (props != null && (this.keys1 instanceof SelectorFunctionKeys || this.keys2 instanceof SelectorFunctionKeys)) {
        int numFields1 = this.getInput1Type().getTotalFields();
        int numFields2 = this.getInput2Type().getTotalFields();
        int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
        int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;
        props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
    }
    return props;
}

Example 2 with SelectorFunctionKeys

use of org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys in project flink by apache.

the class GroupReduceOperator method getSemanticProperties.

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {
    SingleInputSemanticProperties props = super.getSemanticProperties();
    // offset semantic information by extracted key fields
    if (props != null && this.grouper != null && this.grouper.keys instanceof SelectorFunctionKeys) {
        int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
        if (this.grouper instanceof SortedGrouping) {
            offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
        }
        props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
    }
    return props;
}

Example 3 with SelectorFunctionKeys

use of org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys in project flink by apache.

the class PartitionOperator method translateSelectorFunctionPartitioner.

@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(SelectorFunctionKeys<T, ?> rawKeys, PartitionMethod pMethod, String name, Operator<T> input, int partitionDop, Partitioner<?> customPartitioner, Order[] orders) {
    final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
    TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
    Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);
    PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput = new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[] { 0 }, name);
    keyedPartitionedInput.setInput(keyedInput);
    keyedPartitionedInput.setCustomPartitioner(customPartitioner);
    keyedPartitionedInput.setParallelism(partitionDop);
    keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));
    return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}

Example 4 with SelectorFunctionKeys

use of org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys in project flink by apache.

the class ReduceOperator method getSemanticProperties.

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {
    SingleInputSemanticProperties props = super.getSemanticProperties();
    // offset semantic information by extracted key fields
    if (props != null && this.grouper != null && this.grouper.keys instanceof SelectorFunctionKeys) {
        int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
        if (this.grouper instanceof SortedGrouping) {
            offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
        }
        props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
    }
    return props;
}

Example 5 with SelectorFunctionKeys

use of org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys in project flink by apache.

the class CoGroupOperator method translateToDataFlow.

@Override
@Internal
protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
        keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
    }
    final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;
    if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
        po = translateSelectorFunctionCoGroup(selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);
        po.setParallelism(getParallelism());
        po.setCustomPartitioner(customPartitioner);
    } else if (keys2 instanceof SelectorFunctionKeys) {
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
        po = translateSelectorFunctionCoGroupRight(logicalKeyPositions1, selectorKeys2, function, getInput1Type(), getResultType(), name, input1, input2);
        po.setParallelism(getParallelism());
        po.setCustomPartitioner(customPartitioner);
    } else if (keys1 instanceof SelectorFunctionKeys) {
        @SuppressWarnings("unchecked") SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        po = translateSelectorFunctionCoGroupLeft(selectorKeys1, logicalKeyPositions2, function, getInput2Type(), getResultType(), name, input1, input2);
    } else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
        try {
            keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException e) {
            throw new InvalidProgramException("The types of the key fields do not match.", e);
        }
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op = new CoGroupOperatorBase<>(function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name);
        op.setFirstInput(input1);
        op.setSecondInput(input2);
        po = op;
    } else {
        throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }
    // configure shared characteristics
    po.setParallelism(getParallelism());
    po.setCustomPartitioner(customPartitioner);
    if (groupSortKeyOrderFirst.size() > 0) {
        Ordering o = new Ordering();
        for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
            o.appendOrdering(entry.getLeft(), null, entry.getRight());
        }
        po.setGroupOrderForInputOne(o);
    }
    if (groupSortKeyOrderSecond.size() > 0) {
        Ordering o = new Ordering();
        for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
            o.appendOrdering(entry.getLeft(), null, entry.getRight());
        }
        po.setGroupOrderForInputTwo(o);
    }
    return po;
}