Examples with InternalTypeInfo org.apache.flink.table.runtime.typeutils.InternalTypeInfo used on opensource projects

Example 1 with InternalTypeInfo

use of org.apache.flink.table.runtime.typeutils.InternalTypeInfo in project flink by apache.

the class StreamExecDeduplicate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    final InternalTypeInfo<RowData> rowTypeInfo = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
    final TypeSerializer<RowData> rowSerializer = rowTypeInfo.createSerializer(planner.getExecEnv().getConfig());
    final OneInputStreamOperator<RowData, RowData> operator;
    if (isRowtime) {
        operator = new RowtimeDeduplicateOperatorTranslator(config, rowTypeInfo, rowSerializer, inputRowType, keepLastRow, generateUpdateBefore).createDeduplicateOperator();
    } else {
        operator = new ProcTimeDeduplicateOperatorTranslator(config, rowTypeInfo, rowSerializer, inputRowType, keepLastRow, generateUpdateBefore).createDeduplicateOperator();
    }
    final OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(DEDUPLICATE_TRANSFORMATION, config), operator, rowTypeInfo, inputTransform.getParallelism());
    final RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(uniqueKeys, rowTypeInfo);
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    return transform;
}

Example 2 with InternalTypeInfo

use of org.apache.flink.table.runtime.typeutils.InternalTypeInfo in project flink by apache.

the class StreamExecExchange method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final Transformation<RowData> inputTransform = (Transformation<RowData>) getInputEdges().get(0).translateToPlan(planner);
    final StreamPartitioner<RowData> partitioner;
    final int parallelism;
    final InputProperty inputProperty = getInputProperties().get(0);
    final InputProperty.DistributionType distributionType = inputProperty.getRequiredDistribution().getType();
    switch(distributionType) {
        case SINGLETON:
            partitioner = new GlobalPartitioner<>();
            parallelism = 1;
            break;
        case HASH:
            // TODO Eliminate duplicate keys
            int[] keys = ((HashDistribution) inputProperty.getRequiredDistribution()).getKeys();
            InternalTypeInfo<RowData> inputType = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
            RowDataKeySelector keySelector = KeySelectorUtil.getRowDataSelector(keys, inputType);
            partitioner = new KeyGroupStreamPartitioner<>(keySelector, DEFAULT_LOWER_BOUND_MAX_PARALLELISM);
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        default:
            throw new TableException(String.format("%s is not supported now!", distributionType));
    }
    final Transformation<RowData> transformation = new PartitionTransformation<>(inputTransform, partitioner);
    createTransformationMeta(EXCHANGE_TRANSFORMATION, config).fill(transformation);
    transformation.setParallelism(parallelism);
    transformation.setOutputType(InternalTypeInfo.of(getOutputType()));
    return transformation;
}

Example 3 with InternalTypeInfo

use of org.apache.flink.table.runtime.typeutils.InternalTypeInfo in project flink by apache.

the class StreamExecIntervalJoin method createProcTimeJoin.

private TwoInputTransformation<RowData, RowData, RowData> createProcTimeJoin(Transformation<RowData> leftInputTransform, Transformation<RowData> rightInputTransform, InternalTypeInfo<RowData> returnTypeInfo, IntervalJoinFunction joinFunction, JoinSpec joinSpec, IntervalJoinSpec.WindowBounds windowBounds, ReadableConfig config) {
    InternalTypeInfo<RowData> leftTypeInfo = (InternalTypeInfo<RowData>) leftInputTransform.getOutputType();
    InternalTypeInfo<RowData> rightTypeInfo = (InternalTypeInfo<RowData>) rightInputTransform.getOutputType();
    ProcTimeIntervalJoin procJoinFunc = new ProcTimeIntervalJoin(joinSpec.getJoinType(), windowBounds.getLeftLowerBound(), windowBounds.getLeftUpperBound(), leftTypeInfo, rightTypeInfo, joinFunction);
    return ExecNodeUtil.createTwoInputTransformation(leftInputTransform, rightInputTransform, createTransformationMeta(INTERVAL_JOIN_TRANSFORMATION, config), new KeyedCoProcessOperator<>(procJoinFunc), returnTypeInfo, leftInputTransform.getParallelism());
}

Example 4 with InternalTypeInfo

use of org.apache.flink.table.runtime.typeutils.InternalTypeInfo in project flink by apache.

the class StreamExecMatch method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    checkOrderKeys(inputRowType);
    final EventComparator<RowData> eventComparator = createEventComparator(config, inputRowType);
    final Transformation<RowData> timestampedInputTransform = translateOrder(inputTransform, inputRowType);
    final Tuple2<Pattern<RowData, RowData>, List<String>> cepPatternAndNames = translatePattern(matchSpec, config.getTableConfig(), planner.getRelBuilder(), inputRowType);
    final Pattern<RowData, RowData> cepPattern = cepPatternAndNames.f0;
    // TODO remove this once it is supported in CEP library
    if (NFACompiler.canProduceEmptyMatches(cepPattern)) {
        throw new TableException("Patterns that can produce empty matches are not supported. There must be at least one non-optional state.");
    }
    // TODO remove this once it is supported in CEP library
    if (cepPattern.getQuantifier().hasProperty(Quantifier.QuantifierProperty.GREEDY)) {
        throw new TableException("Greedy quantifiers are not allowed as the last element of a Pattern yet. " + "Finish your pattern with either a simple variable or reluctant quantifier.");
    }
    if (matchSpec.isAllRows()) {
        throw new TableException("All rows per match mode is not supported yet.");
    }
    final int[] partitionKeys = matchSpec.getPartition().getFieldIndices();
    final SortSpec.SortFieldSpec timeOrderField = matchSpec.getOrderKeys().getFieldSpec(0);
    final LogicalType timeOrderFieldType = inputRowType.getTypeAt(timeOrderField.getFieldIndex());
    final boolean isProctime = TypeCheckUtils.isProcTime(timeOrderFieldType);
    final InternalTypeInfo<RowData> inputTypeInfo = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
    final TypeSerializer<RowData> inputSerializer = inputTypeInfo.createSerializer(planner.getExecEnv().getConfig());
    final NFACompiler.NFAFactory<RowData> nfaFactory = NFACompiler.compileFactory(cepPattern, false);
    final MatchCodeGenerator generator = new MatchCodeGenerator(new CodeGeneratorContext(config.getTableConfig()), planner.getRelBuilder(), // nullableInput
    false, JavaScalaConversionUtil.toScala(cepPatternAndNames.f1), JavaScalaConversionUtil.toScala(Optional.empty()), CodeGenUtils.DEFAULT_COLLECTOR_TERM());
    generator.bindInput(inputRowType, CodeGenUtils.DEFAULT_INPUT1_TERM(), JavaScalaConversionUtil.toScala(Optional.empty()));
    final PatternProcessFunctionRunner patternProcessFunction = generator.generateOneRowPerMatchExpression((RowType) getOutputType(), partitionKeys, matchSpec.getMeasures());
    final CepOperator<RowData, RowData, RowData> operator = new CepOperator<>(inputSerializer, isProctime, nfaFactory, eventComparator, cepPattern.getAfterMatchSkipStrategy(), patternProcessFunction, null);
    final OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(timestampedInputTransform, createTransformationMeta(MATCH_TRANSFORMATION, config), operator, InternalTypeInfo.of(getOutputType()), timestampedInputTransform.getParallelism());
    final RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(partitionKeys, inputTypeInfo);
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    if (inputsContainSingleton()) {
        transform.setParallelism(1);
        transform.setMaxParallelism(1);
    }
    return transform;
}

Example 5 with InternalTypeInfo

use of org.apache.flink.table.runtime.typeutils.InternalTypeInfo in project flink by apache.

the class StreamExecIntervalJoin method createRowTimeJoin.

private TwoInputTransformation<RowData, RowData, RowData> createRowTimeJoin(Transformation<RowData> leftInputTransform, Transformation<RowData> rightInputTransform, InternalTypeInfo<RowData> returnTypeInfo, IntervalJoinFunction joinFunction, JoinSpec joinSpec, IntervalJoinSpec.WindowBounds windowBounds, ReadableConfig config) {
    InternalTypeInfo<RowData> leftTypeInfo = (InternalTypeInfo<RowData>) leftInputTransform.getOutputType();
    InternalTypeInfo<RowData> rightTypeInfo = (InternalTypeInfo<RowData>) rightInputTransform.getOutputType();
    RowTimeIntervalJoin rowJoinFunc = new RowTimeIntervalJoin(joinSpec.getJoinType(), windowBounds.getLeftLowerBound(), windowBounds.getLeftUpperBound(), // allowedLateness
    0L, leftTypeInfo, rightTypeInfo, joinFunction, windowBounds.getLeftTimeIdx(), windowBounds.getRightTimeIdx());
    return ExecNodeUtil.createTwoInputTransformation(leftInputTransform, rightInputTransform, createTransformationMeta(INTERVAL_JOIN_TRANSFORMATION, config), new KeyedCoProcessOperatorWithWatermarkDelay<>(rowJoinFunc, rowJoinFunc.getMaxOutputDelay()), returnTypeInfo, leftInputTransform.getParallelism());
}