Examples with Transformation org.apache.flink.api.dag.Transformation used on opensource projects

Example 66 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecCorrelate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig()).setOperatorBaseClass(operatorBaseClass);
    return CorrelateCodeGenerator.generateCorrelateTransformation(config.getTableConfig(), ctx, inputTransform, (RowType) inputEdge.getOutputType(), invocation, JavaScalaConversionUtil.toScala(Optional.ofNullable(condition)), (RowType) getOutputType(), joinType, inputTransform.getParallelism(), retainHeader, getClass().getSimpleName(), createTransformationMeta(CORRELATE_TRANSFORMATION, config));
}

Example 67 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecExpand method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final CodeGenOperatorFactory<RowData> operatorFactory = ExpandCodeGenerator.generateExpandOperator(new CodeGeneratorContext(config.getTableConfig()), (RowType) inputEdge.getOutputType(), (RowType) getOutputType(), projects, retainHeader, getClass().getSimpleName());
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(EXPAND_TRANSFORMATION, config), operatorFactory, InternalTypeInfo.of(getOutputType()), inputTransform.getParallelism());
}

Example 68 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecSink method applyUpsertMaterialize.

private Transformation<RowData> applyUpsertMaterialize(Transformation<RowData> inputTransform, int[] primaryKeys, int sinkParallelism, ReadableConfig config, RowType physicalRowType) {
    GeneratedRecordEqualiser equaliser = new EqualiserCodeGenerator(physicalRowType).generateRecordEqualiser("SinkMaterializeEqualiser");
    SinkUpsertMaterializer operator = new SinkUpsertMaterializer(StateConfigUtil.createTtlConfig(config.get(ExecutionConfigOptions.IDLE_STATE_RETENTION).toMillis()), InternalSerializers.create(physicalRowType), equaliser);
    final String[] fieldNames = physicalRowType.getFieldNames().toArray(new String[0]);
    final List<String> pkFieldNames = Arrays.stream(primaryKeys).mapToObj(idx -> fieldNames[idx]).collect(Collectors.toList());
    OneInputTransformation<RowData, RowData> materializeTransform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(UPSERT_MATERIALIZE_TRANSFORMATION, String.format("SinkMaterializer(pk=[%s])", String.join(", ", pkFieldNames)), "SinkMaterializer", config), operator, inputTransform.getOutputType(), sinkParallelism);
    RowDataKeySelector keySelector = KeySelectorUtil.getRowDataSelector(primaryKeys, InternalTypeInfo.of(physicalRowType));
    materializeTransform.setStateKeySelector(keySelector);
    materializeTransform.setStateKeyType(keySelector.getProducedType());
    return materializeTransform;
}

Example 69 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecSink method applySinkProvider.

private Transformation<?> applySinkProvider(Transformation<RowData> inputTransform, StreamExecutionEnvironment env, SinkRuntimeProvider runtimeProvider, int rowtimeFieldIndex, int sinkParallelism, ReadableConfig config) {
    TransformationMetadata sinkMeta = createTransformationMeta(SINK_TRANSFORMATION, config);
    if (runtimeProvider instanceof DataStreamSinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final DataStreamSinkProvider provider = (DataStreamSinkProvider) runtimeProvider;
        return provider.consumeDataStream(createProviderContext(), dataStream).getTransformation();
    } else if (runtimeProvider instanceof TransformationSinkProvider) {
        final TransformationSinkProvider provider = (TransformationSinkProvider) runtimeProvider;
        return provider.createTransformation(new TransformationSinkProvider.Context() {

            @Override
            public Transformation<RowData> getInputTransformation() {
                return inputTransform;
            }

            @Override
            public int getRowtimeIndex() {
                return rowtimeFieldIndex;
            }

            @Override
            public Optional<String> generateUid(String name) {
                return createProviderContext().generateUid(name);
            }
        });
    } else if (runtimeProvider instanceof SinkFunctionProvider) {
        final SinkFunction<RowData> sinkFunction = ((SinkFunctionProvider) runtimeProvider).createSinkFunction();
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof OutputFormatProvider) {
        OutputFormat<RowData> outputFormat = ((OutputFormatProvider) runtimeProvider).createOutputFormat();
        final SinkFunction<RowData> sinkFunction = new OutputFormatSinkFunction<>(outputFormat);
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof SinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSinkV1(dataStream, ((SinkProvider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else if (runtimeProvider instanceof SinkV2Provider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSink(dataStream, ((SinkV2Provider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else {
        throw new TableException("Unsupported sink runtime provider.");
    }
}

Example 70 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecSink method createSinkTransformation.

@SuppressWarnings("unchecked")
protected Transformation<Object> createSinkTransformation(StreamExecutionEnvironment streamExecEnv, ReadableConfig config, Transformation<RowData> inputTransform, DynamicTableSink tableSink, int rowtimeFieldIndex, boolean upsertMaterialize) {
    final ResolvedSchema schema = tableSinkSpec.getContextResolvedTable().getResolvedSchema();
    final SinkRuntimeProvider runtimeProvider = tableSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(isBounded));
    final RowType physicalRowType = getPhysicalRowType(schema);
    final int[] primaryKeys = getPrimaryKeyIndices(physicalRowType, schema);
    final int sinkParallelism = deriveSinkParallelism(inputTransform, runtimeProvider);
    final int inputParallelism = inputTransform.getParallelism();
    final boolean inputInsertOnly = inputChangelogMode.containsOnly(RowKind.INSERT);
    final boolean hasPk = primaryKeys.length > 0;
    if (!inputInsertOnly && sinkParallelism != inputParallelism && !hasPk) {
        throw new TableException(String.format("The sink for table '%s' has a configured parallelism of %s, while the input parallelism is %s. " + "Since the configured parallelism is different from the input's parallelism and " + "the changelog mode is not insert-only, a primary key is required but could not " + "be found.", tableSinkSpec.getContextResolvedTable().getIdentifier().asSummaryString(), sinkParallelism, inputParallelism));
    }
    // only add materialization if input has change
    final boolean needMaterialization = !inputInsertOnly && upsertMaterialize;
    Transformation<RowData> sinkTransform = applyConstraintValidations(inputTransform, config, physicalRowType);
    if (hasPk) {
        sinkTransform = applyKeyBy(config, sinkTransform, primaryKeys, sinkParallelism, inputParallelism, inputInsertOnly, needMaterialization);
    }
    if (needMaterialization) {
        sinkTransform = applyUpsertMaterialize(sinkTransform, primaryKeys, sinkParallelism, config, physicalRowType);
    }
    return (Transformation<Object>) applySinkProvider(sinkTransform, streamExecEnv, runtimeProvider, rowtimeFieldIndex, sinkParallelism, config);
}