Examples with Transformation org.apache.flink.api.dag.Transformation used on opensource projects

Example 61 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class SideOutputTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final SideOutputTransformation<OUT> transformation, final Context context) {
    checkNotNull(transformation);
    checkNotNull(context);
    final StreamGraph streamGraph = context.getStreamGraph();
    final List<Transformation<?>> parentTransformations = transformation.getInputs();
    checkState(parentTransformations.size() == 1, "Expected exactly one input transformation but found " + parentTransformations.size());
    final List<Integer> virtualResultIds = new ArrayList<>();
    final Transformation<?> parentTransformation = parentTransformations.get(0);
    for (int inputId : context.getStreamNodeIds(parentTransformation)) {
        final int virtualId = Transformation.getNewNodeId();
        streamGraph.addVirtualSideOutputNode(inputId, virtualId, transformation.getOutputTag());
        virtualResultIds.add(virtualId);
    }
    return virtualResultIds;
}

Example 62 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class StreamGraphGeneratorTest method testResetBatchExchangeModeInStreamingExecution.

@Test
public void testResetBatchExchangeModeInStreamingExecution() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    PartitionTransformation<Integer> transformation = new PartitionTransformation<>(sourceDataStream.getTransformation(), new RebalancePartitioner<>(), StreamExchangeMode.BATCH);
    DataStream<Integer> partitionStream = new DataStream<>(env, transformation);
    partitionStream.map(value -> value).print();
    final StreamGraph streamGraph = env.getStreamGraph();
    Assertions.assertThat(streamGraph.getStreamEdges(1, 3)).hasSize(1).satisfies(e -> Assertions.assertThat(e.get(0).getExchangeMode()).isEqualTo(StreamExchangeMode.UNDEFINED));
}

Example 63 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class TableEnvironmentImpl method executeInternal.

@Override
public TableResultInternal executeInternal(List<ModifyOperation> operations) {
    List<Transformation<?>> transformations = translate(operations);
    List<String> sinkIdentifierNames = extractSinkIdentifierNames(operations);
    TableResultInternal result = executeInternal(transformations, sinkIdentifierNames);
    if (tableConfig.getConfiguration().get(TABLE_DML_SYNC)) {
        try {
            result.await();
        } catch (InterruptedException | ExecutionException e) {
            result.getJobClient().ifPresent(JobClient::cancel);
            throw new TableException("Fail to wait execution finish.", e);
        }
    }
    return result;
}

Example 64 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecLookupJoin method translateToPlanInternal.

@Override
@SuppressWarnings("unchecked")
public Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    RelOptTable temporalTable = temporalTableSourceSpec.getTemporalTable(planner.getFlinkContext());
    // validate whether the node is valid and supported.
    validate(temporalTable);
    final ExecEdge inputEdge = getInputEdges().get(0);
    RowType inputRowType = (RowType) inputEdge.getOutputType();
    RowType tableSourceRowType = FlinkTypeFactory.toLogicalRowType(temporalTable.getRowType());
    RowType resultRowType = (RowType) getOutputType();
    validateLookupKeyType(lookupKeys, inputRowType, tableSourceRowType);
    boolean isAsyncEnabled = false;
    UserDefinedFunction userDefinedFunction = LookupJoinUtil.getLookupFunction(temporalTable, lookupKeys.keySet());
    UserDefinedFunctionHelper.prepareInstance(config, userDefinedFunction);
    if (userDefinedFunction instanceof AsyncTableFunction) {
        isAsyncEnabled = true;
    }
    boolean isLeftOuterJoin = joinType == FlinkJoinType.LEFT;
    StreamOperatorFactory<RowData> operatorFactory;
    if (isAsyncEnabled) {
        operatorFactory = createAsyncLookupJoin(temporalTable, config, lookupKeys, (AsyncTableFunction<Object>) userDefinedFunction, planner.getRelBuilder(), inputRowType, tableSourceRowType, resultRowType, isLeftOuterJoin);
    } else {
        operatorFactory = createSyncLookupJoin(temporalTable, config, lookupKeys, (TableFunction<Object>) userDefinedFunction, planner.getRelBuilder(), inputRowType, tableSourceRowType, resultRowType, isLeftOuterJoin, planner.getExecEnv().getConfig().isObjectReuseEnabled());
    }
    Transformation<RowData> inputTransformation = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    return ExecNodeUtil.createOneInputTransformation(inputTransformation, createTransformationMeta(LOOKUP_JOIN_TRANSFORMATION, config), operatorFactory, InternalTypeInfo.of(resultRowType), inputTransformation.getParallelism());
}

Example 65 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class CommonExecTableSourceScan method translateToPlanInternal.

@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final StreamExecutionEnvironment env = planner.getExecEnv();
    final TransformationMetadata meta = createTransformationMeta(SOURCE_TRANSFORMATION, config);
    final InternalTypeInfo<RowData> outputTypeInfo = InternalTypeInfo.of((RowType) getOutputType());
    final ScanTableSource tableSource = tableSourceSpec.getScanTableSource(planner.getFlinkContext());
    ScanTableSource.ScanRuntimeProvider provider = tableSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
    if (provider instanceof SourceFunctionProvider) {
        final SourceFunctionProvider sourceFunctionProvider = (SourceFunctionProvider) provider;
        final SourceFunction<RowData> function = sourceFunctionProvider.createSourceFunction();
        final Transformation<RowData> transformation = createSourceFunctionTransformation(env, function, sourceFunctionProvider.isBounded(), meta.getName(), outputTypeInfo);
        return meta.fill(transformation);
    } else if (provider instanceof InputFormatProvider) {
        final InputFormat<RowData, ?> inputFormat = ((InputFormatProvider) provider).createInputFormat();
        final Transformation<RowData> transformation = createInputFormatTransformation(env, inputFormat, outputTypeInfo, meta.getName());
        return meta.fill(transformation);
    } else if (provider instanceof SourceProvider) {
        final Source<RowData, ?, ?> source = ((SourceProvider) provider).createSource();
        // TODO: Push down watermark strategy to source scan
        final Transformation<RowData> transformation = env.fromSource(source, WatermarkStrategy.noWatermarks(), meta.getName(), outputTypeInfo).getTransformation();
        return meta.fill(transformation);
    } else if (provider instanceof DataStreamScanProvider) {
        Transformation<RowData> transformation = ((DataStreamScanProvider) provider).produceDataStream(createProviderContext(), env).getTransformation();
        meta.fill(transformation);
        transformation.setOutputType(outputTypeInfo);
        return transformation;
    } else if (provider instanceof TransformationScanProvider) {
        final Transformation<RowData> transformation = ((TransformationScanProvider) provider).createTransformation(createProviderContext());
        meta.fill(transformation);
        transformation.setOutputType(outputTypeInfo);
        return transformation;
    } else {
        throw new UnsupportedOperationException(provider.getClass().getSimpleName() + " is unsupported now.");
    }
}