Examples with Transformation org.apache.flink.api.dag.Transformation used on opensource projects

Example 56 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method testCompatibleExchangeModeWithBufferTimeout.

private void testCompatibleExchangeModeWithBufferTimeout(StreamExchangeMode exchangeMode) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setBufferTimeout(100);
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    PartitionTransformation<Integer> transformation = new PartitionTransformation<>(sourceDataStream.getTransformation(), new RebalancePartitioner<>(), exchangeMode);
    DataStream<Integer> partitionStream = new DataStream<>(env, transformation);
    partitionStream.map(value -> value).print();
    StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 57 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class StreamGraphGeneratorBatchExecutionTest method testNoSupportForIterationsInBatchHelper.

private void testNoSupportForIterationsInBatchHelper(final Transformation<?>... transformations) {
    final List<Transformation<?>> registeredTransformations = new ArrayList<>();
    Collections.addAll(registeredTransformations, transformations);
    final Configuration configuration = new Configuration();
    configuration.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH);
    StreamGraphGenerator streamGraphGenerator = new StreamGraphGenerator(registeredTransformations, new ExecutionConfig(), new CheckpointConfig(), configuration);
    expectedException.expect(UnsupportedOperationException.class);
    expectedException.expectMessage("Iterations are not supported in BATCH execution mode.");
    streamGraphGenerator.generate();
}

Example 58 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class LegacySinkTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final LegacySinkTransformation<IN> transformation, final Context context) {
    checkNotNull(transformation);
    checkNotNull(context);
    final StreamGraph streamGraph = context.getStreamGraph();
    final String slotSharingGroup = context.getSlotSharingGroup();
    final int transformationId = transformation.getId();
    final ExecutionConfig executionConfig = streamGraph.getExecutionConfig();
    final List<Transformation<?>> parentTransformations = transformation.getInputs();
    checkState(parentTransformations.size() == 1, "Expected exactly one input transformation but found " + parentTransformations.size());
    final Transformation<?> input = parentTransformations.get(0);
    streamGraph.addSink(transformationId, slotSharingGroup, transformation.getCoLocationGroupKey(), transformation.getOperatorFactory(), input.getOutputType(), null, "Sink: " + transformation.getName());
    final int parallelism = transformation.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ? transformation.getParallelism() : executionConfig.getParallelism();
    streamGraph.setParallelism(transformationId, parallelism);
    streamGraph.setMaxParallelism(transformationId, transformation.getMaxParallelism());
    for (Integer inputId : context.getStreamNodeIds(input)) {
        streamGraph.addEdge(inputId, transformationId, 0);
    }
    if (transformation.getStateKeySelector() != null) {
        TypeSerializer<?> keySerializer = transformation.getStateKeyType().createSerializer(executionConfig);
        streamGraph.setOneInputStateKey(transformationId, transformation.getStateKeySelector(), keySerializer);
    }
    return Collections.emptyList();
}

Example 59 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class MultiInputTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final AbstractMultipleInputTransformation<OUT> transformation, final Context context) {
    checkNotNull(transformation);
    checkNotNull(context);
    final List<Transformation<?>> inputTransformations = transformation.getInputs();
    checkArgument(!inputTransformations.isEmpty(), "Empty inputs for MultipleInputTransformation. Did you forget to add inputs?");
    MultipleInputSelectionHandler.checkSupportedInputCount(inputTransformations.size());
    final StreamGraph streamGraph = context.getStreamGraph();
    final String slotSharingGroup = context.getSlotSharingGroup();
    final int transformationId = transformation.getId();
    final ExecutionConfig executionConfig = streamGraph.getExecutionConfig();
    streamGraph.addMultipleInputOperator(transformationId, slotSharingGroup, transformation.getCoLocationGroupKey(), transformation.getOperatorFactory(), transformation.getInputTypes(), transformation.getOutputType(), transformation.getName());
    final int parallelism = transformation.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ? transformation.getParallelism() : executionConfig.getParallelism();
    streamGraph.setParallelism(transformationId, parallelism);
    streamGraph.setMaxParallelism(transformationId, transformation.getMaxParallelism());
    if (transformation instanceof KeyedMultipleInputTransformation) {
        KeyedMultipleInputTransformation<OUT> keyedTransform = (KeyedMultipleInputTransformation<OUT>) transformation;
        TypeSerializer<?> keySerializer = keyedTransform.getStateKeyType().createSerializer(executionConfig);
        streamGraph.setMultipleInputStateKey(transformationId, keyedTransform.getStateKeySelectors(), keySerializer);
    }
    for (int i = 0; i < inputTransformations.size(); i++) {
        final Transformation<?> inputTransformation = inputTransformations.get(i);
        final Collection<Integer> inputIds = context.getStreamNodeIds(inputTransformation);
        for (Integer inputId : inputIds) {
            streamGraph.addEdge(inputId, transformationId, i + 1);
        }
    }
    return Collections.singleton(transformationId);
}

Example 60 with Transformation

use of org.apache.flink.api.dag.Transformation in project flink by apache.

the class PartitionTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final PartitionTransformation<OUT> transformation, final Context context, boolean supportsBatchExchange) {
    checkNotNull(transformation);
    checkNotNull(context);
    final StreamGraph streamGraph = context.getStreamGraph();
    final List<Transformation<?>> parentTransformations = transformation.getInputs();
    checkState(parentTransformations.size() == 1, "Expected exactly one input transformation but found " + parentTransformations.size());
    final Transformation<?> input = parentTransformations.get(0);
    List<Integer> resultIds = new ArrayList<>();
    StreamExchangeMode exchangeMode = transformation.getExchangeMode();
    // UNDEFINED and let Flink decide on the best exchange mode.
    if (!supportsBatchExchange && exchangeMode == StreamExchangeMode.BATCH) {
        exchangeMode = StreamExchangeMode.UNDEFINED;
    }
    for (Integer inputId : context.getStreamNodeIds(input)) {
        final int virtualId = Transformation.getNewNodeId();
        streamGraph.addVirtualPartitionNode(inputId, virtualId, transformation.getPartitioner(), exchangeMode);
        resultIds.add(virtualId);
    }
    return resultIds;
}