Examples with StreamExchangeMode org.apache.flink.streaming.api.transformations.StreamExchangeMode used on opensource projects

Example 1 with StreamExchangeMode

use of org.apache.flink.streaming.api.transformations.StreamExchangeMode in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeUndefined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#UNDEFINED}.
 */
@Test
public void testExchangeModeUndefined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.UNDEFINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.UNDEFINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with UNDEFINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // UNDEFINED exchange mode is translated into PIPELINED_BOUNDED result partition by default
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 2 with StreamExchangeMode

use of org.apache.flink.streaming.api.transformations.StreamExchangeMode in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModePipelined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#PIPELINED}.
 */
@Test
public void testExchangeModePipelined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.PIPELINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.PIPELINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with PIPELINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // PIPELINED exchange mode is translated into PIPELINED_BOUNDED result partition
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 3 with StreamExchangeMode

use of org.apache.flink.streaming.api.transformations.StreamExchangeMode in project flink by apache.

the class StreamingJobGraphGeneratorTest method testCompatibleExchangeModeWithBufferTimeout.

private void testCompatibleExchangeModeWithBufferTimeout(StreamExchangeMode exchangeMode) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setBufferTimeout(100);
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    PartitionTransformation<Integer> transformation = new PartitionTransformation<>(sourceDataStream.getTransformation(), new RebalancePartitioner<>(), exchangeMode);
    DataStream<Integer> partitionStream = new DataStream<>(env, transformation);
    partitionStream.map(value -> value).print();
    StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 4 with StreamExchangeMode

use of org.apache.flink.streaming.api.transformations.StreamExchangeMode in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeBatch.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#BATCH}.
 */
@Test
public void testExchangeModeBatch() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    env.setBufferTimeout(-1);
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.BATCH));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.BATCH));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(3, verticesSorted.size());
    // it can not be chained with BATCH exchange mode
    JobVertex sourceVertex = verticesSorted.get(0);
    JobVertex mapVertex = verticesSorted.get(1);
    // BATCH exchange mode is translated into BLOCKING result partition
    assertEquals(ResultPartitionType.BLOCKING, sourceVertex.getProducedDataSets().get(0).getResultType());
    assertEquals(ResultPartitionType.BLOCKING, mapVertex.getProducedDataSets().get(0).getResultType());
}

Example 5 with StreamExchangeMode

use of org.apache.flink.streaming.api.transformations.StreamExchangeMode in project flink by apache.

the class PartitionTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final PartitionTransformation<OUT> transformation, final Context context, boolean supportsBatchExchange) {
    checkNotNull(transformation);
    checkNotNull(context);
    final StreamGraph streamGraph = context.getStreamGraph();
    final List<Transformation<?>> parentTransformations = transformation.getInputs();
    checkState(parentTransformations.size() == 1, "Expected exactly one input transformation but found " + parentTransformations.size());
    final Transformation<?> input = parentTransformations.get(0);
    List<Integer> resultIds = new ArrayList<>();
    StreamExchangeMode exchangeMode = transformation.getExchangeMode();
    // UNDEFINED and let Flink decide on the best exchange mode.
    if (!supportsBatchExchange && exchangeMode == StreamExchangeMode.BATCH) {
        exchangeMode = StreamExchangeMode.UNDEFINED;
    }
    for (Integer inputId : context.getStreamNodeIds(input)) {
        final int virtualId = Transformation.getNewNodeId();
        streamGraph.addVirtualPartitionNode(inputId, virtualId, transformation.getPartitioner(), exchangeMode);
        resultIds.add(virtualId);
    }
    return resultIds;
}