Examples with PartitionTransformation org.apache.flink.streaming.api.transformations.PartitionTransformation used on opensource projects

Example 6 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class PythonOperatorChainingOptimizer method replaceInput.

private static void replaceInput(Transformation<?> transformation, Transformation<?> oldInput, Transformation<?> newInput) {
    try {
        if (transformation instanceof OneInputTransformation || transformation instanceof FeedbackTransformation || transformation instanceof SideOutputTransformation || transformation instanceof ReduceTransformation || transformation instanceof SinkTransformation || transformation instanceof LegacySinkTransformation || transformation instanceof TimestampsAndWatermarksTransformation || transformation instanceof PartitionTransformation) {
            final Field inputField = transformation.getClass().getDeclaredField("input");
            inputField.setAccessible(true);
            inputField.set(transformation, newInput);
        } else if (transformation instanceof TwoInputTransformation) {
            final Field inputField;
            if (((TwoInputTransformation<?, ?, ?>) transformation).getInput1() == oldInput) {
                inputField = transformation.getClass().getDeclaredField("input1");
            } else {
                inputField = transformation.getClass().getDeclaredField("input2");
            }
            inputField.setAccessible(true);
            inputField.set(transformation, newInput);
        } else if (transformation instanceof UnionTransformation || transformation instanceof AbstractMultipleInputTransformation) {
            final Field inputsField = transformation.getClass().getDeclaredField("inputs");
            inputsField.setAccessible(true);
            List<Transformation<?>> newInputs = Lists.newArrayList();
            newInputs.addAll(transformation.getInputs());
            newInputs.remove(oldInput);
            newInputs.add(newInput);
            inputsField.set(transformation, newInputs);
        } else if (transformation instanceof AbstractBroadcastStateTransformation) {
            final Field inputField;
            if (((AbstractBroadcastStateTransformation<?, ?, ?>) transformation).getRegularInput() == oldInput) {
                inputField = transformation.getClass().getDeclaredField("regularInput");
            } else {
                inputField = transformation.getClass().getDeclaredField("broadcastInput");
            }
            inputField.setAccessible(true);
            inputField.set(transformation, newInput);
        } else {
            throw new RuntimeException("Unsupported transformation: " + transformation);
        }
    } catch (NoSuchFieldException | IllegalAccessException e) {
        // This should never happen
        throw new RuntimeException(e);
    }
}

Example 7 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method createStreamGraphForSlotSharingTest.

/**
 * Create a StreamGraph as below.
 *
 * <p>source1 --(rebalance & pipelined)--> Map1
 *
 * <p>source2 --(rebalance & blocking)--> Map2
 */
private StreamGraph createStreamGraphForSlotSharingTest(Configuration config) {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
    env.setBufferTimeout(-1);
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    final DataStream<Integer> source1 = env.fromElements(1, 2, 3).name("source1");
    source1.rebalance().map(v -> v).name("map1");
    final DataStream<Integer> source2 = env.fromElements(4, 5, 6).name("source2");
    final DataStream<Integer> partitioned = new DataStream<>(env, new PartitionTransformation<>(source2.getTransformation(), new RebalancePartitioner<>(), StreamExchangeMode.BATCH));
    partitioned.map(v -> v).name("map2");
    return env.getStreamGraph();
}

Example 8 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method testCompatibleExchangeModeWithBufferTimeout.

private void testCompatibleExchangeModeWithBufferTimeout(StreamExchangeMode exchangeMode) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setBufferTimeout(100);
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    PartitionTransformation<Integer> transformation = new PartitionTransformation<>(sourceDataStream.getTransformation(), new RebalancePartitioner<>(), exchangeMode);
    DataStream<Integer> partitionStream = new DataStream<>(env, transformation);
    partitionStream.map(value -> value).print();
    StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 9 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeBatch.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#BATCH}.
 */
@Test
public void testExchangeModeBatch() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    env.setBufferTimeout(-1);
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.BATCH));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.BATCH));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(3, verticesSorted.size());
    // it can not be chained with BATCH exchange mode
    JobVertex sourceVertex = verticesSorted.get(0);
    JobVertex mapVertex = verticesSorted.get(1);
    // BATCH exchange mode is translated into BLOCKING result partition
    assertEquals(ResultPartitionType.BLOCKING, sourceVertex.getProducedDataSets().get(0).getResultType());
    assertEquals(ResultPartitionType.BLOCKING, mapVertex.getProducedDataSets().get(0).getResultType());
}

Example 10 with PartitionTransformation

use of org.apache.flink.streaming.api.transformations.PartitionTransformation in project flink by apache.

the class PartitionTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final PartitionTransformation<OUT> transformation, final Context context, boolean supportsBatchExchange) {
    checkNotNull(transformation);
    checkNotNull(context);
    final StreamGraph streamGraph = context.getStreamGraph();
    final List<Transformation<?>> parentTransformations = transformation.getInputs();
    checkState(parentTransformations.size() == 1, "Expected exactly one input transformation but found " + parentTransformations.size());
    final Transformation<?> input = parentTransformations.get(0);
    List<Integer> resultIds = new ArrayList<>();
    StreamExchangeMode exchangeMode = transformation.getExchangeMode();
    // UNDEFINED and let Flink decide on the best exchange mode.
    if (!supportsBatchExchange && exchangeMode == StreamExchangeMode.BATCH) {
        exchangeMode = StreamExchangeMode.UNDEFINED;
    }
    for (Integer inputId : context.getStreamNodeIds(input)) {
        final int virtualId = Transformation.getNewNodeId();
        streamGraph.addVirtualPartitionNode(inputId, virtualId, transformation.getPartitioner(), exchangeMode);
        resultIds.add(virtualId);
    }
    return resultIds;
}