Examples with MultipleInputTransformation org.apache.flink.streaming.api.transformations.MultipleInputTransformation used on opensource projects

Example 1 with MultipleInputTransformation

use of org.apache.flink.streaming.api.transformations.MultipleInputTransformation in project flink by apache.

the class BatchExecMultipleInput method translateToPlanInternal.

@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final List<Transformation<?>> inputTransforms = new ArrayList<>();
    for (ExecEdge inputEdge : getInputEdges()) {
        inputTransforms.add(inputEdge.translateToPlan(planner));
    }
    final Transformation<?> outputTransform = rootNode.translateToPlan(planner);
    final int[] readOrders = getInputProperties().stream().map(InputProperty::getPriority).mapToInt(i -> i).toArray();
    final TableOperatorWrapperGenerator generator = new TableOperatorWrapperGenerator(inputTransforms, outputTransform, readOrders);
    generator.generate();
    final List<Pair<Transformation<?>, InputSpec>> inputTransformAndInputSpecPairs = generator.getInputTransformAndInputSpecPairs();
    final MultipleInputTransformation<RowData> multipleInputTransform = new MultipleInputTransformation<>(createTransformationName(config), new BatchMultipleInputStreamOperatorFactory(inputTransformAndInputSpecPairs.stream().map(Pair::getValue).collect(Collectors.toList()), generator.getHeadWrappers(), generator.getTailWrapper()), InternalTypeInfo.of(getOutputType()), generator.getParallelism());
    multipleInputTransform.setDescription(createTransformationDescription(config));
    inputTransformAndInputSpecPairs.forEach(input -> multipleInputTransform.addInput(input.getKey()));
    if (generator.getMaxParallelism() > 0) {
        multipleInputTransform.setMaxParallelism(generator.getMaxParallelism());
    }
    // set resources
    multipleInputTransform.setResources(generator.getMinResources(), generator.getPreferredResources());
    final int memoryWeight = generator.getManagedMemoryWeight();
    final long memoryBytes = (long) memoryWeight << 20;
    ExecNodeUtil.setManagedMemoryWeight(multipleInputTransform, memoryBytes);
    // set chaining strategy for source chaining
    multipleInputTransform.setChainingStrategy(ChainingStrategy.HEAD_WITH_SOURCES);
    return multipleInputTransform;
}

Example 2 with MultipleInputTransformation

use of org.apache.flink.streaming.api.transformations.MultipleInputTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method createJobGraphWithDescription.

private JobGraph createJobGraphWithDescription(StreamExecutionEnvironment env, String... inputNames) {
    env.setParallelism(1);
    DataStream<Long> source;
    if (inputNames.length == 1) {
        source = env.fromElements(1L, 2L, 3L).setDescription(inputNames[0]);
    } else {
        MultipleInputTransformation<Long> transform = new MultipleInputTransformation<>("mit", new UnusedOperatorFactory(), Types.LONG, env.getParallelism());
        transform.setDescription("operator chained with source");
        transform.setChainingStrategy(ChainingStrategy.HEAD_WITH_SOURCES);
        Arrays.stream(inputNames).map(name -> env.fromSource(new NumberSequenceSource(1, 2), WatermarkStrategy.noWatermarks(), name).setDescription(name).getTransformation()).forEach(transform::addInput);
        source = new DataStream<>(env, transform);
    }
    DataStream<Long> map1 = source.map(x -> x + 1).setDescription("x + 1");
    DataStream<Long> map2 = source.map(x -> x + 2).setDescription("x + 2");
    map1.print().setDescription("first print of map1");
    map1.print().setDescription("second print of map1");
    map2.print().setDescription("first print of map2");
    map2.print().setDescription("second print of map2");
    return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 3 with MultipleInputTransformation

use of org.apache.flink.streaming.api.transformations.MultipleInputTransformation in project flink by apache.

the class StreamingJobGraphGeneratorTest method createGraphWithMultipleInputs.

public JobGraph createGraphWithMultipleInputs(boolean chain, String... inputNames) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    MultipleInputTransformation<Long> transform = new MultipleInputTransformation<>("mit", new UnusedOperatorFactory(), Types.LONG, env.getParallelism());
    Arrays.stream(inputNames).map(name -> env.fromSource(new NumberSequenceSource(1, 2), WatermarkStrategy.noWatermarks(), name).getTransformation()).forEach(transform::addInput);
    transform.setChainingStrategy(chain ? ChainingStrategy.HEAD_WITH_SOURCES : ChainingStrategy.NEVER);
    env.addOperator(transform);
    return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 4 with MultipleInputTransformation

use of org.apache.flink.streaming.api.transformations.MultipleInputTransformation in project flink by apache.

the class MultiInputTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final AbstractMultipleInputTransformation<OUT> transformation, final Context context) {
    checkNotNull(transformation);
    checkNotNull(context);
    final List<Transformation<?>> inputTransformations = transformation.getInputs();
    checkArgument(!inputTransformations.isEmpty(), "Empty inputs for MultipleInputTransformation. Did you forget to add inputs?");
    MultipleInputSelectionHandler.checkSupportedInputCount(inputTransformations.size());
    final StreamGraph streamGraph = context.getStreamGraph();
    final String slotSharingGroup = context.getSlotSharingGroup();
    final int transformationId = transformation.getId();
    final ExecutionConfig executionConfig = streamGraph.getExecutionConfig();
    streamGraph.addMultipleInputOperator(transformationId, slotSharingGroup, transformation.getCoLocationGroupKey(), transformation.getOperatorFactory(), transformation.getInputTypes(), transformation.getOutputType(), transformation.getName());
    final int parallelism = transformation.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ? transformation.getParallelism() : executionConfig.getParallelism();
    streamGraph.setParallelism(transformationId, parallelism);
    streamGraph.setMaxParallelism(transformationId, transformation.getMaxParallelism());
    if (transformation instanceof KeyedMultipleInputTransformation) {
        KeyedMultipleInputTransformation<OUT> keyedTransform = (KeyedMultipleInputTransformation<OUT>) transformation;
        TypeSerializer<?> keySerializer = keyedTransform.getStateKeyType().createSerializer(executionConfig);
        streamGraph.setMultipleInputStateKey(transformationId, keyedTransform.getStateKeySelectors(), keySerializer);
    }
    for (int i = 0; i < inputTransformations.size(); i++) {
        final Transformation<?> inputTransformation = inputTransformations.get(i);
        final Collection<Integer> inputIds = context.getStreamNodeIds(inputTransformation);
        for (Integer inputId : inputIds) {
            streamGraph.addEdge(inputId, transformationId, i + 1);
        }
    }
    return Collections.singleton(transformationId);
}

Example 5 with MultipleInputTransformation

use of org.apache.flink.streaming.api.transformations.MultipleInputTransformation in project flink by apache.

the class StreamGraphGeneratorTest method testMultipleInputTransformation.

@Test
public void testMultipleInputTransformation() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source1 = env.fromElements(1, 10);
    DataStream<Long> source2 = env.fromElements(2L, 11L);
    DataStream<String> source3 = env.fromElements("42", "44");
    MultipleInputTransformation<String> transform = new MultipleInputTransformation<String>("My Operator", new MultipleInputOperatorFactory(), BasicTypeInfo.STRING_TYPE_INFO, 3);
    env.addOperator(transform.addInput(source1.getTransformation()).addInput(source2.getTransformation()).addInput(source3.getTransformation()));
    StreamGraph streamGraph = env.getStreamGraph();
    assertEquals(4, streamGraph.getStreamNodes().size());
    assertEquals(1, streamGraph.getStreamEdges(source1.getId(), transform.getId()).size());
    assertEquals(1, streamGraph.getStreamEdges(source2.getId(), transform.getId()).size());
    assertEquals(1, streamGraph.getStreamEdges(source3.getId(), transform.getId()).size());
    assertEquals(1, streamGraph.getStreamEdges(source1.getId()).size());
    assertEquals(1, streamGraph.getStreamEdges(source2.getId()).size());
    assertEquals(1, streamGraph.getStreamEdges(source3.getId()).size());
    assertEquals(0, streamGraph.getStreamEdges(transform.getId()).size());
}