Examples with MultipleConnectedStreams org.apache.flink.streaming.api.datastream.MultipleConnectedStreams used on opensource projects

Example 1 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by apache.

the class StreamGraphGeneratorBatchExecutionTest method testInputSelectableMultiInputTransformation.

@Test
public void testInputSelectableMultiInputTransformation() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Integer> elements1 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements2 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements3 = env.fromElements(1, 2);
    MultipleInputOperatorFactory selectableOperator = new MultipleInputOperatorFactory(3, true);
    KeyedMultipleInputTransformation<Integer> multipleInputTransformation = new KeyedMultipleInputTransformation<>("operator", selectableOperator, BasicTypeInfo.INT_TYPE_INFO, 1, BasicTypeInfo.INT_TYPE_INFO);
    multipleInputTransformation.addInput(elements1.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements2.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements3.getTransformation(), e -> e);
    DataStreamSink<Integer> sink = new MultipleConnectedStreams(env).transform(multipleInputTransformation).addSink(new DiscardingSink<>());
    expectedException.expect(IllegalStateException.class);
    expectedException.expectMessage("Batch state backend and sorting inputs are not supported in graphs with an InputSelectable operator.");
    getStreamGraphInBatchMode(sink);
}

Example 2 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by apache.

the class StreamGraphGeneratorBatchExecutionTest method testMultiInputTransformation.

@Test
public void testMultiInputTransformation() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Integer> elements1 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements2 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements3 = env.fromElements(1, 2);
    MultipleInputOperatorFactory selectableOperator = new MultipleInputOperatorFactory(3, false);
    KeyedMultipleInputTransformation<Integer> multipleInputTransformation = new KeyedMultipleInputTransformation<>("operator", selectableOperator, BasicTypeInfo.INT_TYPE_INFO, 1, BasicTypeInfo.INT_TYPE_INFO);
    multipleInputTransformation.addInput(elements1.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements2.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements3.getTransformation(), e -> e);
    DataStreamSink<Integer> sink = new MultipleConnectedStreams(env).transform(multipleInputTransformation).addSink(new DiscardingSink<>());
    StreamGraph graph = getStreamGraphInBatchMode(sink);
    StreamNode operatorNode = graph.getStreamNode(multipleInputTransformation.getId());
    assertThat(operatorNode.getInputRequirements().get(0), equalTo(StreamConfig.InputRequirement.SORTED));
    assertThat(operatorNode.getInputRequirements().get(1), equalTo(StreamConfig.InputRequirement.SORTED));
    assertThat(operatorNode.getOperatorFactory().getChainingStrategy(), equalTo(ChainingStrategy.HEAD));
    assertThat(graph.getStateBackend(), instanceOf(BatchExecutionStateBackend.class));
    // the provider is passed as a lambda therefore we cannot assert the class of the provider
    assertThat(graph.getTimerServiceProvider(), notNullValue());
}

Example 3 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by apache.

the class DataStreamBatchExecutionITCase method batchMixedKeyedAndNonKeyedMultiInputOperator.

@Test
public void batchMixedKeyedAndNonKeyedMultiInputOperator() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    DataStream<Tuple2<String, Integer>> bc1Input = env.fromElements(Tuple2.of("bc3", 3), Tuple2.of("bc2", 2)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1)).broadcast();
    DataStream<Tuple2<String, Integer>> bc2Input = env.fromElements(Tuple2.of("bc1", 1)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1)).broadcast();
    DataStream<Tuple2<String, Integer>> regularInput = env.fromElements(Tuple2.of("regular1", 1), Tuple2.of("regular1", 2), Tuple2.of("regular1", 3), Tuple2.of("regular1", 4), Tuple2.of("regular2", 3), Tuple2.of("regular2", 5), Tuple2.of("regular1", 3)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1)).keyBy(input -> input.f0);
    KeyedMultipleInputTransformation<String> multipleInputTransformation = new KeyedMultipleInputTransformation<>("operator", mixedInputsOperatorFactory, BasicTypeInfo.STRING_TYPE_INFO, 1, BasicTypeInfo.STRING_TYPE_INFO);
    multipleInputTransformation.addInput(regularInput.getTransformation(), input -> ((Tuple2<String, Integer>) input).f0);
    multipleInputTransformation.addInput(bc1Input.getTransformation(), null);
    multipleInputTransformation.addInput(bc2Input.getTransformation(), null);
    DataStream<String> result = new MultipleConnectedStreams(env).transform(multipleInputTransformation);
    try (CloseableIterator<String> resultIterator = result.executeAndCollect()) {
        List<String> results = CollectionUtil.iteratorToList(resultIterator);
        assertThat(results, equalTo(Arrays.asList("(regular1,1): [bc3, bc2, bc1]", "(regular1,2): [bc3, bc2, bc1]", "(regular1,3): [bc3, bc2, bc1]", "(regular1,3): [bc3, bc2, bc1]", "(regular1,4): [bc3, bc2, bc1]", "(regular2,3): [bc3, bc2, bc1]", "(regular2,5): [bc3, bc2, bc1]")));
    }
}

Example 4 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by apache.

the class SourceNAryInputChainingITCase method nAryInputStreamOperation.

private static DataStream<Long> nAryInputStreamOperation(final DataStream<?>... inputs) {
    final StreamExecutionEnvironment env = inputs[0].getExecutionEnvironment();
    // this is still clumsy due to the raw API
    final MultipleInputTransformation<Long> transform = new MultipleInputTransformation<>("MultipleInputOperator", new NAryUnionOpFactory(inputs.length), Types.LONG, env.getParallelism());
    for (DataStream<?> input : inputs) {
        transform.addInput(input.getTransformation());
    }
    transform.setChainingStrategy(ChainingStrategy.HEAD_WITH_SOURCES);
    env.addOperator(transform);
    return new MultipleConnectedStreams(env).transform(transform);
}

Example 5 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by apache.

the class MultipleInputITCase method testNonKeyed.

public void testNonKeyed(boolean withUnion) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    TestListResultSink<Long> resultSink = new TestListResultSink<>();
    DataStream<Long> source1 = env.fromElements(1L, 10L);
    DataStream<Long> source2 = env.fromElements(2L, 11L);
    DataStream<String> source3 = env.fromElements("42", "44");
    MultipleInputTransformation<Long> multipleInput = new MultipleInputTransformation<>("My Operator", new SumAllInputOperatorFactory(withUnion ? 2 : 3), BasicTypeInfo.LONG_TYPE_INFO, 1);
    MultipleInputTransformation<Long> multipleInputTransformation;
    if (withUnion) {
        UnionTransformation<Long> union = new UnionTransformation<>(Arrays.asList(source1.getTransformation(), source2.getTransformation()));
        multipleInputTransformation = multipleInput.addInput(union);
    } else {
        multipleInputTransformation = multipleInput.addInput(source1.getTransformation()).addInput(source2.getTransformation());
    }
    env.addOperator(multipleInputTransformation.addInput(source3.getTransformation()));
    new MultipleConnectedStreams(env).transform(multipleInput).addSink(resultSink);
    env.execute();
    List<Long> result = resultSink.getResult();
    Collections.sort(result);
    long actualSum = result.get(result.size() - 1);
    assertEquals(1 + 10 + 2 + 11 + 42 + 44, actualSum);
}