Examples with MultipleConnectedStreams org.apache.flink.streaming.api.datastream.MultipleConnectedStreams used on opensource projects

Example 16 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by splunk.

the class MultipleInputITCase method testNonKeyed.

public void testNonKeyed(boolean withUnion) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    TestListResultSink<Long> resultSink = new TestListResultSink<>();
    DataStream<Long> source1 = env.fromElements(1L, 10L);
    DataStream<Long> source2 = env.fromElements(2L, 11L);
    DataStream<String> source3 = env.fromElements("42", "44");
    MultipleInputTransformation<Long> multipleInput = new MultipleInputTransformation<>("My Operator", new SumAllInputOperatorFactory(withUnion ? 2 : 3), BasicTypeInfo.LONG_TYPE_INFO, 1);
    MultipleInputTransformation<Long> multipleInputTransformation;
    if (withUnion) {
        UnionTransformation<Long> union = new UnionTransformation<>(Arrays.asList(source1.getTransformation(), source2.getTransformation()));
        multipleInputTransformation = multipleInput.addInput(union);
    } else {
        multipleInputTransformation = multipleInput.addInput(source1.getTransformation()).addInput(source2.getTransformation());
    }
    env.addOperator(multipleInputTransformation.addInput(source3.getTransformation()));
    new MultipleConnectedStreams(env).transform(multipleInput).addSink(resultSink);
    env.execute();
    List<Long> result = resultSink.getResult();
    Collections.sort(result);
    long actualSum = result.get(result.size() - 1);
    assertEquals(1 + 10 + 2 + 11 + 42 + 44, actualSum);
}

Example 17 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink by splunk.

the class StreamGraphGeneratorBatchExecutionTest method testMultiInputTransformation.

@Test
public void testMultiInputTransformation() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Integer> elements1 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements2 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements3 = env.fromElements(1, 2);
    MultipleInputOperatorFactory selectableOperator = new MultipleInputOperatorFactory(3, false);
    KeyedMultipleInputTransformation<Integer> multipleInputTransformation = new KeyedMultipleInputTransformation<>("operator", selectableOperator, BasicTypeInfo.INT_TYPE_INFO, 1, BasicTypeInfo.INT_TYPE_INFO);
    multipleInputTransformation.addInput(elements1.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements2.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements3.getTransformation(), e -> e);
    DataStreamSink<Integer> sink = new MultipleConnectedStreams(env).transform(multipleInputTransformation).addSink(new DiscardingSink<>());
    StreamGraph graph = getStreamGraphInBatchMode(sink);
    StreamNode operatorNode = graph.getStreamNode(multipleInputTransformation.getId());
    assertThat(operatorNode.getInputRequirements().get(0), equalTo(StreamConfig.InputRequirement.SORTED));
    assertThat(operatorNode.getInputRequirements().get(1), equalTo(StreamConfig.InputRequirement.SORTED));
    assertThat(operatorNode.getOperatorFactory().getChainingStrategy(), equalTo(ChainingStrategy.HEAD));
    assertThat(graph.getStateBackend(), instanceOf(BatchExecutionStateBackend.class));
    // the provider is passed as a lambda therefore we cannot assert the class of the provider
    assertThat(graph.getTimerServiceProvider(), notNullValue());
}

Example 18 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink-mirror by flink-ci.

the class MultipleInputITCase method testNonKeyed.

public void testNonKeyed(boolean withUnion) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    TestListResultSink<Long> resultSink = new TestListResultSink<>();
    DataStream<Long> source1 = env.fromElements(1L, 10L);
    DataStream<Long> source2 = env.fromElements(2L, 11L);
    DataStream<String> source3 = env.fromElements("42", "44");
    MultipleInputTransformation<Long> multipleInput = new MultipleInputTransformation<>("My Operator", new SumAllInputOperatorFactory(withUnion ? 2 : 3), BasicTypeInfo.LONG_TYPE_INFO, 1);
    MultipleInputTransformation<Long> multipleInputTransformation;
    if (withUnion) {
        UnionTransformation<Long> union = new UnionTransformation<>(Arrays.asList(source1.getTransformation(), source2.getTransformation()));
        multipleInputTransformation = multipleInput.addInput(union);
    } else {
        multipleInputTransformation = multipleInput.addInput(source1.getTransformation()).addInput(source2.getTransformation());
    }
    env.addOperator(multipleInputTransformation.addInput(source3.getTransformation()));
    new MultipleConnectedStreams(env).transform(multipleInput).addSink(resultSink);
    env.execute();
    List<Long> result = resultSink.getResult();
    Collections.sort(result);
    long actualSum = result.get(result.size() - 1);
    assertEquals(1 + 10 + 2 + 11 + 42 + 44, actualSum);
}

Example 19 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink-mirror by flink-ci.

the class StreamGraphGeneratorBatchExecutionTest method testMultiInputTransformation.

@Test
public void testMultiInputTransformation() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Integer> elements1 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements2 = env.fromElements(1, 2);
    DataStreamSource<Integer> elements3 = env.fromElements(1, 2);
    MultipleInputOperatorFactory selectableOperator = new MultipleInputOperatorFactory(3, false);
    KeyedMultipleInputTransformation<Integer> multipleInputTransformation = new KeyedMultipleInputTransformation<>("operator", selectableOperator, BasicTypeInfo.INT_TYPE_INFO, 1, BasicTypeInfo.INT_TYPE_INFO);
    multipleInputTransformation.addInput(elements1.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements2.getTransformation(), e -> e);
    multipleInputTransformation.addInput(elements3.getTransformation(), e -> e);
    DataStreamSink<Integer> sink = new MultipleConnectedStreams(env).transform(multipleInputTransformation).addSink(new DiscardingSink<>());
    StreamGraph graph = getStreamGraphInBatchMode(sink);
    StreamNode operatorNode = graph.getStreamNode(multipleInputTransformation.getId());
    assertThat(operatorNode.getInputRequirements().get(0), equalTo(StreamConfig.InputRequirement.SORTED));
    assertThat(operatorNode.getInputRequirements().get(1), equalTo(StreamConfig.InputRequirement.SORTED));
    assertThat(operatorNode.getOperatorFactory().getChainingStrategy(), equalTo(ChainingStrategy.HEAD));
    assertThat(graph.getStateBackend(), instanceOf(BatchExecutionStateBackend.class));
    // the provider is passed as a lambda therefore we cannot assert the class of the provider
    assertThat(graph.getTimerServiceProvider(), notNullValue());
}

Example 20 with MultipleConnectedStreams

use of org.apache.flink.streaming.api.datastream.MultipleConnectedStreams in project flink-benchmarks by apache.

the class MultipleInputBenchmark method connectAndDiscard.

private static void connectAndDiscard(StreamExecutionEnvironment env, DataStream<?> source1, DataStream<?> source2) {
    MultipleInputTransformation<Long> transform = new MultipleInputTransformation<>("custom operator", new MultiplyByTwoOperatorFactory(), BasicTypeInfo.LONG_TYPE_INFO, 1);
    transform.addInput(source1.getTransformation());
    transform.addInput(source2.getTransformation());
    env.addOperator(transform);
    new MultipleConnectedStreams(env).transform(transform).addSink(new DiscardingSink<>());
}