Examples with SourceTransformation org.apache.flink.streaming.api.transformations.SourceTransformation used on opensource projects

Example 1 with SourceTransformation

use of org.apache.flink.streaming.api.transformations.SourceTransformation in project flink by apache.

the class FoldApplyProcessWindowFunctionTest method testFoldAllWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldAllWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyProcessAllWindowFunction<TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyProcessAllWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new ProcessAllWindowFunction<Integer, Integer, TimeWindow>() {

        @Override
        public void process(Context context, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Byte, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableProcessAllWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Byte>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Byte getKey(Integer value) throws Exception {
            return 0;
        }
    }, ByteSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.process(foldWindowFunction.new Context() {

        @Override
        public TimeWindow window() {
            return new TimeWindow(0, 1);
        }
    }, input, new ListCollector<>(result));
    Assert.assertEquals(expected, result);
}

Example 2 with SourceTransformation

use of org.apache.flink.streaming.api.transformations.SourceTransformation in project flink by apache.

the class FoldApplyWindowFunctionTest method testFoldWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyWindowFunction<Integer, TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        private static final long serialVersionUID = -4849549768529720587L;

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new WindowFunction<Integer, Integer, Integer, TimeWindow>() {

        @Override
        public void apply(Integer integer, TimeWindow window, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, IntSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.apply(0, new TimeWindow(0, 1), input, new ListCollector<Integer>(result));
    Assert.assertEquals(expected, result);
}

Example 3 with SourceTransformation

use of org.apache.flink.streaming.api.transformations.SourceTransformation in project flink by apache.

the class PythonOperatorChainingOptimizerTest method testChainingMultipleOperators.

@Test
public void testChainingMultipleOperators() {
    PythonKeyedProcessOperator<?> keyedProcessOperator = createKeyedProcessOperator("f1", new RowTypeInfo(Types.INT(), Types.INT()), Types.STRING());
    PythonProcessOperator<?, ?> processOperator1 = createProcessOperator("f2", Types.STRING(), Types.LONG());
    PythonProcessOperator<?, ?> processOperator2 = createProcessOperator("f3", Types.LONG(), Types.INT());
    Transformation<?> sourceTransformation = mock(SourceTransformation.class);
    OneInputTransformation<?, ?> keyedProcessTransformation = new OneInputTransformation(sourceTransformation, "keyedProcess", keyedProcessOperator, keyedProcessOperator.getProducedType(), 2);
    Transformation<?> processTransformation1 = new OneInputTransformation(keyedProcessTransformation, "process", processOperator1, processOperator1.getProducedType(), 2);
    Transformation<?> processTransformation2 = new OneInputTransformation(processTransformation1, "process", processOperator2, processOperator2.getProducedType(), 2);
    List<Transformation<?>> transformations = new ArrayList<>();
    transformations.add(sourceTransformation);
    transformations.add(keyedProcessTransformation);
    transformations.add(processTransformation1);
    transformations.add(processTransformation2);
    List<Transformation<?>> optimized = PythonOperatorChainingOptimizer.optimize(transformations);
    assertEquals(2, optimized.size());
    OneInputTransformation<?, ?> chainedTransformation = (OneInputTransformation<?, ?>) optimized.get(1);
    assertEquals(sourceTransformation.getOutputType(), chainedTransformation.getInputType());
    assertEquals(processOperator2.getProducedType(), chainedTransformation.getOutputType());
    OneInputStreamOperator<?, ?> chainedOperator = chainedTransformation.getOperator();
    assertTrue(chainedOperator instanceof PythonKeyedProcessOperator);
    validateChainedPythonFunctions(((PythonKeyedProcessOperator<?>) chainedOperator).getPythonFunctionInfo(), "f3", "f2", "f1");
}

Example 4 with SourceTransformation

use of org.apache.flink.streaming.api.transformations.SourceTransformation in project flink by apache.

the class PythonOperatorChainingOptimizerTest method testContinuousKeyedOperators.

@Test
public void testContinuousKeyedOperators() {
    PythonKeyedProcessOperator<?> keyedProcessOperator1 = createKeyedProcessOperator("f1", new RowTypeInfo(Types.INT(), Types.INT()), new RowTypeInfo(Types.INT(), Types.INT()));
    PythonKeyedProcessOperator<?> keyedProcessOperator2 = createKeyedProcessOperator("f2", new RowTypeInfo(Types.INT(), Types.INT()), Types.STRING());
    Transformation<?> sourceTransformation = mock(SourceTransformation.class);
    OneInputTransformation<?, ?> processTransformation1 = new OneInputTransformation(sourceTransformation, "KeyedProcess1", keyedProcessOperator1, keyedProcessOperator1.getProducedType(), 2);
    OneInputTransformation<?, ?> processTransformation2 = new OneInputTransformation(processTransformation1, "KeyedProcess2", keyedProcessOperator2, keyedProcessOperator2.getProducedType(), 2);
    List<Transformation<?>> transformations = new ArrayList<>();
    transformations.add(sourceTransformation);
    transformations.add(processTransformation1);
    transformations.add(processTransformation2);
    List<Transformation<?>> optimized = PythonOperatorChainingOptimizer.optimize(transformations);
    assertEquals(3, optimized.size());
    assertEquals(processTransformation1, optimized.get(1));
    assertEquals(processTransformation2, optimized.get(2));
}

Example 5 with SourceTransformation

use of org.apache.flink.streaming.api.transformations.SourceTransformation in project flink by apache.

the class PythonOperatorChainingOptimizerTest method testSingleTransformation.

@Test
public void testSingleTransformation() {
    PythonKeyedProcessOperator<?> keyedProcessOperator = createKeyedProcessOperator("f1", new RowTypeInfo(Types.INT(), Types.INT()), Types.STRING());
    PythonProcessOperator<?, ?> processOperator1 = createProcessOperator("f2", Types.STRING(), Types.LONG());
    PythonProcessOperator<?, ?> processOperator2 = createProcessOperator("f3", Types.LONG(), Types.INT());
    Transformation<?> sourceTransformation = mock(SourceTransformation.class);
    OneInputTransformation<?, ?> keyedProcessTransformation = new OneInputTransformation(sourceTransformation, "keyedProcess", keyedProcessOperator, keyedProcessOperator.getProducedType(), 2);
    Transformation<?> processTransformation1 = new OneInputTransformation(keyedProcessTransformation, "process", processOperator1, processOperator1.getProducedType(), 2);
    Transformation<?> processTransformation2 = new OneInputTransformation(processTransformation1, "process", processOperator2, processOperator2.getProducedType(), 2);
    List<Transformation<?>> transformations = new ArrayList<>();
    transformations.add(processTransformation2);
    List<Transformation<?>> optimized = PythonOperatorChainingOptimizer.optimize(transformations);
    assertEquals(2, optimized.size());
    OneInputTransformation<?, ?> chainedTransformation = (OneInputTransformation<?, ?>) optimized.get(0);
    assertEquals(sourceTransformation.getOutputType(), chainedTransformation.getInputType());
    assertEquals(processOperator2.getProducedType(), chainedTransformation.getOutputType());
    OneInputStreamOperator<?, ?> chainedOperator = chainedTransformation.getOperator();
    assertTrue(chainedOperator instanceof PythonKeyedProcessOperator);
    validateChainedPythonFunctions(((PythonKeyedProcessOperator<?>) chainedOperator).getPythonFunctionInfo(), "f3", "f2", "f1");
}