Examples with SourceFunction org.apache.flink.streaming.api.functions.source.SourceFunction used on opensource projects

Example 6 with SourceFunction

use of org.apache.flink.streaming.api.functions.source.SourceFunction in project flink by apache.

the class FoldApplyProcessWindowFunctionTest method testFoldAllWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldAllWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyProcessAllWindowFunction<TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyProcessAllWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new ProcessAllWindowFunction<Integer, Integer, TimeWindow>() {

        @Override
        public void process(Context context, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Byte, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableProcessAllWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Byte>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Byte getKey(Integer value) throws Exception {
            return 0;
        }
    }, ByteSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.process(foldWindowFunction.new Context() {

        @Override
        public TimeWindow window() {
            return new TimeWindow(0, 1);
        }
    }, input, new ListCollector<>(result));
    Assert.assertEquals(expected, result);
}

Example 7 with SourceFunction

use of org.apache.flink.streaming.api.functions.source.SourceFunction in project flink by apache.

the class FoldApplyWindowFunctionTest method testFoldWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyWindowFunction<Integer, TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        private static final long serialVersionUID = -4849549768529720587L;

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new WindowFunction<Integer, Integer, Integer, TimeWindow>() {

        @Override
        public void apply(Integer integer, TimeWindow window, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, IntSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.apply(0, new TimeWindow(0, 1), input, new ListCollector<Integer>(result));
    Assert.assertEquals(expected, result);
}

Example 8 with SourceFunction

use of org.apache.flink.streaming.api.functions.source.SourceFunction in project ignite by apache.

the class FlinkIgniteSinkSelfTest method testFlinkIgniteSink.

/**
 * Tests for the Flink sink.
 * Ignite started in sink based on what is specified in the configuration file.
 *
 * @throws Exception
 */
@SuppressWarnings("unchecked")
public void testFlinkIgniteSink() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().disableSysoutLogging();
    IgniteSink igniteSink = new IgniteSink(TEST_CACHE, GRID_CONF_FILE);
    igniteSink.setAllowOverwrite(true);
    igniteSink.setAutoFlushFrequency(10);
    igniteSink.start();
    CacheListener listener = subscribeToPutEvents();
    DataStream<Map> stream = env.addSource(new SourceFunction<Map>() {

        private boolean running = true;

        @Override
        public void run(SourceContext<Map> ctx) throws Exception {
            Map testDataMap = new HashMap<>();
            long cnt = 0;
            while (running && (cnt < DFLT_STREAMING_EVENT)) {
                testDataMap.put(cnt, "ignite-" + cnt);
                cnt++;
            }
            ctx.collect(testDataMap);
        }

        @Override
        public void cancel() {
            running = false;
        }
    }).setParallelism(1);
    assertEquals(0, ignite.cache(TEST_CACHE).size());
    // sink data into the grid.
    stream.addSink(igniteSink);
    try {
        env.execute();
        CountDownLatch latch = listener.getLatch();
        // Enough events was handled in 10 seconds. Limited by test's timeout.
        latch.await();
        unsubscribeToPutEvents(listener);
        assertEquals(DFLT_STREAMING_EVENT, ignite.getOrCreateCache(TEST_CACHE).size());
        for (long i = 0; i < DFLT_STREAMING_EVENT; i++) assertEquals("ignite-" + i, ignite.getOrCreateCache(TEST_CACHE).get(i));
    } finally {
        igniteSink.stop();
    }
}

Example 9 with SourceFunction

use of org.apache.flink.streaming.api.functions.source.SourceFunction in project flink by apache.

the class KafkaProducerTestBase method testCustomPartitioning.

/**
 * This tests verifies that custom partitioning works correctly, with a default topic and
 * dynamic topic. The number of partitions for each topic is deliberately different.
 *
 * <p>Test topology:
 *
 * <pre>
 *             +------> (sink) --+--> [DEFAULT_TOPIC-1] --> (source) -> (map) -----+
 *            /                  |                             |          |        |
 *           |                   |                             |          |  ------+--> (sink)
 *             +------> (sink) --+--> [DEFAULT_TOPIC-2] --> (source) -> (map) -----+
 *            /                  |
 *           |                   |
 * (source) ----------> (sink) --+--> [DYNAMIC_TOPIC-1] --> (source) -> (map) -----+
 *           |                   |                             |          |        |
 *            \                  |                             |          |        |
 *             +------> (sink) --+--> [DYNAMIC_TOPIC-2] --> (source) -> (map) -----+--> (sink)
 *           |                   |                             |          |        |
 *            \                  |                             |          |        |
 *             +------> (sink) --+--> [DYNAMIC_TOPIC-3] --> (source) -> (map) -----+
 * </pre>
 *
 * <p>Each topic has an independent mapper that validates the values come consistently from the
 * correct Kafka partition of the topic is is responsible of.
 *
 * <p>Each topic also has a final sink that validates that there are no duplicates and that all
 * partitions are present.
 */
@Test
public void testCustomPartitioning() {
    try {
        LOG.info("Starting KafkaProducerITCase.testCustomPartitioning()");
        final String defaultTopic = "defaultTopic";
        final int defaultTopicPartitions = 2;
        final String dynamicTopic = "dynamicTopic";
        final int dynamicTopicPartitions = 3;
        createTestTopic(defaultTopic, defaultTopicPartitions, 1);
        createTestTopic(dynamicTopic, dynamicTopicPartitions, 1);
        Map<String, Integer> expectedTopicsToNumPartitions = new HashMap<>(2);
        expectedTopicsToNumPartitions.put(defaultTopic, defaultTopicPartitions);
        expectedTopicsToNumPartitions.put(dynamicTopic, dynamicTopicPartitions);
        TypeInformation<Tuple2<Long, String>> longStringInfo = TypeInformation.of(new TypeHint<Tuple2<Long, String>>() {
        });
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRestartStrategy(RestartStrategies.noRestart());
        TypeInformationSerializationSchema<Tuple2<Long, String>> serSchema = new TypeInformationSerializationSchema<>(longStringInfo, env.getConfig());
        TypeInformationSerializationSchema<Tuple2<Long, String>> deserSchema = new TypeInformationSerializationSchema<>(longStringInfo, env.getConfig());
        // ------ producing topology ---------
        // source has DOP 1 to make sure it generates no duplicates
        DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {

            private boolean running = true;

            @Override
            public void run(SourceContext<Tuple2<Long, String>> ctx) throws Exception {
                long cnt = 0;
                while (running) {
                    ctx.collect(new Tuple2<Long, String>(cnt, "kafka-" + cnt));
                    cnt++;
                    if (cnt % 100 == 0) {
                        Thread.sleep(1);
                    }
                }
            }

            @Override
            public void cancel() {
                running = false;
            }
        }).setParallelism(1);
        Properties props = new Properties();
        props.putAll(FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings));
        props.putAll(secureProps);
        // sink partitions into
        kafkaServer.produceIntoKafka(stream, defaultTopic, // dynamic topic
        new CustomKeyedSerializationSchemaWrapper(serSchema, defaultTopic, dynamicTopic), props, new CustomPartitioner(expectedTopicsToNumPartitions)).setParallelism(Math.max(defaultTopicPartitions, dynamicTopicPartitions));
        // ------ consuming topology ---------
        Properties consumerProps = new Properties();
        consumerProps.putAll(standardProps);
        consumerProps.putAll(secureProps);
        FlinkKafkaConsumerBase<Tuple2<Long, String>> defaultTopicSource = kafkaServer.getConsumer(defaultTopic, deserSchema, consumerProps);
        FlinkKafkaConsumerBase<Tuple2<Long, String>> dynamicTopicSource = kafkaServer.getConsumer(dynamicTopic, deserSchema, consumerProps);
        env.addSource(defaultTopicSource).setParallelism(defaultTopicPartitions).map(new PartitionValidatingMapper(defaultTopicPartitions)).setParallelism(defaultTopicPartitions).addSink(new PartitionValidatingSink(defaultTopicPartitions)).setParallelism(1);
        env.addSource(dynamicTopicSource).setParallelism(dynamicTopicPartitions).map(new PartitionValidatingMapper(dynamicTopicPartitions)).setParallelism(dynamicTopicPartitions).addSink(new PartitionValidatingSink(dynamicTopicPartitions)).setParallelism(1);
        tryExecute(env, "custom partitioning test");
        deleteTestTopic(defaultTopic);
        deleteTestTopic(dynamicTopic);
        LOG.info("Finished KafkaProducerITCase.testCustomPartitioning()");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with SourceFunction

use of org.apache.flink.streaming.api.functions.source.SourceFunction in project flink by apache.

the class BufferTimeoutITCase method testDisablingBufferTimeout.

/**
 * The test verifies that it is possible to disable explicit buffer flushing. It checks that
 * OutputFlasher thread would not be started when the task is running. But this doesn't
 * guarantee that the unfinished buffers can not be flushed by another events.
 */
@Test
public void testDisablingBufferTimeout() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.setBufferTimeout(-1);
    final SharedReference<ArrayList<Integer>> results = sharedObjects.add(new ArrayList<>());
    env.addSource(new SourceFunction<Integer>() {

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
            ctx.collect(1);
            // just sleep forever
            Thread.sleep(Long.MAX_VALUE);
        }

        @Override
        public void cancel() {
        }
    }).slotSharingGroup("source").addSink(new SinkFunction<Integer>() {

        @Override
        public void invoke(Integer value, Context context) {
            results.get().add(value);
        }
    }).slotSharingGroup("sink");
    final JobClient jobClient = env.executeAsync();
    CommonTestUtils.waitForAllTaskRunning(MINI_CLUSTER_RESOURCE.getMiniCluster(), jobClient.getJobID(), false);
    assertTrue(RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME + " thread is unexpectedly running", Thread.getAllStackTraces().keySet().stream().noneMatch(thread -> thread.getName().startsWith(RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME)));
}