Examples with CustomWatermarkExtractor org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor used on opensource projects

Example 1 with CustomWatermarkExtractor

use of org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor in project flink by apache.

the class KinesisExample method main.

public static void main(String[] args) throws Exception {
    // parse input arguments
    final ParameterTool parameterTool = ParameterTool.fromArgs(args);
    StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool);
    String inputStream = parameterTool.getRequired("input-stream");
    String outputStream = parameterTool.getRequired("output-stream");
    FlinkKinesisConsumer<KafkaEvent> consumer = new FlinkKinesisConsumer<>(inputStream, new KafkaEventSchema(), parameterTool.getProperties());
    consumer.setPeriodicWatermarkAssigner(new CustomWatermarkExtractor());
    Properties producerProperties = new Properties(parameterTool.getProperties());
    // producer needs region even when URL is specified
    producerProperties.putIfAbsent(ConsumerConfigConstants.AWS_REGION, "us-east-1");
    // test driver does not deaggregate
    producerProperties.putIfAbsent("AggregationEnabled", String.valueOf(false));
    // KPL does not recognize endpoint URL..
    String kinesisUrl = producerProperties.getProperty(ConsumerConfigConstants.AWS_ENDPOINT);
    if (kinesisUrl != null) {
        URL url = new URL(kinesisUrl);
        producerProperties.put("KinesisEndpoint", url.getHost());
        producerProperties.put("KinesisPort", Integer.toString(url.getPort()));
        producerProperties.put("VerifyCertificate", "false");
    }
    FlinkKinesisProducer<KafkaEvent> producer = new FlinkKinesisProducer<>(new KafkaEventSchema(), producerProperties);
    producer.setDefaultStream(outputStream);
    producer.setDefaultPartition("fakePartition");
    DataStream<KafkaEvent> input = env.addSource(consumer).keyBy("word").map(new RollingAdditionMapper());
    input.addSink(producer);
    env.execute();
}