Examples with FlinkKinesisProducer org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer used on opensource projects

Example 1 with FlinkKinesisProducer

use of org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer in project flink by apache.

the class KinesisEventsGeneratorProducerThread method create.

public static Thread create(final int totalEventCount, final int parallelism, final String awsAccessKey, final String awsSecretKey, final String awsRegion, final String kinesisStreamName, final AtomicReference<Throwable> errorHandler, final int flinkPort, final Configuration flinkConfig) {
    Runnable kinesisEventsGeneratorProducer = new Runnable() {

        @Override
        public void run() {
            try {
                StreamExecutionEnvironment see = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort, flinkConfig);
                see.setParallelism(parallelism);
                // start data generator
                DataStream<String> simpleStringStream = see.addSource(new KinesisEventsGeneratorProducerThread.EventsGenerator(totalEventCount)).setParallelism(1);
                Properties producerProps = new Properties();
                producerProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, awsAccessKey);
                producerProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, awsSecretKey);
                producerProps.setProperty(AWSConfigConstants.AWS_REGION, awsRegion);
                FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(new SimpleStringSchema(), producerProps);
                kinesis.setFailOnError(true);
                kinesis.setDefaultStream(kinesisStreamName);
                kinesis.setDefaultPartition("0");
                simpleStringStream.addSink(kinesis);
                LOG.info("Starting producing topology");
                see.execute("Producing topology");
                LOG.info("Producing topo finished");
            } catch (Exception e) {
                LOG.warn("Error while running producing topology", e);
                errorHandler.set(e);
            }
        }
    };
    return new Thread(kinesisEventsGeneratorProducer);
}

Example 2 with FlinkKinesisProducer

use of org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer in project flink by apache.

the class ProduceIntoKinesis method main.

public static void main(String[] args) throws Exception {
    ParameterTool pt = ParameterTool.fromArgs(args);
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(1);
    DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());
    Properties kinesisProducerConfig = new Properties();
    kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
    kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
    kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));
    FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(new SimpleStringSchema(), kinesisProducerConfig);
    kinesis.setFailOnError(true);
    kinesis.setDefaultStream("flink-test");
    kinesis.setDefaultPartition("0");
    simpleStringStream.addSink(kinesis);
    see.execute();
}

Example 3 with FlinkKinesisProducer

use of org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer in project aws-doc-sdk-examples by awsdocs.

the class StreamingJob method createSinkFromApplicationProperties.

private static FlinkKinesisProducer<String> createSinkFromApplicationProperties() throws IOException {
    Map<String, Properties> applicationProperties = KinesisAnalyticsRuntime.getApplicationProperties();
    FlinkKinesisProducer<String> sink = new FlinkKinesisProducer<>(new SimpleStringSchema(), applicationProperties.get("ProducerConfigProperties"));
    sink.setDefaultStream(outputStreamName);
    sink.setDefaultPartition("0");
    return sink;
}

Example 4 with FlinkKinesisProducer

use of org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer in project flink by apache.

the class KinesisExample method main.

public static void main(String[] args) throws Exception {
    // parse input arguments
    final ParameterTool parameterTool = ParameterTool.fromArgs(args);
    StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool);
    String inputStream = parameterTool.getRequired("input-stream");
    String outputStream = parameterTool.getRequired("output-stream");
    FlinkKinesisConsumer<KafkaEvent> consumer = new FlinkKinesisConsumer<>(inputStream, new KafkaEventSchema(), parameterTool.getProperties());
    consumer.setPeriodicWatermarkAssigner(new CustomWatermarkExtractor());
    Properties producerProperties = new Properties(parameterTool.getProperties());
    // producer needs region even when URL is specified
    producerProperties.putIfAbsent(ConsumerConfigConstants.AWS_REGION, "us-east-1");
    // test driver does not deaggregate
    producerProperties.putIfAbsent("AggregationEnabled", String.valueOf(false));
    // KPL does not recognize endpoint URL..
    String kinesisUrl = producerProperties.getProperty(ConsumerConfigConstants.AWS_ENDPOINT);
    if (kinesisUrl != null) {
        URL url = new URL(kinesisUrl);
        producerProperties.put("KinesisEndpoint", url.getHost());
        producerProperties.put("KinesisPort", Integer.toString(url.getPort()));
        producerProperties.put("VerifyCertificate", "false");
    }
    FlinkKinesisProducer<KafkaEvent> producer = new FlinkKinesisProducer<>(new KafkaEventSchema(), producerProperties);
    producer.setDefaultStream(outputStream);
    producer.setDefaultPartition("fakePartition");
    DataStream<KafkaEvent> input = env.addSource(consumer).keyBy("word").map(new RollingAdditionMapper());
    input.addSink(producer);
    env.execute();
}

Example 5 with FlinkKinesisProducer

use of org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer in project flink by apache.

the class ManualConsumerProducerTest method main.

public static void main(String[] args) throws Exception {
    ParameterTool pt = ParameterTool.fromArgs(args);
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(4);
    DataStream<String> simpleStringStream = see.addSource(new ProduceIntoKinesis.EventsGenerator());
    Properties kinesisProducerConfig = new Properties();
    kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
    kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
    kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));
    FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(new KinesisSerializationSchema<String>() {

        @Override
        public ByteBuffer serialize(String element) {
            return ByteBuffer.wrap(element.getBytes(ConfigConstants.DEFAULT_CHARSET));
        }

        // every 10th element goes into a different stream
        @Override
        public String getTargetStream(String element) {
            if (element.split("-")[0].endsWith("0")) {
                return "flink-test-2";
            }
            // send to default stream
            return null;
        }
    }, kinesisProducerConfig);
    kinesis.setFailOnError(true);
    kinesis.setDefaultStream("test-flink");
    kinesis.setDefaultPartition("0");
    kinesis.setCustomPartitioner(new KinesisPartitioner<String>() {

        @Override
        public String getPartitionId(String element) {
            int l = element.length();
            return element.substring(l - 1, l);
        }
    });
    simpleStringStream.addSink(kinesis);
    // consuming topology
    Properties consumerProps = new Properties();
    consumerProps.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
    consumerProps.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));
    consumerProps.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
    DataStream<String> consuming = see.addSource(new FlinkKinesisConsumer<>("test-flink", new SimpleStringSchema(), consumerProps));
    // validate consumed records for correctness
    consuming.flatMap(new FlatMapFunction<String, String>() {

        @Override
        public void flatMap(String value, Collector<String> out) throws Exception {
            String[] parts = value.split("-");
            try {
                long l = Long.parseLong(parts[0]);
                if (l < 0) {
                    throw new RuntimeException("Negative");
                }
            } catch (NumberFormatException nfe) {
                throw new RuntimeException("First part of '" + value + "' is not a valid numeric type");
            }
            if (parts[1].length() != 12) {
                throw new RuntimeException("Second part of '" + value + "' doesn't have 12 characters");
            }
        }
    });
    consuming.print();
    see.execute();
}