Examples with SparkStreamingPulsarReceiver org.apache.pulsar.spark.SparkStreamingPulsarReceiver used on opensource projects

Example 1 with SparkStreamingPulsarReceiver

use of org.apache.pulsar.spark.SparkStreamingPulsarReceiver in project incubator-pulsar by apache.

the class SparkStreamingPulsarReceiverExample method main.

public static void main(String[] args) throws InterruptedException {
    SparkConf conf = new SparkConf().setMaster("local[*]").setAppName("pulsar-spark");
    JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));
    ClientConfiguration clientConf = new ClientConfiguration();
    ConsumerConfiguration consConf = new ConsumerConfiguration();
    String url = "pulsar://localhost:6650/";
    String topic = "persistent://sample/standalone/ns1/topic1";
    String subs = "sub1";
    JavaReceiverInputDStream<byte[]> msgs = jssc.receiverStream(new SparkStreamingPulsarReceiver(clientConf, consConf, url, topic, subs));
    JavaDStream<Integer> isContainingPulsar = msgs.flatMap(new FlatMapFunction<byte[], Integer>() {

        @Override
        public Iterator<Integer> call(byte[] msg) {
            return Arrays.asList(((new String(msg)).indexOf("Pulsar") != -1) ? 1 : 0).iterator();
        }
    });
    JavaDStream<Integer> numOfPulsar = isContainingPulsar.reduce(new Function2<Integer, Integer, Integer>() {

        @Override
        public Integer call(Integer i1, Integer i2) {
            return i1 + i2;
        }
    });
    numOfPulsar.print();
    jssc.start();
    jssc.awaitTermination();
}