Search in sources :

Example 1 with LocationStrategy

use of org.apache.rocketmq.spark.LocationStrategy in project rocketmq-externals by apache.

the class RocketMqUtilsTest method testConsumer.

@Test
public void testConsumer() throws MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException {
    // start up spark
    Map<String, String> optionParams = new HashMap<>();
    optionParams.put(RocketMQConfig.NAME_SERVER_ADDR, NAME_SERVER);
    SparkConf sparkConf = new SparkConf().setAppName("JavaCustomReceiver").setMaster("local[*]");
    JavaStreamingContext sc = new JavaStreamingContext(sparkConf, new Duration(1000));
    List<String> topics = new ArrayList<>();
    topics.add(TOPIC_DEFAULT);
    LocationStrategy locationStrategy = LocationStrategy.PreferConsistent();
    JavaInputDStream<MessageExt> stream = RocketMqUtils.createJavaMQPullStream(sc, UUID.randomUUID().toString(), topics, ConsumerStrategy.earliest(), false, false, false, locationStrategy, optionParams);
    final Set<MessageExt> result = Collections.synchronizedSet(new HashSet<MessageExt>());
    stream.foreachRDD(new VoidFunction<JavaRDD<MessageExt>>() {

        @Override
        public void call(JavaRDD<MessageExt> messageExtJavaRDD) throws Exception {
            result.addAll(messageExtJavaRDD.collect());
        }
    });
    sc.start();
    long startTime = System.currentTimeMillis();
    boolean matches = false;
    while (!matches && System.currentTimeMillis() - startTime < 20000) {
        matches = MESSAGE_NUM == result.size();
        Thread.sleep(50);
    }
    sc.stop();
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Duration(org.apache.spark.streaming.Duration) MQClientException(org.apache.rocketmq.client.exception.MQClientException) MQBrokerException(org.apache.rocketmq.client.exception.MQBrokerException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) JavaRDD(org.apache.spark.api.java.JavaRDD) JavaStreamingContext(org.apache.spark.streaming.api.java.JavaStreamingContext) MessageExt(org.apache.rocketmq.common.message.MessageExt) LocationStrategy(org.apache.rocketmq.spark.LocationStrategy) SparkConf(org.apache.spark.SparkConf) Test(org.junit.Test)

Example 2 with LocationStrategy

use of org.apache.rocketmq.spark.LocationStrategy in project rocketmq-externals by apache.

the class RocketMqUtilsTest method testGetOffsets.

@Test
public void testGetOffsets() throws MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException {
    Map<String, String> optionParams = new HashMap<>();
    optionParams.put(RocketMQConfig.NAME_SERVER_ADDR, NAME_SERVER);
    SparkConf sparkConf = new SparkConf().setAppName("JavaCustomReceiver").setMaster("local[*]");
    JavaStreamingContext sc = new JavaStreamingContext(sparkConf, new Duration(1000));
    List<String> topics = new ArrayList<>();
    topics.add(TOPIC_DEFAULT);
    LocationStrategy locationStrategy = LocationStrategy.PreferConsistent();
    JavaInputDStream<MessageExt> dStream = RocketMqUtils.createJavaMQPullStream(sc, UUID.randomUUID().toString(), topics, ConsumerStrategy.earliest(), false, false, false, locationStrategy, optionParams);
    // hold a reference to the current offset ranges, so it can be used downstream
    final AtomicReference<Map<TopicQueueId, OffsetRange[]>> offsetRanges = new AtomicReference<>();
    final Set<MessageExt> result = Collections.synchronizedSet(new HashSet<MessageExt>());
    dStream.transform(new Function<JavaRDD<MessageExt>, JavaRDD<MessageExt>>() {

        @Override
        public JavaRDD<MessageExt> call(JavaRDD<MessageExt> v1) throws Exception {
            Map<TopicQueueId, OffsetRange[]> offsets = ((HasOffsetRanges) v1.rdd()).offsetRanges();
            offsetRanges.set(offsets);
            return v1;
        }
    }).foreachRDD(new VoidFunction<JavaRDD<MessageExt>>() {

        @Override
        public void call(JavaRDD<MessageExt> messageExtJavaRDD) throws Exception {
            result.addAll(messageExtJavaRDD.collect());
        }
    });
    sc.start();
    long startTime = System.currentTimeMillis();
    boolean matches = false;
    while (!matches && System.currentTimeMillis() - startTime < 10000) {
        matches = MESSAGE_NUM == result.size();
        Thread.sleep(50);
    }
    sc.stop();
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HasOffsetRanges(org.apache.rocketmq.spark.HasOffsetRanges) JavaStreamingContext(org.apache.spark.streaming.api.java.JavaStreamingContext) VoidFunction(org.apache.spark.api.java.function.VoidFunction) Function(org.apache.spark.api.java.function.Function) Duration(org.apache.spark.streaming.Duration) AtomicReference(java.util.concurrent.atomic.AtomicReference) MQClientException(org.apache.rocketmq.client.exception.MQClientException) MQBrokerException(org.apache.rocketmq.client.exception.MQBrokerException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) JavaRDD(org.apache.spark.api.java.JavaRDD) OffsetRange(org.apache.rocketmq.spark.OffsetRange) MessageExt(org.apache.rocketmq.common.message.MessageExt) LocationStrategy(org.apache.rocketmq.spark.LocationStrategy) SparkConf(org.apache.spark.SparkConf) HashMap(java.util.HashMap) Map(java.util.Map) TopicQueueId(org.apache.rocketmq.spark.TopicQueueId) Test(org.junit.Test)

Aggregations

UnsupportedEncodingException (java.io.UnsupportedEncodingException)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 MQBrokerException (org.apache.rocketmq.client.exception.MQBrokerException)2 MQClientException (org.apache.rocketmq.client.exception.MQClientException)2 MessageExt (org.apache.rocketmq.common.message.MessageExt)2 LocationStrategy (org.apache.rocketmq.spark.LocationStrategy)2 SparkConf (org.apache.spark.SparkConf)2 JavaRDD (org.apache.spark.api.java.JavaRDD)2 Duration (org.apache.spark.streaming.Duration)2 JavaStreamingContext (org.apache.spark.streaming.api.java.JavaStreamingContext)2 Test (org.junit.Test)2 Map (java.util.Map)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 HasOffsetRanges (org.apache.rocketmq.spark.HasOffsetRanges)1 OffsetRange (org.apache.rocketmq.spark.OffsetRange)1 TopicQueueId (org.apache.rocketmq.spark.TopicQueueId)1 Function (org.apache.spark.api.java.function.Function)1 VoidFunction (org.apache.spark.api.java.function.VoidFunction)1