Examples with Consumer org.apache.pulsar.client.api.Consumer used on opensource projects

Example 21 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class MembershipManagerTest method testConsumerEventListener.

@Test
public void testConsumerEventListener() throws Exception {
    PulsarClient mockClient = mock(PulsarClient.class);
    Consumer mockConsumer = mock(Consumer.class);
    AtomicReference<ConsumerEventListener> listenerHolder = new AtomicReference<>();
    when(mockClient.subscribe(eq(workerConfig.getClusterCoordinationTopic()), eq(MembershipManager.COORDINATION_TOPIC_SUBSCRIPTION), any(ConsumerConfiguration.class))).thenAnswer(invocationOnMock -> {
        ConsumerConfiguration conf = invocationOnMock.getArgumentAt(2, ConsumerConfiguration.class);
        listenerHolder.set(conf.getConsumerEventListener());
        return mockConsumer;
    });
    MembershipManager membershipManager = spy(new MembershipManager(workerConfig, mockClient));
    assertFalse(membershipManager.isLeader());
    verify(mockClient, times(1)).subscribe(eq(workerConfig.getClusterCoordinationTopic()), eq(MembershipManager.COORDINATION_TOPIC_SUBSCRIPTION), any(ConsumerConfiguration.class));
    listenerHolder.get().becameActive(mockConsumer, 0);
    assertTrue(membershipManager.isLeader());
    listenerHolder.get().becameInactive(mockConsumer, 0);
    assertFalse(membershipManager.isLeader());
}

Example 22 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class StormExample method main.

public static void main(String[] args) throws PulsarClientException {
    ClientConfiguration clientConf = new ClientConfiguration();
    // String authPluginClassName = "org.apache.pulsar.client.impl.auth.MyAuthentication";
    // String authParams = "key1:val1,key2:val2";
    // clientConf.setAuthentication(authPluginClassName, authParams);
    String topic1 = "persistent://my-property/use/my-ns/my-topic1";
    String topic2 = "persistent://my-property/use/my-ns/my-topic2";
    String subscriptionName1 = "my-subscriber-name1";
    String subscriptionName2 = "my-subscriber-name2";
    // create spout
    PulsarSpoutConfiguration spoutConf = new PulsarSpoutConfiguration();
    spoutConf.setServiceUrl(serviceUrl);
    spoutConf.setTopic(topic1);
    spoutConf.setSubscriptionName(subscriptionName1);
    spoutConf.setMessageToValuesMapper(messageToValuesMapper);
    PulsarSpout spout = new PulsarSpout(spoutConf, clientConf);
    // create bolt
    PulsarBoltConfiguration boltConf = new PulsarBoltConfiguration();
    boltConf.setServiceUrl(serviceUrl);
    boltConf.setTopic(topic2);
    boltConf.setTupleToMessageMapper(tupleToMessageMapper);
    PulsarBolt bolt = new PulsarBolt(boltConf, clientConf);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", spout);
    builder.setBolt("testBolt", bolt).shuffleGrouping("testSpout");
    Config conf = new Config();
    conf.setNumWorkers(2);
    conf.setDebug(true);
    conf.registerMetricsConsumer(PulsarMetricsConsumer.class);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    PulsarClient pulsarClient = PulsarClient.create(serviceUrl, clientConf);
    // create a consumer on topic2 to receive messages from the bolt when the processing is done
    Consumer consumer = pulsarClient.subscribe(topic2, subscriptionName2);
    // create a producer on topic1 to send messages that will be received by the spout
    Producer producer = pulsarClient.createProducer(topic1);
    for (int i = 0; i < 10; i++) {
        String msg = "msg-" + i;
        producer.send(msg.getBytes());
        LOG.info("Message {} sent", msg);
    }
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        LOG.info("Message {} received", new String(msg.getData()));
    }
    cluster.killTopology("test");
    cluster.shutdown();
}

Example 23 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
        if (arguments.useTls == false) {
            arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
        }
        if (isBlank(arguments.tlsTrustCertsFilePath)) {
            arguments.tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", "");
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final TopicName prefixTopicName = TopicName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener<byte[]> listener = (consumer, msg) -> {
        messagesReceived.increment();
        bytesReceived.add(msg.getData().length);
        if (limiter != null) {
            limiter.acquire();
        }
        long latencyMillis = System.currentTimeMillis() - msg.getPublishTime();
        if (latencyMillis >= 0) {
            recorder.recordValue(latencyMillis);
            cumulativeRecorder.recordValue(latencyMillis);
        }
        consumer.acknowledgeAsync(msg);
    };
    ClientBuilder clientBuilder = // 
    PulsarClient.builder().serviceUrl(// 
    arguments.serviceURL).connectionsPerBroker(// 
    arguments.maxConnections).statsInterval(arguments.statsIntervalSeconds, // 
    TimeUnit.SECONDS).ioThreads(// 
    Runtime.getRuntime().availableProcessors()).enableTls(// 
    arguments.useTls).tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = clientBuilder.build();
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        EncKeyReader(byte[] value) {
            keyInfo.setKey(value);
        }

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            if (keyName.equals(arguments.encKeyName)) {
                return keyInfo;
            }
            return null;
        }
    }
    List<Future<Consumer<byte[]>>> futures = Lists.newArrayList();
    ConsumerBuilder<byte[]> consumerBuilder = // 
    pulsarClient.newConsumer().messageListener(// 
    listener).receiverQueueSize(// 
    arguments.receiverQueueSize).subscriptionType(arguments.subscriptionType);
    if (arguments.encKeyName != null) {
        byte[] pKey = Files.readAllBytes(Paths.get(arguments.encKeyFile));
        EncKeyReader keyReader = new EncKeyReader(pKey);
        consumerBuilder.cryptoKeyReader(keyReader);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        final TopicName topicName = (arguments.numTopics == 1) ? prefixTopicName : TopicName.get(String.format("%s-%d", prefixTopicName, i));
        log.info("Adding {} consumers on topic {}", arguments.numConsumers, topicName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(consumerBuilder.clone().topic(topicName.toString()).subscriptionName(subscriberName).subscribeAsync());
        }
    }
    for (Future<Consumer<byte[]>> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} topics", arguments.numConsumers, arguments.numTopics);
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput received: {}  msg/s -- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", dec.format(rate), dec.format(throughput), dec.format(reportHistogram.getMean()), (long) reportHistogram.getValueAtPercentile(50), (long) reportHistogram.getValueAtPercentile(95), (long) reportHistogram.getValueAtPercentile(99), (long) reportHistogram.getValueAtPercentile(99.9), (long) reportHistogram.getValueAtPercentile(99.99), (long) reportHistogram.getMaxValue());
        reportHistogram.reset();
        oldTime = now;
    }
    pulsarClient.close();
}

Example 24 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class PulsarClientImpl method patternTopicSubscribeAsync.

private <T> CompletableFuture<Consumer<T>> patternTopicSubscribeAsync(ConsumerConfigurationData<T> conf, Schema<T> schema) {
    String regex = conf.getTopicsPattern().pattern();
    TopicName destination = TopicName.get(regex);
    NamespaceName namespaceName = destination.getNamespaceObject();
    CompletableFuture<Consumer<T>> consumerSubscribedFuture = new CompletableFuture<>();
    lookup.getTopicsUnderNamespace(namespaceName).thenAccept(topics -> {
        if (log.isDebugEnabled()) {
            log.debug("Get topics under namespace {}, topics.size: {}", namespaceName.toString(), topics.size());
            topics.forEach(topicName -> log.debug("Get topics under namespace {}, topic: {}", namespaceName.toString(), topicName));
        }
        List<String> topicsList = topicsPatternFilter(topics, conf.getTopicsPattern());
        conf.getTopicNames().addAll(topicsList);
        ConsumerBase<T> consumer = new PatternTopicsConsumerImpl<>(conf.getTopicsPattern(), PulsarClientImpl.this, conf, externalExecutorProvider.getExecutor(), consumerSubscribedFuture, schema);
        synchronized (consumers) {
            consumers.put(consumer, Boolean.TRUE);
        }
    }).exceptionally(ex -> {
        log.warn("[{}] Failed to get topics under namespace", namespaceName);
        consumerSubscribedFuture.completeExceptionally(ex);
        return null;
    });
    return consumerSubscribedFuture;
}

Example 25 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class TopicsConsumerImpl method subscribeAsync.

// subscribe one more given topic
public CompletableFuture<Void> subscribeAsync(String topicName) {
    if (!topicNameValid(topicName)) {
        return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Topic name not valid"));
    }
    if (getState() == State.Closing || getState() == State.Closed) {
        return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Topics Consumer was already closed"));
    }
    CompletableFuture<Void> subscribeResult = new CompletableFuture<>();
    final AtomicInteger partitionNumber = new AtomicInteger(0);
    client.getPartitionedTopicMetadata(topicName).thenAccept(metadata -> {
        if (log.isDebugEnabled()) {
            log.debug("Received topic {} metadata.partitions: {}", topicName, metadata.partitions);
        }
        List<CompletableFuture<Consumer<T>>> futureList;
        if (metadata.partitions > 1) {
            this.topics.putIfAbsent(topicName, metadata.partitions);
            numberTopicPartitions.addAndGet(metadata.partitions);
            partitionNumber.addAndGet(metadata.partitions);
            futureList = IntStream.range(0, partitionNumber.get()).mapToObj(partitionIndex -> {
                String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString();
                CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
                ConsumerImpl<T> newConsumer = new ConsumerImpl<>(client, partitionName, internalConfig, client.externalExecutorProvider().getExecutor(), partitionIndex, subFuture, schema);
                consumers.putIfAbsent(newConsumer.getTopic(), newConsumer);
                return subFuture;
            }).collect(Collectors.toList());
        } else {
            this.topics.putIfAbsent(topicName, 1);
            numberTopicPartitions.incrementAndGet();
            partitionNumber.incrementAndGet();
            CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
            ConsumerImpl<T> newConsumer = new ConsumerImpl<>(client, topicName, internalConfig, client.externalExecutorProvider().getExecutor(), 0, subFuture, schema);
            consumers.putIfAbsent(newConsumer.getTopic(), newConsumer);
            futureList = Collections.singletonList(subFuture);
        }
        FutureUtil.waitForAll(futureList).thenAccept(finalFuture -> {
            try {
                if (numberTopicPartitions.get() > maxReceiverQueueSize) {
                    setMaxReceiverQueueSize(numberTopicPartitions.get());
                }
                int numTopics = this.topics.values().stream().mapToInt(Integer::intValue).sum();
                checkState(numberTopicPartitions.get() == numTopics, "numberTopicPartitions " + numberTopicPartitions.get() + " not equals expected: " + numTopics);
                // We have successfully created new consumers, so we can start receiving messages for them
                startReceivingMessages(consumers.values().stream().filter(consumer1 -> {
                    String consumerTopicName = consumer1.getTopic();
                    if (TopicName.get(consumerTopicName).getPartitionedTopicName().equals(TopicName.get(topicName).getPartitionedTopicName().toString())) {
                        return true;
                    } else {
                        return false;
                    }
                }).collect(Collectors.toList()));
                subscribeResult.complete(null);
                log.info("[{}] [{}] Success subscribe new topic {} in topics consumer, numberTopicPartitions {}", topic, subscription, topicName, numberTopicPartitions.get());
                if (this.namespaceName == null) {
                    this.namespaceName = TopicName.get(topicName).getNamespaceObject();
                }
                return;
            } catch (PulsarClientException e) {
                handleSubscribeOneTopicError(topicName, e);
                subscribeResult.completeExceptionally(e);
            }
        }).exceptionally(ex -> {
            handleSubscribeOneTopicError(topicName, ex);
            subscribeResult.completeExceptionally(ex);
            return null;
        });
    }).exceptionally(ex1 -> {
        log.warn("[{}] Failed to get partitioned topic metadata: {}", topicName, ex1.getMessage());
        subscribeResult.completeExceptionally(ex1);
        return null;
    });
    return subscribeResult;
}