Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 56 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class CmdConsume method run.

/**
     * Run the consume command.
     *
     * @return 0 for success, < 0 otherwise
     */
public int run() throws PulsarClientException, IOException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.serviceURL == null || this.serviceURL.isEmpty())
        throw (new ParameterException("Broker URL is not provided."));
    if (this.subscriptionName == null || this.subscriptionName.isEmpty())
        throw (new ParameterException("Subscription name is not provided."));
    if (this.numMessagesToConsume < 0)
        throw (new ParameterException("Number of messages should be zero or positive."));
    String topic = this.mainOptions.get(0);
    int numMessagesConsumed = 0;
    int returnCode = 0;
    try {
        ConsumerConfiguration consumerConf = new ConsumerConfiguration();
        consumerConf.setSubscriptionType(this.subscriptionType);
        PulsarClient client = PulsarClient.create(this.serviceURL, this.clientConfig);
        Consumer consumer = client.subscribe(topic, this.subscriptionName, consumerConf);
        RateLimiter limiter = (this.consumeRate > 0) ? RateLimiter.create(this.consumeRate) : null;
        while (this.numMessagesToConsume == 0 || numMessagesConsumed < this.numMessagesToConsume) {
            if (limiter != null) {
                limiter.acquire();
            }
            Message msg = consumer.receive(5, TimeUnit.SECONDS);
            if (msg == null) {
                LOG.warn("No message to consume after waiting for 20 seconds.");
            } else {
                numMessagesConsumed += 1;
                System.out.println(MESSAGE_BOUNDARY);
                String output = this.interpretMessage(msg, displayHex);
                System.out.println(output);
                consumer.acknowledge(msg);
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while consuming messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully consumed", numMessagesConsumed);
    }
    return returnCode;
}

Example 57 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizePartitionedTopicInCompatibility.

@Test(expectedExceptions = PulsarClientException.class)
public void zeroQueueSizePartitionedTopicInCompatibility() throws PulsarClientException, PulsarAdminException {
    String key = "zeroQueueSizePartitionedTopicInCompatibility";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, configuration);
}

Example 58 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class StormExample method main.

public static void main(String[] args) throws PulsarClientException {
    ClientConfiguration clientConf = new ClientConfiguration();
    // String authPluginClassName = "com.yahoo.pulsar.client.impl.auth.MyAuthentication";
    // String authParams = "key1:val1,key2:val2";
    // clientConf.setAuthentication(authPluginClassName, authParams);
    String topic1 = "persistent://my-property/use/my-ns/my-topic1";
    String topic2 = "persistent://my-property/use/my-ns/my-topic2";
    String subscriptionName1 = "my-subscriber-name1";
    String subscriptionName2 = "my-subscriber-name2";
    // create spout
    PulsarSpoutConfiguration spoutConf = new PulsarSpoutConfiguration();
    spoutConf.setServiceUrl(serviceUrl);
    spoutConf.setTopic(topic1);
    spoutConf.setSubscriptionName(subscriptionName1);
    spoutConf.setMessageToValuesMapper(messageToValuesMapper);
    PulsarSpout spout = new PulsarSpout(spoutConf, clientConf);
    // create bolt
    PulsarBoltConfiguration boltConf = new PulsarBoltConfiguration();
    boltConf.setServiceUrl(serviceUrl);
    boltConf.setTopic(topic2);
    boltConf.setTupleToMessageMapper(tupleToMessageMapper);
    PulsarBolt bolt = new PulsarBolt(boltConf, clientConf);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", spout);
    builder.setBolt("testBolt", bolt).shuffleGrouping("testSpout");
    Config conf = new Config();
    conf.setNumWorkers(2);
    conf.setDebug(true);
    conf.registerMetricsConsumer(PulsarMetricsConsumer.class);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    PulsarClient pulsarClient = PulsarClient.create(serviceUrl, clientConf);
    // create a consumer on topic2 to receive messages from the bolt when the processing is done
    Consumer consumer = pulsarClient.subscribe(topic2, subscriptionName2);
    // create a producer on topic1 to send messages that will be received by the spout
    Producer producer = pulsarClient.createProducer(topic1);
    for (int i = 0; i < 10; i++) {
        String msg = "msg-" + i;
        producer.send(msg.getBytes());
        LOG.info("Message {} sent", msg);
    }
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        LOG.info("Message {} received", new String(msg.getData()));
    }
    cluster.killTopology("test");
    cluster.shutdown();
}

Example 59 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one destination name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener listener = new MessageListener() {

        public void received(Consumer consumer, Message msg) {
            messagesReceived.increment();
            bytesReceived.add(msg.getData().length);
            if (limiter != null) {
                limiter.acquire();
            }
            consumer.acknowledgeAsync(msg);
        }
    };
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    List<Future<Consumer>> futures = Lists.newArrayList();
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setMessageListener(listener);
    consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize);
    for (int i = 0; i < arguments.numDestinations; i++) {
        final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i));
        log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig));
        }
    }
    for (Future<Consumer> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Throughput received: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}

Example 60 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class AdminApiTest method testUnsubscribeOnNamespace.

@Test(dataProvider = "bundling")
public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception {
    admin.namespaces().createNamespace("prop-xyz/use/ns1-bundles", numBundles);
    // create consumer and subscription
    Consumer consumer1 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub");
    Consumer consumer2 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub-1");
    /* Consumer consumer3 = */
    pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub-2");
    Consumer consumer4 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds1", "my-sub");
    Consumer consumer5 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds1", "my-sub-1");
    try {
        admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub");
        fail("should have failed");
    } catch (PulsarAdminException.PreconditionFailedException e) {
    // ok
    }
    consumer1.close();
    try {
        admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub");
        fail("should have failed");
    } catch (PulsarAdminException.PreconditionFailedException e) {
    // ok
    }
    consumer4.close();
    admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), Lists.newArrayList("my-sub-1", "my-sub-2"));
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds1"), Lists.newArrayList("my-sub-1"));
    consumer2.close();
    consumer5.close();
    admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub-1");
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), Lists.newArrayList("my-sub-2"));
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds1"), Lists.newArrayList());
}