Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 91 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchSyncProducerWithFixedBatchSize.

@Test
public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception {
    int numMsgs = 10;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize";
    final String subscriptionName = "syncsub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(1000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        producer.send(msg);
    }
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 10 messages in the backlog since we sent 10 messages with the batch size set to 5.
    // However, we are using synchronous send and so each message will go as an individual message
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 10);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        Assert.assertEquals(receivedMessage, expectedMessage, "Received message " + receivedMessage + " did not match the expected message " + expectedMessage);
    }
    consumer.close();
    producer.close();
}

Example 92 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class BatchMessageTest method testBatchProducerWithLargeMessage.

@Test(dataProvider = "codec")
public void testBatchProducerWithLargeMessage(CompressionType compressionType) throws Exception {
    int numMsgs = 50;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/finance/testBatchProducerWithLargeMessage";
    final String subscriptionName = "large-message-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        if (i == 25) {
            // send a large message
            byte[] largeMessage = new byte[128 * 1024 + 4];
            Message msg = MessageBuilder.create().setContent(largeMessage).build();
            sendFutureList.add(producer.sendAsync(msg));
        } else {
            byte[] message = ("msg-" + i).getBytes();
            Message msg = MessageBuilder.create().setContent(message).build();
            sendFutureList.add(producer.sendAsync(msg));
        }
    }
    byte[] message = ("msg-" + "last").getBytes();
    Message lastMsg = MessageBuilder.create().setContent(message).build();
    sendFutureList.add(producer.sendAsync(lastMsg));
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 3 messages in the backlog since the large message in the middle should
    // close out the batch and be sent in a batch of its own
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 3);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    for (int i = 0; i <= numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        LOG.info("received msg - {}", msg.getData().toString());
        consumer.acknowledge(msg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 93 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ClientErrorsTest method testFlowSendWhenPartitionedSubscribeCompletes.

@Test
public void testFlowSendWhenPartitionedSubscribeCompletes() throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    AtomicInteger subscribed = new AtomicInteger();
    AtomicBoolean fail = new AtomicBoolean(false);
    mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
        subscribed.incrementAndGet();
        ctx.writeAndFlush(Commands.newSuccess(subscribe.getRequestId()));
    });
    mockBrokerService.setHandleFlow((ctx, sendFlow) -> {
        if (subscribed.get() != 4) {
            fail.set(true);
        }
    });
    Consumer consumer = client.subscribe("persistent://prop/use/ns/multi-part-t1", "my-sub");
    if (fail.get()) {
        fail("Flow command should have been sent after all 4 partitions subscribe successfully");
    }
    mockBrokerService.resetHandleSubscribe();
    mockBrokerService.resetHandleFlow();
    client.close();
}

Example 94 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ResendRequestTest method testExclusiveSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testExclusiveSingleAckedPartitionedTopic() throws Exception {
    String key = "testExclusiveSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 4;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message = consumer.receive();
    int messageCount = 0;
    log.info("Message received " + new String(message.getData()));
    do {
        messageCount += 1;
        log.info("Message received " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageCount, totalMessages);
    // 5. Ask for redeliver
    consumer.redeliverUnacknowledgedMessages();
    // 6. Check if Messages redelivered again
    message = consumer.receive();
    messageCount = 0;
    log.info("Message received " + new String(message.getData()));
    do {
        messageCount += 1;
        log.info("Message received " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageCount, totalMessages);
}

Example 95 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ClientErrorsTest method testMockBrokerService.

@Test
public void testMockBrokerService() throws Exception {
    // test default actions of mock broker service
    try {
        PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setSubscriptionType(SubscriptionType.Exclusive);
        Consumer consumer = client.subscribe("persistent://prop/use/ns/t1", "sub1", conf);
        Producer producer = client.createProducer("persistent://prop/use/ns/t1");
        Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
        producer.send("message".getBytes());
        Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
        consumer.unsubscribe();
        producer.close();
        consumer.close();
        client.close();
    } catch (Exception e) {
        fail("None of the mocked operations should throw a client side exception");
    }
}