Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 71 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithPartition.

@Test(enabled = false)
public void testSimpleConsumerEventsWithPartition() throws Exception {
    int numPartitions = 4;
    final String topicName = "persistent://prop/use/ns-abc/failover-topic2";
    final DestinationName destName = DestinationName.get(topicName);
    final String subName = "sub1";
    final int numMsgs = 100;
    Set<String> uniqueMessages = new HashSet<>();
    admin.persistentTopics().createPartitionedTopic(topicName, numPartitions);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
    consumerConf1.setSubscriptionType(SubscriptionType.Failover);
    consumerConf1.setConsumerName("1");
    ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
    consumerConf2.setSubscriptionType(SubscriptionType.Failover);
    consumerConf2.setConsumerName("2");
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
    PersistentTopic topicRef;
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(0).toString());
    PersistentDispatcherSingleActiveConsumer disp0 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(1).toString());
    PersistentDispatcherSingleActiveConsumer disp1 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(2).toString());
    PersistentDispatcherSingleActiveConsumer disp2 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(3).toString());
    PersistentDispatcherSingleActiveConsumer disp3 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // equal distribution between both consumers
    int totalMessages = 0;
    Message msg = null;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer1.acknowledge(msg);
    }
    Assert.assertEquals(totalMessages, numMsgs / 2);
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer2.acknowledge(msg);
    }
    Assert.assertEquals(totalMessages, numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    totalMessages = 0;
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // add a consumer
    ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
    consumerConf3.setSubscriptionType(SubscriptionType.Failover);
    consumerConf3.setConsumerName("3");
    for (int i = 0; i < 20; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    int consumer1Messages = 0;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer1Messages, 55);
            break;
        }
        consumer1Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    int consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 50);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    int consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 15, 10);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    uniqueMessages.clear();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // remove a consumer
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 70, 5);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 70, 5);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    producer.close();
    consumer2.close();
    consumer3.unsubscribe();
    admin.persistentTopics().deletePartitionedTopic(topicName);
}

Example 72 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithoutPartition.

@Test
public void testSimpleConsumerEventsWithoutPartition() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic1";
    final String subName = "sub1";
    final int numMsgs = 100;
    ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
    consumerConf1.setSubscriptionType(SubscriptionType.Failover);
    consumerConf1.setConsumerName("1");
    ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
    consumerConf2.setSubscriptionType(SubscriptionType.Failover);
    consumerConf2.setConsumerName("2");
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    // 2. validate basic dispatcher state
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    assertEquals(subRef.getDispatcher().getType(), SubType.Failover);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // 3. consumer1 should have all the messages while consumer2 should have no messages
    Message msg = null;
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 5. master consumer failure should resend unacked messages and new messages to another consumer
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    for (int i = 5; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
    // do not ack
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 6. consumer subscription should send messages to the new consumer if its name is highest in the list
    for (int i = 0; i < 5; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    Assert.assertNull(consumer1.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 7. consumer subscription should not send messages to the new consumer if its name is not highest in the list
    ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
    consumerConf3.setSubscriptionType(SubscriptionType.Failover);
    consumerConf3.setConsumerName("3");
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    Assert.assertNull(consumer3.receive(1, TimeUnit.SECONDS));
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 8. unsubscribe not allowed if multiple consumers connected
    try {
        consumer1.unsubscribe();
        fail("should fail");
    } catch (PulsarClientException e) {
    // ok
    }
    // 9. unsubscribe allowed if there is a lone consumer
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    try {
        consumer3.unsubscribe();
    } catch (PulsarClientException e) {
        fail("Should not fail", e);
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getPersistentSubscription(subName);
    assertNull(subRef);
    producer.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}

Example 73 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentQueueE2ETest method testSimpleConsumerEvents.

@Test
public void testSimpleConsumerEvents() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic1";
    final String subName = "sub1";
    final int numMsgs = 100;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    // 2. validate basic dispatcher state
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    assertEquals(subRef.getDispatcher().getType(), SubType.Shared);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs * 2; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs * 2);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // both consumers will together consumer all messages
    Message msg;
    Consumer c = consumer1;
    while (true) {
        try {
            msg = c.receive(1, TimeUnit.SECONDS);
            c.acknowledge(msg);
        } catch (PulsarClientException e) {
            if (c.equals(consumer1)) {
                c = consumer2;
            } else {
                break;
            }
        }
    }
    rolloverPerIntervalStats();
    // 3. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 4. shared consumer unsubscribe not allowed
    try {
        consumer1.unsubscribe();
        fail("should fail");
    } catch (PulsarClientException e) {
    // ok
    }
    // 5. cumulative acks disabled
    consumer1.close();
    producer.send("message".getBytes());
    msg = consumer2.receive();
    try {
        consumer2.acknowledgeCumulative(msg);
        fail("Should fail");
    } catch (PulsarClientException e) {
        assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
    }
    // 6. unsubscribe allowed if this is the lone consumer
    try {
        consumer2.unsubscribe();
    } catch (PulsarClientException e) {
        fail("Should not fail");
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getPersistentSubscription(subName);
    assertNull(subRef);
    producer.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 74 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentQueueE2ETest method testConsumersWithDifferentPermits.

@Test
public void testConsumersWithDifferentPermits() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic4";
    final String subName = "sub4";
    final int numMsgs = 10000;
    final AtomicInteger msgCountConsumer1 = new AtomicInteger(0);
    final AtomicInteger msgCountConsumer2 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs);
    int recvQ1 = 10;
    ConsumerConfiguration conf1 = new ConsumerConfiguration();
    conf1.setSubscriptionType(SubscriptionType.Shared);
    conf1.setReceiverQueueSize(recvQ1);
    conf1.setMessageListener((consumer, msg) -> {
        msgCountConsumer1.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    });
    int recvQ2 = 1;
    ConsumerConfiguration conf2 = new ConsumerConfiguration();
    conf2.setSubscriptionType(SubscriptionType.Shared);
    conf2.setReceiverQueueSize(recvQ2);
    conf2.setMessageListener((consumer, msg) -> {
        msgCountConsumer2.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    });
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(5, TimeUnit.SECONDS);
    assertEquals(msgCountConsumer1.get(), numMsgs - numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    assertEquals(msgCountConsumer2.get(), numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 75 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentTopicE2ETest method testGracefulClose.

@Test(enabled = false)
public // TODO: enable this after java client supports graceful close
void testGracefulClose() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic4";
    final String subName = "sub4";
    Producer producer = pulsarClient.createProducer(topicName);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    ExecutorService executor = Executors.newCachedThreadPool();
    CountDownLatch latch = new CountDownLatch(1);
    executor.submit(() -> {
        for (int i = 0; i < 10; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        latch.countDown();
        return null;
    });
    producer.close();
    // 1. verify there are no pending publish acks once the producer close
    // is completed on client
    assertEquals(topicRef.getProducers().values().iterator().next().getPendingPublishAcks(), 0);
    // safety latch in case of failure,
    // wait for the spawned thread to complete
    latch.await();
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(subRef);
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
    }
    // message acks
    try {
        consumer.close();
        fail("should have failed");
    } catch (IllegalStateException e) {
    // Expected - messages not acked
    }
    consumer.acknowledgeCumulative(msg);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // 3. verify consumer close succeeds once all messages are ack'ed
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
}