Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 26 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentQueueE2ETest method testCancelReadRequestOnLastDisconnect.

@Test(timeOut = 60000)
public void testCancelReadRequestOnLastDisconnect() throws Exception {
    String key = "testCancelReadRequestOnLastDisconnect";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(1000);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            consumer2.acknowledge(message2);
        }
        message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Close Consumer 1 and 2
    log.info("Consumer 1 closed");
    log.info("Consumer 2 closed");
    consumer1.close();
    consumer2.close();
    // 6. Producer produces more messages
    for (int i = totalMessages; i < 2 * totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 7. Consumer reconnects
    consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 8. Check number of messages received
    receivedConsumer1 = 0;
    message1 = consumer1.receive();
    while (message1 != null) {
        log.info("Consumer 1 Received: " + new String(message1.getData()));
        receivedConsumer1++;
        message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    }
    log.info("Total receives by Consumer 2 = " + receivedConsumer2);
    assertEquals(receivedConsumer1, totalMessages);
}

Example 27 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentQueueE2ETest method testReplayOnConsumerDisconnect.

@Test
public void testReplayOnConsumerDisconnect() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic3";
    final String subName = "sub3";
    final int numMsgs = 100;
    final List<String> messagesProduced = Lists.newArrayListWithCapacity(numMsgs);
    final List<String> messagesConsumed = new BlockingArrayQueue<>(numMsgs);
    ConsumerConfiguration conf1 = new ConsumerConfiguration();
    conf1.setSubscriptionType(SubscriptionType.Shared);
    conf1.setMessageListener((consumer, msg) -> {
        try {
            consumer.acknowledge(msg);
            messagesConsumed.add(new String(msg.getData()));
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    ConsumerConfiguration conf2 = new ConsumerConfiguration();
    conf2.setSubscriptionType(SubscriptionType.Shared);
    conf2.setMessageListener((consumer, msg) -> {
        try {
        // do nothing
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
    // consumer2 does not ack messages
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
        messagesProduced.add(message);
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    consumer2.close();
    for (int n = 0; n < 10 && messagesConsumed.size() < numMsgs; n++) {
        Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    }
    // 1. consumer1 gets all messages
    assertTrue(CollectionUtils.subtract(messagesProduced, messagesConsumed).isEmpty());
    consumer1.close();
    admin.persistentTopics().delete(topicName);
}

Example 28 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentQueueE2ETest method testSharedSingleAckedNormalTopic.

@Test(timeOut = 300000)
public void testSharedSingleAckedNormalTopic() throws Exception {
    String key = "test1";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 50;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(10);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
        message1 = consumer1.receive(10000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(10000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Close Consumer 1
    log.info("Consumer 1 closed");
    consumer1.close();
    // 6. Consumer 1's unAcked messages should be sent to Consumer 2
    for (int i = 0; i < totalMessages; i++) {
        message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
        if (message2 == null) {
            log.info("Consumer 2 - No Message in Incoming Message Queue, will try again");
            continue;
        }
        log.info("Consumer 2 Received: " + new String(message2.getData()));
        receivedConsumer2 += 1;
    }
    log.info("Total receives by Consumer 2 = " + receivedConsumer2);
    assertEquals(receivedConsumer2, totalMessages);
}

Example 29 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentTopicE2ETest method testGC.

@Test
public void testGC() throws Exception {
    // 1. Simple successful GC
    String topicName = "persistent://prop/use/ns-abc/topic-10";
    Producer producer = pulsarClient.createProducer(topicName);
    producer.close();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    runGC();
    assertNull(pulsar.getBrokerService().getTopicReference(topicName));
    // 2. Topic is not GCed with live connection
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    String subName = "sub1";
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();
    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    // 4. Topic can be GCed after unsubscribe
    admin.persistentTopics().deleteSubscription(topicName, subName);
    runGC();
    assertNull(pulsar.getBrokerService().getTopicReference(topicName));
}

Example 30 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageExpiryWithFewExpiredBacklog.

@Test
public void testMessageExpiryWithFewExpiredBacklog() throws Exception {
    int messageTTLSecs = 10;
    String namespaceName = "prop/use/expiry-check-1";
    admin.namespaces().createNamespace(namespaceName);
    admin.namespaces().setNamespaceMessageTTL(namespaceName, messageTTLSecs);
    final String topicName = "persistent://prop/use/expiry-check-1/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(TimeUnit.SECONDS.toMillis(messageTTLSecs));
    runMessageExpiryCheck();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(TimeUnit.SECONDS.toMillis(messageTTLSecs / 2));
    runMessageExpiryCheck();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
}