Examples with ConsumerImpl com.yahoo.pulsar.client.impl.ConsumerImpl used on opensource projects

Example 6 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeSharedSubscription.

@Test()
public void zeroQueueSizeSharedSubscription() throws PulsarClientException {
    String key = "zeroQueueSizeSharedSubscription";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    int numOfSubscribers = 4;
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    configuration.setSubscriptionType(SubscriptionType.Shared);
    ConsumerImpl[] consumers = new ConsumerImpl[numOfSubscribers];
    for (int i = 0; i < numOfSubscribers; i++) {
        consumers[i] = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    }
    // 4. Produce Messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 5. Consume messages
    Message message;
    for (int i = 0; i < totalMessages; i++) {
        assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
        message = consumers[i % numOfSubscribers].receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 7 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBrokerInternalRetry.

/**
     * This testcase make sure that once consumer lost connection with broker, it always reconnects with broker by
     * retrying on throttling-error exception also.
     * 
     * <pre>
     * 1. all consumers get connected 
     * 2. broker restarts with maxConcurrentLookupRequest = 1 
     * 3. consumers reconnect and some get TooManyRequestException and again retries
     * 4. eventually all consumers will successfully connect to broker
     * </pre>
     * 
     * @throws Exception
     */
@Test
public void testLookupThrottlingForClientByBrokerInternalRetry() throws Exception {
    // create configuration znode
    ZkUtils.createFullPathOptimistic(mockZookKeeper, BROKER_SERVICE_CONFIGURATION_PATH, "{}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Now, znode is created: set the watch and listener on the znode
    setWatchOnThrottlingZnode();
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    com.yahoo.pulsar.client.api.ClientConfiguration clientConf = new com.yahoo.pulsar.client.api.ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    clientConf.setIoThreads(20);
    clientConf.setConnectionsPerBroker(20);
    String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
    PulsarClient pulsarClient = PulsarClient.create(lookupUrl, clientConf);
    upsertLookupPermits(100);
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    List<Consumer> consumers = Lists.newArrayList();
    ExecutorService executor = Executors.newFixedThreadPool(10);
    final int totalConsumers = 8;
    CountDownLatch latch = new CountDownLatch(totalConsumers);
    for (int i = 0; i < totalConsumers; i++) {
        executor.execute(() -> {
            try {
                consumers.add(pulsarClient.subscribe(topicName, "mysub", consumerConfig));
            } catch (PulsarClientException.TooManyLookupRequestException e) {
            // ok
            } catch (Exception e) {
                fail("it shouldn't failed");
            }
            latch.countDown();
        });
    }
    latch.await();
    stopBroker();
    conf.setMaxConcurrentLookupRequest(1);
    startBroker();
    // wait strategically for all consumers to reconnect
    for (int i = 0; i < 5; i++) {
        if (!areAllConsumersConnected(consumers)) {
            Thread.sleep(1000 + (i * 500));
        } else {
            break;
        }
    }
    int totalConnectedConsumers = 0;
    for (int i = 0; i < consumers.size(); i++) {
        if (((ConsumerImpl) consumers.get(i)).isConnected()) {
            totalConnectedConsumers++;
        }
        consumers.get(i).close();
    }
    assertEquals(totalConnectedConsumers, totalConsumers);
    pulsarClient.close();
}

Example 8 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testReceiveWithTimeout.

@Test
public void testReceiveWithTimeout() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-receive-timeout";
    final String subName = "sub";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(1000);
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    assertEquals(consumer.getAvailablePermits(), 0);
    Message msg = consumer.receive(10, TimeUnit.MILLISECONDS);
    assertNull(msg);
    assertEquals(consumer.getAvailablePermits(), 0);
    producer.send("test".getBytes());
    Thread.sleep(100);
    assertEquals(consumer.getAvailablePermits(), 0);
    msg = consumer.receive(10, TimeUnit.MILLISECONDS);
    assertNotNull(msg);
    assertEquals(consumer.getAvailablePermits(), 1);
    msg = consumer.receive(10, TimeUnit.MILLISECONDS);
    assertNull(msg);
    assertEquals(consumer.getAvailablePermits(), 1);
}

Example 9 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause.

/**
     * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
     * blocked due to unacked messsages
     *
     * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
     *
     * @throws Exception
     */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 20;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), unAckedMessagesBufferSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 10 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testShouldNotBlockConsumerIfRedeliverBeforeReceive.

@Test
public void testShouldNotBlockConsumerIfRedeliverBeforeReceive() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    int totalReceiveMsg = 0;
    try {
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 100;
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setAckTimeout(1, TimeUnit.SECONDS);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) wait for consumer to receive messages
        Thread.sleep(200);
        assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
        // (3) wait for messages to expire, we should've received more
        Thread.sleep(2000);
        assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
        for (int i = 0; i < totalProducedMsgs; i++) {
            Message msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                consumer.acknowledge(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs, totalReceiveMsg);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}