Examples with PulsarClient com.yahoo.pulsar.client.api.PulsarClient used on opensource projects

Example 41 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBroker.

/**
     * Verifies: Broker side throttling:
     * 
     * <pre>
     * 1. concurrent_consumer_creation > maxConcurrentLookupRequest at broker 
     * 2. few of the consumer creation must fail with TooManyLookupRequestException.
     * </pre>
     * 
     * @throws Exception
     */
@Test
public void testLookupThrottlingForClientByBroker() throws Exception {
    // create configuration znode
    ZkUtils.createFullPathOptimistic(mockZookKeeper, BROKER_SERVICE_CONFIGURATION_PATH, "{}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Now, znode is created: set the watch and listener on the znode
    setWatchOnThrottlingZnode();
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    com.yahoo.pulsar.client.api.ClientConfiguration clientConf = new com.yahoo.pulsar.client.api.ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    clientConf.setIoThreads(20);
    clientConf.setConnectionsPerBroker(20);
    String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
    PulsarClient pulsarClient = PulsarClient.create(lookupUrl, clientConf);
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    int newPermits = 1;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }
    List<Consumer> successfulConsumers = Lists.newArrayList();
    ExecutorService executor = Executors.newFixedThreadPool(10);
    final int totalConsumers = 20;
    CountDownLatch latch = new CountDownLatch(totalConsumers);
    for (int i = 0; i < totalConsumers; i++) {
        executor.execute(() -> {
            try {
                successfulConsumers.add(pulsarClient.subscribe(topicName, "mysub", consumerConfig));
            } catch (PulsarClientException.TooManyLookupRequestException e) {
            // ok
            } catch (Exception e) {
                fail("it shouldn't failed");
            }
            latch.countDown();
        });
    }
    latch.await();
    for (int i = 0; i < successfulConsumers.size(); i++) {
        successfulConsumers.get(i).close();
    }
    pulsarClient.close();
    assertNotEquals(successfulConsumers.size(), totalConsumers);
}

Example 42 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerQueueFullNonBlocking.

@Test
public void testProducerQueueFullNonBlocking() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyzx";
    final int messages = 10;
    // 1. Producer connect
    PulsarClient client = PulsarClient.create(brokerUrl.toString());
    ProducerConfiguration producerConfiguration = new ProducerConfiguration().setMaxPendingMessages(messages).setBlockIfQueueFull(false).setSendTimeout(1, TimeUnit.SECONDS);
    ProducerImpl producer = (ProducerImpl) client.createProducer(topicName, producerConfiguration);
    // 2. Stop broker
    cleanup();
    // 2. producer publish messages
    long startTime = System.nanoTime();
    for (int i = 0; i < messages; i++) {
        // Should never block
        producer.sendAsync("msg".getBytes());
    }
    // Verify thread was not blocked
    long delayNs = System.nanoTime() - startTime;
    assertTrue(delayNs < TimeUnit.SECONDS.toNanos(1));
    assertEquals(producer.getPendingQueueSize(), messages);
    // Next send operation must fail and not block
    startTime = System.nanoTime();
    try {
        producer.send("msg".getBytes());
        fail("Send should have failed");
    } catch (PulsarClientException.ProducerQueueIsFullError e) {
    // Expected
    }
    delayNs = System.nanoTime() - startTime;
    assertTrue(delayNs < TimeUnit.SECONDS.toNanos(1));
    assertEquals(producer.getPendingQueueSize(), messages);
    // 4. producer disconnect
    producer.close();
    client.close();
    // 5. Restart broker
    setup();
}

Example 43 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testNoEviction.

@Test
public void testNoEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic13";
    final String subName1 = "c13";
    final String subName2 = "c23";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final Consumer consumer1 = client.subscribe(topic1, subName1);
    final Consumer consumer2 = client.subscribe(topic1, subName2);
    final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
    Thread producerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                com.yahoo.pulsar.client.api.Producer producer = client2.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs; i++) {
                    consumer1.acknowledge(consumer1.receive());
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread.start();
    ConsumerThread.start();
    counter.await();
    assertTrue(!gotException.get());
    client.close();
    client2.close();
}

Example 44 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testAheadProducerOnHold.

@Test
public void testAheadProducerOnHold() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/hold";
    final String subName1 = "c1hold";
    final int numMsgs = 10;
    Consumer consumer = client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i <= numMsgs; i++) {
        try {
            producer.send(content);
            LOG.info("sent [{}]", i);
        } catch (PulsarClientException.TimeoutException cte) {
            // producer close may cause a timeout on send
            LOG.info("timeout on [{}]", i);
        }
    }
    for (int i = 0; i < numMsgs; i++) {
        consumer.receive();
        LOG.info("received [{}]", i);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertEquals(stats.publishers.size(), 0, "Number of producers on topic " + topic1 + " are [" + stats.publishers.size() + "]");
    client.close();
}

Example 45 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testProducerException.

@Test
public void testProducerException() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/except";
    final String subName1 = "c1except";
    boolean gotException = false;
    client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException ce) {
        Assert.assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException || ce instanceof PulsarClientException.TimeoutException, ce.getMessage());
        gotException = true;
    }
    Assert.assertTrue(gotException, "backlog exceeded exception did not occur");
    client.close();
}