Examples with PulsarClient org.apache.pulsar.client.api.PulsarClient used on opensource projects

Example 41 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBroker0Permit.

/**
 * Broker has maxConcurrentLookupRequest = 0 so, it rejects incoming lookup request and it cause consumer creation
 * failure.
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBroker0Permit() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
    consumer.close();
    int newPermits = 0;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }
    try {
        consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
        consumer.close();
        fail("It should fail as throttling should not receive any request");
    } catch (org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException e) {
    // ok as throttling set to 0
    }
}

Example 42 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class ZooKeeperClientAspectJTest method testZkOpStatsMetrics.

/**
 * Verifies that aspect-advice calculates the latency of of zk-operation and updates PulsarStats
 *
 * @throws Exception
 */
@Test(enabled = false, timeOut = 7000)
public void testZkOpStatsMetrics() throws Exception {
    OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build();
    ZooKeeperClientFactory zkf = new ZookeeperBkClientFactoryImpl(executor);
    CompletableFuture<ZooKeeper> zkFuture = zkf.create("127.0.0.1:" + LOCAL_ZOOKEEPER_PORT, SessionType.ReadWrite, (int) ZOOKEEPER_SESSION_TIMEOUT_MILLIS);
    localZkc = zkFuture.get(ZOOKEEPER_SESSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    MockPulsar mockPulsar = new MockPulsar(localZkc);
    mockPulsar.setup();
    try {
        PulsarClient pulsarClient = mockPulsar.getClient();
        PulsarService pulsar = mockPulsar.getPulsar();
        pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
        Metrics zkOpMetric = getMetric(pulsar, "zk_write_latency");
        Assert.assertNotNull(zkOpMetric);
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_rate_s"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_95percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_9_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_mean_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_median_ms"));
        zkOpMetric = getMetric(pulsar, "zk_read_latency");
        Assert.assertNotNull(zkOpMetric);
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_rate_s"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_95percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_9_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_mean_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_median_ms"));
        CountDownLatch createLatch = new CountDownLatch(1);
        CountDownLatch deleteLatch = new CountDownLatch(1);
        CountDownLatch readLatch = new CountDownLatch(1);
        CountDownLatch existLatch = new CountDownLatch(1);
        localZkc.create("/createTest", "data".getBytes(), Acl, CreateMode.EPHEMERAL, (rc, path, ctx, name) -> {
            createLatch.countDown();
        }, "create");
        localZkc.delete("/deleteTest", -1, (rc, path, ctx) -> {
            deleteLatch.countDown();
        }, "delete");
        localZkc.exists("/createTest", null, (int rc, String path, Object ctx, Stat stat) -> {
            existLatch.countDown();
        }, null);
        localZkc.getData("/createTest", null, (int rc, String path, Object ctx, byte[] data, Stat stat) -> {
            readLatch.countDown();
        }, null);
        createLatch.await();
        deleteLatch.await();
        existLatch.await();
        readLatch.await();
        Thread.sleep(10);
        BrokerService brokerService = pulsar.getBrokerService();
        brokerService.updateRates();
        List<Metrics> metrics = brokerService.getTopicMetrics();
        AtomicDouble writeRate = new AtomicDouble();
        AtomicDouble readRate = new AtomicDouble();
        metrics.forEach(m -> {
            if ("zk_write_latency".equalsIgnoreCase(m.getDimension("metric"))) {
                writeRate.set((double) m.getMetrics().get("brk_zk_write_latency_rate_s"));
            } else if ("zk_read_latency".equalsIgnoreCase(m.getDimension("metric"))) {
                readRate.set((double) m.getMetrics().get("brk_zk_read_latency_rate_s"));
            }
        });
        Assert.assertTrue(readRate.get() > 0);
        Assert.assertTrue(writeRate.get() > 0);
    } finally {
        mockPulsar.cleanup();
        if (localZkc != null) {
            localZkc.close();
        }
        executor.shutdown();
    }
}

Example 43 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class AdminApiTest method persistentTopics.

@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // Force to create a topic
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 0);
    PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
    assertEquals(messages.size(), 3);
    for (int i = 0; i < 3; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
    admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
    assertEquals(topicStats.publishers.size(), 0);
    try {
        admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    } catch (NotFoundException e) {
    }
    admin.persistentTopics().delete(persistentTopicName);
    try {
        admin.persistentTopics().delete(persistentTopicName);
        fail("Should have received 404");
    } catch (NotFoundException e) {
    }
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}

Example 44 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class CompactorTool method main.

public static void main(String[] args) throws Exception {
    Arguments arguments = new Arguments();
    JCommander jcommander = new JCommander(arguments);
    jcommander.setProgramName("PulsarTopicCompactor");
    // parse args by JCommander
    jcommander.parse(args);
    if (arguments.help) {
        jcommander.usage();
        System.exit(-1);
    }
    // init broker config
    ServiceConfiguration brokerConfig;
    if (isBlank(arguments.brokerConfigFile)) {
        jcommander.usage();
        throw new IllegalArgumentException("Need to specify a configuration file for broker");
    } else {
        brokerConfig = PulsarConfigurationLoader.create(arguments.brokerConfigFile, ServiceConfiguration.class);
    }
    String pulsarServiceUrl = PulsarService.brokerUrl(brokerConfig);
    ClientConfiguration clientConfig = new ClientConfiguration();
    if (isNotBlank(brokerConfig.getBrokerClientAuthenticationPlugin())) {
        clientConfig.setAuthentication(brokerConfig.getBrokerClientAuthenticationPlugin(), brokerConfig.getBrokerClientAuthenticationParameters());
    }
    clientConfig.setUseTls(brokerConfig.isTlsEnabled());
    clientConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection());
    clientConfig.setTlsTrustCertsFilePath(brokerConfig.getTlsCertificateFilePath());
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("compaction-%d").setDaemon(true).build());
    OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build();
    ZooKeeperClientFactory zkClientFactory = new ZookeeperBkClientFactoryImpl(executor);
    ZooKeeper zk = zkClientFactory.create(brokerConfig.getZookeeperServers(), ZooKeeperClientFactory.SessionType.ReadWrite, (int) brokerConfig.getZooKeeperSessionTimeoutMillis()).get();
    BookKeeperClientFactory bkClientFactory = new BookKeeperClientFactoryImpl();
    BookKeeper bk = bkClientFactory.create(brokerConfig, zk);
    try (PulsarClient pulsar = PulsarClient.create(pulsarServiceUrl, clientConfig)) {
        Compactor compactor = new TwoPhaseCompactor(brokerConfig, pulsar, bk, scheduler);
        long ledgerId = compactor.compact(arguments.topic).get();
        log.info("Compaction of topic {} complete. Compacted to ledger {}", arguments.topic, ledgerId);
    } finally {
        bk.close();
        bkClientFactory.close();
        zk.close();
        scheduler.shutdownNow();
        executor.shutdown();
    }
}

Example 45 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class ProxyAuthenticatedProducerConsumerTest method testTlsSyncProducerAndConsumer.

/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings("deprecation")
@Test
public void testTlsSyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String proxyServiceUrl = "pulsar://localhost:" + proxyConfig.getServicePortTls();
    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyServiceUrl);
    admin.clusters().createCluster(configClusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
    Consumer<byte[]> consumer = proxyClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1").subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}