Examples with PulsarClient org.apache.pulsar.client.api.PulsarClient used on opensource projects

Example 36 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class BrokerBkEnsemblesTests method testSkipCorruptDataLedger.

/**
 * It verifies broker-configuration using which broker can skip non-recoverable data-ledgers.
 *
 * <pre>
 * 1. publish messages in 5 data-ledgers each with 20 entries under managed-ledger
 * 2. delete first 4 data-ledgers
 * 3. consumer will fail to consume any message as first data-ledger is non-recoverable
 * 4. enable dynamic config to skip non-recoverable data-ledgers
 * 5. consumer will be able to consume 20 messages from last non-deleted ledger
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 6000)
public void testSkipCorruptDataLedger() throws Exception {
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String ns1 = "prop/usc/crash-broker";
    final int totalMessages = 100;
    final int totalDataLedgers = 5;
    final int entriesPerLedger = totalMessages / totalDataLedgers;
    admin.namespaces().createNamespace(ns1);
    final String topic1 = "persistent://" + ns1 + "/my-topic";
    // Create subscription
    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").receiverQueueSize(5).subscribe();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic1).get();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger();
    ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next();
    Field configField = ManagedCursorImpl.class.getDeclaredField("config");
    configField.setAccessible(true);
    // Create multiple data-ledger
    ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor);
    config.setMaxEntriesPerLedger(entriesPerLedger);
    config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
    // bookkeeper client
    Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper");
    bookKeeperField.setAccessible(true);
    // Create multiple data-ledger
    BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml);
    // (1) publish messages in 5 data-ledgers each with 20 entries under managed-ledger
    Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // validate: consumer is able to consume msg and close consumer after reading 1 entry
    Assert.assertNotNull(consumer.receive(1, TimeUnit.SECONDS));
    consumer.close();
    NavigableMap<Long, LedgerInfo> ledgerInfo = ml.getLedgersInfo();
    Assert.assertEquals(ledgerInfo.size(), totalDataLedgers);
    Entry<Long, LedgerInfo> lastLedger = ledgerInfo.lastEntry();
    // (2) delete first 4 data-ledgers
    ledgerInfo.entrySet().forEach(entry -> {
        if (!entry.equals(lastLedger)) {
            try {
                bookKeeper.deleteLedger(entry.getKey());
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    });
    // clean managed-ledger and recreate topic to clean any data from the cache
    producer.close();
    pulsar.getBrokerService().removeTopicFromCache(topic1);
    ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    @SuppressWarnings("unchecked") ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(factory);
    ledgers.clear();
    // (3) consumer will fail to consume any message as first data-ledger is non-recoverable
    Message<byte[]> msg = null;
    // start consuming message
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    msg = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(msg);
    consumer.close();
    // (4) enable dynamic config to skip non-recoverable data-ledgers
    admin.brokers().updateDynamicConfiguration("autoSkipNonRecoverableData", "true");
    retryStrategically((test) -> config.isAutoSkipNonRecoverableData(), 5, 100);
    // (5) consumer will be able to consume 20 messages from last non-deleted ledger
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    for (int i = 0; i < entriesPerLedger; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        System.out.println(i);
        consumer.acknowledge(msg);
    }
    producer.close();
    consumer.close();
    client.close();
}

Example 37 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class BrokerServiceTest method testTlsAuthDisallowInsecure.

@SuppressWarnings("deprecation")
@Test
public void testTlsAuthDisallowInsecure() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    final String subName = "newSub";
    Authentication auth;
    Set<String> providers = new HashSet<>();
    providers.add("org.apache.pulsar.broker.authentication.AuthenticationProviderTls");
    conf.setAuthenticationEnabled(true);
    conf.setAuthenticationProviders(providers);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf.setTlsAllowInsecureConnection(false);
    restartBroker();
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    PulsarClient pulsarClient = null;
    // Case 1: Access without client certificate
    try {
        pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
        @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("Authentication required"));
    } finally {
        pulsarClient.close();
    }
    // Case 2: Access with client certificate
    try {
        auth = new AuthenticationTls();
        auth.configure(authParams);
        pulsarClient = PulsarClient.builder().authentication(auth).serviceUrl(brokerUrlTls.toString()).enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
        @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("Authentication required"));
    } finally {
        pulsarClient.close();
    }
}

Example 38 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class BrokerServiceTest method testTlsAuthAllowInsecure.

@SuppressWarnings("deprecation")
@Test
public void testTlsAuthAllowInsecure() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    final String subName = "newSub";
    Authentication auth;
    Set<String> providers = new HashSet<>();
    providers.add("org.apache.pulsar.broker.authentication.AuthenticationProviderTls");
    conf.setAuthenticationEnabled(true);
    conf.setAuthenticationProviders(providers);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf.setTlsAllowInsecureConnection(true);
    restartBroker();
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    PulsarClient pulsarClient = null;
    // Case 1: Access without client certificate
    try {
        pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
        @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("Authentication required"));
    } finally {
        pulsarClient.close();
    }
    // Case 2: Access with client certificate
    try {
        auth = new AuthenticationTls();
        auth.configure(authParams);
        pulsarClient = PulsarClient.builder().authentication(auth).serviceUrl(brokerUrlTls.toString()).enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
        @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
}

Example 39 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class BrokerServiceTest method testTlsAuthUseTrustCert.

@SuppressWarnings("deprecation")
@Test
public void testTlsAuthUseTrustCert() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    final String subName = "newSub";
    Authentication auth;
    Set<String> providers = new HashSet<>();
    providers.add("org.apache.pulsar.broker.authentication.AuthenticationProviderTls");
    conf.setAuthenticationEnabled(true);
    conf.setAuthenticationProviders(providers);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf.setTlsAllowInsecureConnection(false);
    conf.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
    restartBroker();
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    PulsarClient pulsarClient = null;
    // Case 1: Access without client certificate
    try {
        pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
        @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("Authentication required"));
    } finally {
        pulsarClient.close();
    }
    // Case 2: Access with client certificate
    try {
        auth = new AuthenticationTls();
        auth.configure(authParams);
        pulsarClient = PulsarClient.builder().authentication(auth).serviceUrl(brokerUrlTls.toString()).enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
        @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
}

Example 40 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBroker.

/**
 * Verifies: Broker side throttling:
 *
 * <pre>
 * 1. concurrent_consumer_creation > maxConcurrentLookupRequest at broker
 * 2. few of the consumer creation must fail with TooManyLookupRequestException.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBroker() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS).ioThreads(20).connectionsPerBroker(20).build();
    int newPermits = 1;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }
    List<Consumer<byte[]>> successfulConsumers = Collections.synchronizedList(Lists.newArrayList());
    ExecutorService executor = Executors.newFixedThreadPool(10);
    final int totalConsumers = 20;
    CountDownLatch latch = new CountDownLatch(totalConsumers);
    for (int i = 0; i < totalConsumers; i++) {
        executor.execute(() -> {
            try {
                successfulConsumers.add(pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscriptionType(SubscriptionType.Shared).subscribe());
            } catch (PulsarClientException.TooManyRequestsException e) {
            // ok
            } catch (Exception e) {
                fail("it shouldn't failed");
            }
            latch.countDown();
        });
    }
    latch.await();
    for (Consumer<?> c : successfulConsumers) {
        if (c != null) {
            c.close();
        }
    }
    pulsarClient.close();
    executor.shutdown();
    assertNotEquals(successfulConsumers.size(), totalConsumers);
}