Examples with PulsarClient com.yahoo.pulsar.client.api.PulsarClient used on opensource projects

Example 26 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class AdminApiTest method testPersistentTopicsExpireMessages.

/**
     * Verify: PersistentTopics.expireMessages()/expireMessagesForAllSubscriptions()
     * 1. Created multiple shared subscriptions and publisher on topic
     * 2. Publish messages on the topic
     * 3. expire message on sub-1 : backlog for sub-1 must be 0
     * 4. expire message on all subscriptions: backlog for all subscription must be 0
     *
     * @throws Exception
     */
@Test
public void testPersistentTopicsExpireMessages() throws Exception {
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2"));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub1", conf);
    Consumer consumer2 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub2", conf);
    Consumer consumer3 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub3", conf);
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2").size(), 3);
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    // wait for 1 seconds to expire message
    Thread.sleep(1000);
    admin.persistentTopics().expireMessages("persistent://prop-xyz/use/ns1/ds2", "my-sub1", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    admin.persistentTopics().expireMessagesForAllSubscriptions("persistent://prop-xyz/use/ns1/ds2", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 0);
    consumer1.close();
    consumer2.close();
    consumer3.close();
}

Example 27 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class PersistentTopicTest method testAtomicReplicationRemoval.

/**
     * {@link PersistentReplicator.removeReplicator} doesn't remove replicator in atomic way and does in multiple step:
     * 1. disconnect replicator producer
     * <p>
     * 2. close cursor
     * <p>
     * 3. remove from replicator-list.
     * <p>
     * 
     * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer.
     * 
     * @throws Exception
     */
@Test
public void testAtomicReplicationRemoval() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns-abc/successTopic";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + remoteCluster;
    ConcurrentOpenHashMap<String, PersistentReplicator> replicatorMap = topic.getReplicators();
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = PulsarClient.create(brokerUrl.toString());
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = spy(new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService));
    replicatorMap.put(remoteReplicatorName, replicator);
    // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed
    Method removeMethod = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeMethod.setAccessible(true);
    removeMethod.invoke(topic, remoteReplicatorName);
    // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the
    // replicator
    when(pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", DestinationName.get(globalTopicName).getNamespace()))).thenReturn(Optional.of(new Policies()));
    // try to start replicator again
    topic.startReplProducers();
    // verify: replicator.startProducer is not invoked
    verify(replicator, Mockito.times(0)).startProducer();
    // step-3 : complete the callback to remove replicator from the list
    ArgumentCaptor<DeleteCursorCallback> captor = ArgumentCaptor.forClass(DeleteCursorCallback.class);
    Mockito.verify(ledgerMock).asyncDeleteCursor(anyObject(), captor.capture(), anyObject());
    DeleteCursorCallback callback = captor.getValue();
    callback.deleteCursorComplete(null);
}

Example 28 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class PersistentTopicTest method testClosingReplicationProducerTwice.

@Test
public void testClosingReplicationProducerTwice() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns/testClosingReplicationProducerTwice";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + localCluster;
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = spy(PulsarClient.create(brokerUrl.toString()));
    PulsarClientImpl clientImpl = (PulsarClientImpl) client;
    Field conf = PersistentReplicator.class.getDeclaredField("producerConfiguration");
    conf.setAccessible(true);
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService);
    doReturn(new CompletableFuture<Producer>()).when(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
    replicator.startProducer();
    verify(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
    replicator.disconnect(false);
    replicator.disconnect(false);
    replicator.startProducer();
    verify(clientImpl, Mockito.times(2)).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
}

Example 29 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BrokerServiceTest method testTlsEnabled.

@Test
public void testTlsEnabled() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/newTopic";
    final String subName = "newSub";
    ClientConfiguration clientConfig;
    ConsumerConfiguration consumerConfig;
    Consumer consumer;
    conf.setAuthenticationEnabled(false);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    restartBroker();
    // Case 1: Access without TLS
    PulsarClient pulsarClient = null;
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrl.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
    // Case 2: Access with TLS (Allow insecure TLS connection)
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
    // Case 3: Access with TLS (Disallow insecure TLS connection)
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(false);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("General SSLEngine problem"));
    } finally {
        pulsarClient.close();
    }
    // Case 4: Access with TLS (Use trusted certificates)
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(false);
        clientConfig.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
}

Example 30 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BrokerServiceTest method testTlsAuthUseTrustCert.

@Test
public void testTlsAuthUseTrustCert() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    final String subName = "newSub";
    ClientConfiguration clientConfig;
    ConsumerConfiguration consumerConfig;
    Consumer consumer;
    Authentication auth;
    Set<String> providers = new HashSet<>();
    providers.add("com.yahoo.pulsar.broker.authentication.AuthenticationProviderTls");
    conf.setAuthenticationEnabled(true);
    conf.setAuthenticationProviders(providers);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf.setTlsAllowInsecureConnection(false);
    conf.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
    restartBroker();
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    PulsarClient pulsarClient = null;
    // Case 1: Access without client certificate
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("Authentication required"));
    } finally {
        pulsarClient.close();
    }
    // Case 2: Access with client certificate
    try {
        auth = new AuthenticationTls();
        auth.configure(authParams);
        clientConfig = new ClientConfiguration();
        clientConfig.setAuthentication(auth);
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
}