Examples with PulsarService org.apache.pulsar.broker.PulsarService used on opensource projects

Example 21 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testWebserviceServiceTls.

/**
 * 1. Start broker1 and broker2 with tls enable 2. Hit HTTPS lookup url at broker2 which redirects to HTTPS broker1
 *
 * @throws Exception
 */
@Test
public void testWebserviceServiceTls() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
    final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
    final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
    final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
    /**
     ** start broker-2 ***
     */
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setTlsAllowInsecureConnection(true);
    conf2.setTlsEnabled(true);
    conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");
    PulsarService pulsar2 = startBroker(conf2);
    // restart broker1 with tls enabled
    conf.setTlsAllowInsecureConnection(true);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    stopBroker();
    startBroker();
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    // mock: return Broker2 as a Least-loaded broker when leader receies
    // request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
    /**
     ** started broker-2 ***
     */
    URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
    final String lookupResourceUrl = "/lookup/v2/destination/persistent/my-property/use/my-ns/my-topic1";
    // set client cert_key file
    KeyManager[] keyManagers = null;
    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
    KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
    ks.load(null, null);
    ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
    KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
    kmf.init(ks, "".toCharArray());
    keyManagers = kmf.getKeyManagers();
    TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
    SSLContext sslCtx = SSLContext.getInstance("TLS");
    sslCtx.init(keyManagers, trustManagers, new SecureRandom());
    HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());
    // hit broker2 url
    URLConnection con = new URL(pulsar2.getWebServiceAddressTls() + lookupResourceUrl).openConnection();
    log.info("orignal url: {}", con.getURL());
    con.connect();
    log.info("connected url: {} ", con.getURL());
    // assert connect-url: broker2-https
    assertEquals(con.getURL().getPort(), conf2.getWebServicePortTls());
    InputStream is = con.getInputStream();
    // assert redirect-url: broker1-https only
    log.info("redirected url: {}", con.getURL());
    assertEquals(con.getURL().getPort(), conf.getWebServicePortTls());
    is.close();
    pulsarClient2.close();
    pulsar2.close();
    loadManager1 = null;
    loadManager2 = null;
}

Example 22 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class NamespaceBundleTest method getNamespaceBundleFactory.

@SuppressWarnings("unchecked")
private NamespaceBundleFactory getNamespaceBundleFactory() {
    PulsarService pulsar = mock(PulsarService.class);
    LocalZooKeeperCacheService localZkCache = mock(LocalZooKeeperCacheService.class);
    ZooKeeperDataCache<LocalPolicies> poilciesCache = mock(ZooKeeperDataCache.class);
    when(pulsar.getLocalZkCacheService()).thenReturn(localZkCache);
    when(localZkCache.policiesCache()).thenReturn(poilciesCache);
    doNothing().when(poilciesCache).registerListener(any());
    return NamespaceBundleFactory.createFactory(pulsar, Hashing.crc32());
}

Example 23 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testReplicator.

/**
 * verifies that non-persistent topic replicates using replicator
 */
@Test
public void testReplicator() throws Exception {
    ReplicationClusterManager replication = new ReplicationClusterManager();
    replication.setupReplicationCluster();
    try {
        final String globalTopicName = "non-persistent://pulsar/global/ns/nonPersistentTopic";
        final int timeWaitToSync = 100;
        NonPersistentTopicStats stats;
        SubscriptionStats subStats;
        PulsarClient client1 = PulsarClient.builder().serviceUrl(replication.url1.toString()).build();
        PulsarClient client2 = PulsarClient.builder().serviceUrl(replication.url2.toString()).build();
        PulsarClient client3 = PulsarClient.builder().serviceUrl(replication.url3.toString()).build();
        ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-2").subscribe();
        ConsumerImpl<byte[]> repl2Consumer = (ConsumerImpl<byte[]>) client2.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> repl3Consumer = (ConsumerImpl<byte[]>) client3.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        Producer<byte[]> producer = client1.newProducer().topic(globalTopicName).create();
        Thread.sleep(timeWaitToSync);
        PulsarService replicationPulasr = replication.pulsar1;
        // Replicator for r1 -> r2,r3
        NonPersistentTopic topicRef = (NonPersistentTopic) replication.pulsar1.getBrokerService().getTopicReference(globalTopicName);
        NonPersistentReplicator replicatorR2 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r2");
        NonPersistentReplicator replicatorR3 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r3");
        assertNotNull(topicRef);
        assertNotNull(replicatorR2);
        assertNotNull(replicatorR3);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        // subscription stats
        assertEquals(stats.getSubscriptions().keySet().size(), 2);
        assertEquals(subStats.consumers.size(), 1);
        Thread.sleep(timeWaitToSync);
        int totalProducedMessages = 100;
        for (int i = 0; i < totalProducedMessages; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (1) consume by consumer1
        Message<?> msg = null;
        Set<String> messageSet = Sets.newHashSet();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer1.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (2) consume by consumer2
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer2.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (3) consume by repl2consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl2Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (4) consume by repl3consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl3Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        Thread.sleep(timeWaitToSync);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        assertTrue(subStats.msgRateOut > 0);
        assertEquals(subStats.consumers.size(), 1);
        assertTrue(subStats.msgThroughputOut > 0);
        // consumer stats
        assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
        assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
        assertEquals(subStats.msgRateRedeliver, 0.0);
        producer.close();
        consumer1.close();
        repl2Consumer.close();
        repl3Consumer.close();
        client1.close();
        client2.close();
        client3.close();
    } finally {
        replication.shutdownReplicationCluster();
    }
}

Example 24 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class TopicLookup method lookupTopicAsync.

/**
 * Lookup broker-service address for a given namespace-bundle which contains given topic.
 *
 * a. Returns broker-address if namespace-bundle is already owned by any broker
 * b. If current-broker receives lookup-request and if it's not a leader
 * then current broker redirects request to leader by returning leader-service address.
 * c. If current-broker is leader then it finds out least-loaded broker to own namespace bundle and
 * redirects request by returning least-loaded broker.
 * d. If current-broker receives request to own the namespace-bundle then it owns a bundle and returns
 * success(connect) response to client.
 *
 * @param pulsarService
 * @param topicName
 * @param authoritative
 * @param clientAppId
 * @param requestId
 * @return
 */
public static CompletableFuture<ByteBuf> lookupTopicAsync(PulsarService pulsarService, TopicName topicName, boolean authoritative, String clientAppId, AuthenticationDataSource authenticationData, long requestId) {
    final CompletableFuture<ByteBuf> validationFuture = new CompletableFuture<>();
    final CompletableFuture<ByteBuf> lookupfuture = new CompletableFuture<>();
    final String cluster = topicName.getCluster();
    // (1) validate cluster
    getClusterDataIfDifferentCluster(pulsarService, cluster, clientAppId).thenAccept(differentClusterData -> {
        if (differentClusterData != null) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Redirecting the lookup call to {}/{} cluster={}", clientAppId, differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), cluster);
            }
            validationFuture.complete(newLookupResponse(differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId, false));
        } else {
            // (2) authorize client
            try {
                checkAuthorization(pulsarService, topicName, clientAppId, authenticationData);
            } catch (RestException authException) {
                log.warn("Failed to authorized {} on cluster {}", clientAppId, topicName.toString());
                validationFuture.complete(newLookupErrorResponse(ServerError.AuthorizationError, authException.getMessage(), requestId));
                return;
            } catch (Exception e) {
                log.warn("Unknown error while authorizing {} on cluster {}", clientAppId, topicName.toString());
                validationFuture.completeExceptionally(e);
                return;
            }
            // (3) validate global namespace
            checkLocalOrGetPeerReplicationCluster(pulsarService, topicName.getNamespaceObject()).thenAccept(peerClusterData -> {
                if (peerClusterData == null) {
                    // (4) all validation passed: initiate lookup
                    validationFuture.complete(null);
                    return;
                }
                // request should be redirect to the peer-cluster
                if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl()) && StringUtils.isBlank(peerClusterData.getBrokerServiceUrl())) {
                    validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, "Redirected cluster's brokerService url is not configured", requestId));
                    return;
                }
                validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(), peerClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId, false));
            }).exceptionally(ex -> {
                validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, ex.getMessage(), requestId));
                return null;
            });
        }
    }).exceptionally(ex -> {
        validationFuture.completeExceptionally(ex);
        return null;
    });
    // Initiate lookup once validation completes
    validationFuture.thenAccept(validaitonFailureResponse -> {
        if (validaitonFailureResponse != null) {
            lookupfuture.complete(validaitonFailureResponse);
        } else {
            pulsarService.getNamespaceService().getBrokerServiceUrlAsync(topicName, authoritative).thenAccept(lookupResult -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Lookup result {}", topicName.toString(), lookupResult);
                }
                if (!lookupResult.isPresent()) {
                    lookupfuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, "No broker was available to own " + topicName, requestId));
                    return;
                }
                LookupData lookupData = lookupResult.get().getLookupData();
                if (lookupResult.get().isRedirect()) {
                    boolean newAuthoritative = isLeaderBroker(pulsarService);
                    lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), newAuthoritative, LookupType.Redirect, requestId, false));
                } else {
                    // When running in standalone mode we want to redirect the client through the service
                    // url, so that the advertised address configuration is not relevant anymore.
                    boolean redirectThroughServiceUrl = pulsarService.getConfiguration().isRunningStandalone();
                    lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), true, /* authoritative */
                    LookupType.Connect, requestId, redirectThroughServiceUrl));
                }
            }).exceptionally(ex -> {
                if (ex instanceof CompletionException && ex.getCause() instanceof IllegalStateException) {
                    log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), ex.getCause().getMessage());
                } else {
                    log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), ex.getMessage(), ex);
                }
                lookupfuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId));
                return null;
            });
        }
    }).exceptionally(ex -> {
        if (ex instanceof CompletionException && ex.getCause() instanceof IllegalStateException) {
            log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), ex.getCause().getMessage());
        } else {
            log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), ex.getMessage(), ex);
        }
        lookupfuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId));
        return null;
    });
    return lookupfuture;
}

Example 25 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class AntiAffinityNamespaceGroupTest method setup.

@BeforeMethod
void setup() throws Exception {
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
    bkEnsemble.start();
    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(PRIMARY_BROKER_WEBSERVICE_PORT);
    config1.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
    config1.setBrokerServicePort(PRIMARY_BROKER_PORT);
    config1.setFailureDomainsEnabled(true);
    config1.setLoadBalancerEnabled(true);
    config1.setAdvertisedAddress("localhost");
    createCluster(bkEnsemble.getZkClient(), config1);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();
    primaryHost = String.format("%s:%d", "localhost", PRIMARY_BROKER_WEBSERVICE_PORT);
    url1 = new URL("http://127.0.0.1" + ":" + PRIMARY_BROKER_WEBSERVICE_PORT);
    admin1 = new PulsarAdmin(url1, (Authentication) null);
    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(SECONDARY_BROKER_WEBSERVICE_PORT);
    config2.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
    config2.setBrokerServicePort(SECONDARY_BROKER_PORT);
    config2.setFailureDomainsEnabled(true);
    pulsar2 = new PulsarService(config2);
    secondaryHost = String.format("%s:%d", "localhost", SECONDARY_BROKER_WEBSERVICE_PORT);
    pulsar2.start();
    url2 = new URL("http://127.0.0.1" + ":" + SECONDARY_BROKER_WEBSERVICE_PORT);
    admin2 = new PulsarAdmin(url2, (Authentication) null);
    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}