Examples with PulsarService com.yahoo.pulsar.broker.PulsarService used on opensource projects

Example 11 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class PulsarStandaloneStarter method start.

void start() throws Exception {
    if (config == null) {
        System.exit(1);
    }
    log.debug("--- setup PulsarStandaloneStarter ---");
    if (!onlyBroker) {
        // Start LocalBookKeeper
        bkEnsemble = new LocalBookkeeperEnsemble(numOfBk, zkPort, bkPort, zkDir, bkDir, wipeData);
        bkEnsemble.start();
    }
    if (noBroker) {
        return;
    }
    // Start Broker
    broker = new PulsarService(config);
    broker.start();
    // Create a sample namespace
    URL webServiceUrl = new URL(String.format("http://%s:%d", config.getAdvertisedAddress(), config.getWebServicePort()));
    String brokerServiceUrl = String.format("pulsar://%s:%d", config.getAdvertisedAddress(), config.getBrokerServicePort());
    admin = new PulsarAdmin(webServiceUrl, config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters());
    String property = "sample";
    String cluster = config.getClusterName();
    String namespace = property + "/" + cluster + "/ns1";
    try {
        ClusterData clusterData = new ClusterData(webServiceUrl.toString(), null, /* serviceUrlTls */
        brokerServiceUrl, null);
        if (!admin.clusters().getClusters().contains(cluster)) {
            admin.clusters().createCluster(cluster, clusterData);
        } else {
            admin.clusters().updateCluster(cluster, clusterData);
        }
        if (!admin.properties().getProperties().contains(property)) {
            admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList(config.getSuperUserRoles()), Sets.newHashSet(cluster)));
        }
        if (!admin.namespaces().getNamespaces(property).contains(namespace)) {
            admin.namespaces().createNamespace(namespace);
        }
    } catch (PulsarAdminException e) {
        log.info(e.getMessage());
    }
    log.debug("--- setup completed ---");
}

Example 12 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class PersistentDispatcherFailoverConsumerTest method setup.

@BeforeMethod
public void setup() throws Exception {
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    PulsarService pulsar = spy(new PulsarService(svcConfig));
    doReturn(svcConfig).when(pulsar).getConfiguration();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = mock(ZooKeeper.class);
    doReturn(mockZk).when(pulsar).getZkClient();
    configCacheService = mock(ConfigurationCacheService.class);
    @SuppressWarnings("unchecked") ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    doReturn(configCacheService).when(pulsar).getConfigurationCache();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    serverCnx = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnx).isActive();
    doReturn(true).when(serverCnx).isWritable();
    doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress();
    NamespaceService nsSvc = mock(NamespaceService.class);
    doReturn(nsSvc).when(pulsar).getNamespaceService();
    doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(nsSvc).isServiceUnitActive(any(DestinationName.class));
    setupMLAsyncCallbackMocks();
}

Example 13 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class AdvertisedAddressTest method setup.

@Before
public void setup() throws Exception {
    bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, 5001);
    bkEnsemble.start();
    ServiceConfiguration config = new ServiceConfiguration();
    config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
    config.setWebServicePort(BROKER_WEBSERVICE_PORT);
    config.setClusterName("usc");
    config.setBrokerServicePort(BROKER_SERVICE_PORT);
    config.setAdvertisedAddress(advertisedAddress);
    config.setManagedLedgerMaxEntriesPerLedger(5);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsar = new PulsarService(config);
    pulsar.start();
}

Example 14 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class DestinationLookup method lookupDestinationAsync.

/**
     * 
     * Lookup broker-service address for a given namespace-bundle which contains given topic.
     * 
     * a. Returns broker-address if namespace-bundle is already owned by any broker
     * b. If current-broker receives lookup-request and if it's not a leader
     * then current broker redirects request to leader by returning leader-service address. 
     * c. If current-broker is leader then it finds out least-loaded broker to own namespace bundle and 
     * redirects request by returning least-loaded broker.
     * d. If current-broker receives request to own the namespace-bundle then it owns a bundle and returns 
     * success(connect) response to client.
     * 
     * @param pulsarService
     * @param fqdn
     * @param authoritative
     * @param clientAppId
     * @param requestId
     * @return
     */
public static CompletableFuture<ByteBuf> lookupDestinationAsync(PulsarService pulsarService, DestinationName fqdn, boolean authoritative, String clientAppId, long requestId) {
    final CompletableFuture<ByteBuf> validationFuture = new CompletableFuture<>();
    final CompletableFuture<ByteBuf> lookupfuture = new CompletableFuture<>();
    final String cluster = fqdn.getCluster();
    // (1) validate cluster
    getClusterDataIfDifferentCluster(pulsarService, cluster, clientAppId).thenAccept(differentClusterData -> {
        if (differentClusterData != null) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Redirecting the lookup call to {}/{} cluster={}", clientAppId, differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), cluster);
            }
            validationFuture.complete(newLookupResponse(differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId));
        } else {
            try {
                checkAuthorization(pulsarService, fqdn, clientAppId);
            } catch (RestException authException) {
                log.warn("Failed to authorized {} on cluster {}", clientAppId, fqdn.toString());
                validationFuture.complete(newLookupResponse(ServerError.AuthorizationError, authException.getMessage(), requestId));
                return;
            } catch (Exception e) {
                log.warn("Unknown error while authorizing {} on cluster {}", clientAppId, fqdn.toString());
                validationFuture.completeExceptionally(e);
                return;
            }
            validateReplicationSettingsOnNamespaceAsync(pulsarService, fqdn.getNamespaceObject()).thenAccept(success -> {
                validationFuture.complete(null);
            }).exceptionally(ex -> {
                validationFuture.complete(newLookupResponse(ServerError.MetadataError, ex.getMessage(), requestId));
                return null;
            });
        }
    }).exceptionally(ex -> {
        validationFuture.completeExceptionally(ex);
        return null;
    });
    // Initiate lookup once validation completes
    validationFuture.thenAccept(validaitonFailureResponse -> {
        if (validaitonFailureResponse != null) {
            lookupfuture.complete(validaitonFailureResponse);
        } else {
            pulsarService.getNamespaceService().getBrokerServiceUrlAsync(fqdn, authoritative).thenAccept(lookupResult -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Lookup result {}", fqdn.toString(), lookupResult);
                }
                LookupData lookupData = lookupResult.getLookupData();
                if (lookupResult.isRedirect()) {
                    boolean newAuthoritative = isLeaderBroker(pulsarService);
                    lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), newAuthoritative, LookupType.Redirect, requestId));
                } else {
                    lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), true, LookupType.Connect, requestId));
                }
            }).exceptionally(e -> {
                log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, fqdn.toString(), e.getMessage(), e);
                lookupfuture.complete(newLookupResponse(ServerError.ServiceNotReady, e.getMessage(), requestId));
                return null;
            });
        }
    }).exceptionally(ex -> {
        log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, fqdn.toString(), ex.getMessage(), ex);
        lookupfuture.complete(newLookupResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId));
        return null;
    });
    return lookupfuture;
}

Example 15 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class BrokerServiceLookupTest method testWebserviceServiceTls.

/**
     * 1. Start broker1 and broker2 with tls enable
     * 2. Hit HTTPS lookup url at broker2 which redirects to HTTPS broker1  
     * 
     * @throws Exception
     */
@Test
public void testWebserviceServiceTls() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
    final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
    final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
    final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setTlsAllowInsecureConnection(true);
    conf2.setTlsEnabled(true);
    conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf2.setClusterName(conf.getClusterName());
    PulsarService pulsar2 = startBroker(conf2);
    // restart broker1 with tls enabled
    conf.setTlsAllowInsecureConnection(true);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    stopBroker();
    startBroker();
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), loadManager2);
    // mock: return Broker2 as a Least-loaded broker when leader receies
    // request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(resourceUnit).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), loadManager1);
    /**** started broker-2 ****/
    URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
    final String lookupResourceUrl = "/lookup/v2/destination/persistent/my-property/use/my-ns/my-topic1";
    // set client cert_key file 
    KeyManager[] keyManagers = null;
    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
    KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
    ks.load(null, null);
    ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
    KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
    kmf.init(ks, "".toCharArray());
    keyManagers = kmf.getKeyManagers();
    TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
    SSLContext sslCtx = SSLContext.getInstance("TLS");
    sslCtx.init(keyManagers, trustManagers, new SecureRandom());
    HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());
    // hit broker2 url
    URLConnection con = new URL(pulsar2.getWebServiceAddressTls() + lookupResourceUrl).openConnection();
    log.info("orignal url: {}", con.getURL());
    con.connect();
    log.info("connected url: {} ", con.getURL());
    // assert connect-url: broker2-https
    Assert.assertEquals(con.getURL().getPort(), conf2.getWebServicePortTls());
    InputStream is = con.getInputStream();
    // assert redirect-url: broker1-https only
    log.info("redirected url: {}", con.getURL());
    Assert.assertEquals(con.getURL().getPort(), conf.getWebServicePortTls());
    is.close();
    pulsarClient2.close();
    pulsar2.close();
    loadManager1 = null;
    loadManager2 = null;
}