Examples with PulsarService org.apache.pulsar.broker.PulsarService used on opensource projects

Example 11 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class WebServiceTest method setupEnv.

private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowUnversionedClients, boolean enableTls, boolean enableAuth, boolean allowInsecure) throws Exception {
    Set<String> providers = new HashSet<>();
    providers.add("org.apache.pulsar.broker.authentication.AuthenticationProviderTls");
    Set<String> roles = new HashSet<>();
    roles.add("client");
    ServiceConfiguration config = new ServiceConfiguration();
    config.setAdvertisedAddress("localhost");
    config.setWebServicePort(BROKER_WEBSERVICE_PORT);
    config.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS);
    config.setClientLibraryVersionCheckEnabled(enableFilter);
    config.setAuthenticationEnabled(enableAuth);
    config.setAuthenticationProviders(providers);
    config.setAuthorizationEnabled(false);
    config.setSuperUserRoles(roles);
    config.setTlsEnabled(enableTls);
    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsAllowInsecureConnection(allowInsecure);
    config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
    config.setClusterName("local");
    // TLS certificate expects localhost
    config.setAdvertisedAddress("localhost");
    config.setZookeeperServers("localhost:2181");
    pulsar = spy(new PulsarService(config));
    doReturn(new MockedZooKeeperClientFactoryImpl()).when(pulsar).getZooKeeperClientFactory();
    doReturn(new MockedBookKeeperClientFactory()).when(pulsar).getBookKeeperClientFactory();
    pulsar.start();
    try {
        pulsar.getZkClient().delete("/minApiVersion", -1);
    } catch (Exception ex) {
    }
    pulsar.getZkClient().create("/minApiVersion", minApiVersion.getBytes(), null, CreateMode.PERSISTENT);
    String serviceUrl = BROKER_URL_BASE;
    ClientConfiguration clientConfig = new ClientConfiguration();
    if (enableTls && enableAuth) {
        serviceUrl = BROKER_URL_BASE_TLS;
        Map<String, String> authParams = new HashMap<>();
        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
        Authentication auth = new AuthenticationTls();
        auth.configure(authParams);
        clientConfig.setAuthentication(auth);
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
    }
    PulsarAdmin pulsarAdmin = new PulsarAdmin(new URL(serviceUrl), clientConfig);
    try {
        pulsarAdmin.clusters().createCluster(config.getClusterName(), new ClusterData(pulsar.getWebServiceAddress()));
    } catch (ConflictException ce) {
    // This is OK.
    } finally {
        pulsarAdmin.close();
    }
}

Example 12 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class ZooKeeperClientAspectJTest method testZkOpStatsMetrics.

/**
 * Verifies that aspect-advice calculates the latency of of zk-operation and updates PulsarStats
 *
 * @throws Exception
 */
@Test(enabled = false, timeOut = 7000)
public void testZkOpStatsMetrics() throws Exception {
    OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build();
    ZooKeeperClientFactory zkf = new ZookeeperBkClientFactoryImpl(executor);
    CompletableFuture<ZooKeeper> zkFuture = zkf.create("127.0.0.1:" + LOCAL_ZOOKEEPER_PORT, SessionType.ReadWrite, (int) ZOOKEEPER_SESSION_TIMEOUT_MILLIS);
    localZkc = zkFuture.get(ZOOKEEPER_SESSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    MockPulsar mockPulsar = new MockPulsar(localZkc);
    mockPulsar.setup();
    try {
        PulsarClient pulsarClient = mockPulsar.getClient();
        PulsarService pulsar = mockPulsar.getPulsar();
        pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
        Metrics zkOpMetric = getMetric(pulsar, "zk_write_latency");
        Assert.assertNotNull(zkOpMetric);
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_rate_s"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_95percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_9_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_mean_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_median_ms"));
        zkOpMetric = getMetric(pulsar, "zk_read_latency");
        Assert.assertNotNull(zkOpMetric);
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_rate_s"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_95percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_9_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_mean_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_median_ms"));
        CountDownLatch createLatch = new CountDownLatch(1);
        CountDownLatch deleteLatch = new CountDownLatch(1);
        CountDownLatch readLatch = new CountDownLatch(1);
        CountDownLatch existLatch = new CountDownLatch(1);
        localZkc.create("/createTest", "data".getBytes(), Acl, CreateMode.EPHEMERAL, (rc, path, ctx, name) -> {
            createLatch.countDown();
        }, "create");
        localZkc.delete("/deleteTest", -1, (rc, path, ctx) -> {
            deleteLatch.countDown();
        }, "delete");
        localZkc.exists("/createTest", null, (int rc, String path, Object ctx, Stat stat) -> {
            existLatch.countDown();
        }, null);
        localZkc.getData("/createTest", null, (int rc, String path, Object ctx, byte[] data, Stat stat) -> {
            readLatch.countDown();
        }, null);
        createLatch.await();
        deleteLatch.await();
        existLatch.await();
        readLatch.await();
        Thread.sleep(10);
        BrokerService brokerService = pulsar.getBrokerService();
        brokerService.updateRates();
        List<Metrics> metrics = brokerService.getTopicMetrics();
        AtomicDouble writeRate = new AtomicDouble();
        AtomicDouble readRate = new AtomicDouble();
        metrics.forEach(m -> {
            if ("zk_write_latency".equalsIgnoreCase(m.getDimension("metric"))) {
                writeRate.set((double) m.getMetrics().get("brk_zk_write_latency_rate_s"));
            } else if ("zk_read_latency".equalsIgnoreCase(m.getDimension("metric"))) {
                readRate.set((double) m.getMetrics().get("brk_zk_read_latency_rate_s"));
            }
        });
        Assert.assertTrue(readRate.get() > 0);
        Assert.assertTrue(writeRate.get() > 0);
    } finally {
        mockPulsar.cleanup();
        if (localZkc != null) {
            localZkc.close();
        }
        executor.shutdown();
    }
}

Example 13 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class SLAMonitoringTest method setup.

@BeforeClass
void setup() throws Exception {
    log.info("---- Initializing SLAMonitoringTest -----");
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
    bkEnsemble.start();
    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {
        brokerWebServicePorts[i] = PortManager.nextFreePort();
        brokerNativeBrokerPorts[i] = PortManager.nextFreePort();
        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
        config.setClusterName("my-cluster");
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(brokerWebServicePorts[i]);
        config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
        config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
        config.setDefaultNumberOfNamespaceBundles(1);
        config.setLoadBalancerEnabled(false);
        configurations[i] = config;
        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();
        brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
        pulsarAdmins[i] = new PulsarAdmin(brokerUrls[i], (Authentication) null);
    }
    Thread.sleep(100);
    createProperty(pulsarAdmins[BROKER_COUNT - 1]);
    for (int i = 0; i < BROKER_COUNT; i++) {
        String topic = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]);
        pulsarAdmins[0].namespaces().createNamespace(topic);
    }
}

Example 14 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class MockedPulsarServiceBaseTest method startBroker.

protected PulsarService startBroker(ServiceConfiguration conf) throws Exception {
    PulsarService pulsar = spy(new PulsarService(conf));
    setupBrokerMocks(pulsar);
    boolean isAuthorizationEnabled = conf.isAuthorizationEnabled();
    // enable authrorization to initialize authorization service which is used by grant-permission
    conf.setAuthorizationEnabled(true);
    pulsar.start();
    conf.setAuthorizationEnabled(isAuthorizationEnabled);
    return pulsar;
}

Example 15 with PulsarService

use of org.apache.pulsar.broker.PulsarService in project incubator-pulsar by apache.

the class PulsarWebResource method checkLocalOrGetPeerReplicationCluster.

protected static CompletableFuture<ClusterData> checkLocalOrGetPeerReplicationCluster(PulsarService pulsarService, NamespaceName namespace) {
    if (!namespace.isGlobal()) {
        return CompletableFuture.completedFuture(null);
    }
    final CompletableFuture<ClusterData> validationFuture = new CompletableFuture<>();
    final String localCluster = pulsarService.getConfiguration().getClusterName();
    final String path = AdminResource.path(POLICIES, namespace.toString());
    pulsarService.getConfigurationCache().policiesCache().getAsync(path).thenAccept(policiesResult -> {
        if (policiesResult.isPresent()) {
            Policies policies = policiesResult.get();
            if (policies.replication_clusters.isEmpty()) {
                String msg = String.format("Global namespace does not have any clusters configured : local_cluster=%s ns=%s", localCluster, namespace.toString());
                log.warn(msg);
                validationFuture.completeExceptionally(new RestException(Status.PRECONDITION_FAILED, msg));
            } else if (!policies.replication_clusters.contains(localCluster)) {
                ClusterData ownerPeerCluster = getOwnerFromPeerClusterList(pulsarService, policies.replication_clusters);
                if (ownerPeerCluster != null) {
                    // found a peer that own this namespace
                    validationFuture.complete(ownerPeerCluster);
                    return;
                }
                String msg = String.format("Global namespace missing local cluster name in replication list : local_cluster=%s ns=%s repl_clusters=%s", localCluster, namespace.toString(), policies.replication_clusters);
                log.warn(msg);
                validationFuture.completeExceptionally(new RestException(Status.PRECONDITION_FAILED, msg));
            } else {
                validationFuture.complete(null);
            }
        } else {
            String msg = String.format("Policies not found for %s namespace", namespace.toString());
            log.error(msg);
            validationFuture.completeExceptionally(new RestException(Status.NOT_FOUND, msg));
        }
    }).exceptionally(ex -> {
        String msg = String.format("Failed to validate global cluster configuration : cluster=%s ns=%s  emsg=%s", localCluster, namespace, ex.getMessage());
        log.error(msg);
        validationFuture.completeExceptionally(new RestException(ex));
        return null;
    });
    return validationFuture;
}