Examples with PulsarServiceNameResolver org.apache.pulsar.client.impl.PulsarServiceNameResolver used on opensource projects

Example 1 with PulsarServiceNameResolver

use of org.apache.pulsar.client.impl.PulsarServiceNameResolver in project pulsar by yahoo.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBroker.

/**
 * Verifies: Broker side throttling:
 *
 * <pre>
 * 1. concurrent_consumer_creation > maxConcurrentLookupRequest at broker
 * 2. few of the consumer creation must fail with TooManyLookupRequestException.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBroker() throws Exception {
    final String topicName = "persistent://prop/ns-abc/newTopic";
    @Cleanup PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).statsInterval(0, TimeUnit.SECONDS).ioThreads(20).connectionsPerBroker(20).build();
    int newPermits = 1;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }
    PulsarServiceNameResolver resolver = new PulsarServiceNameResolver();
    resolver.updateServiceUrl(pulsar.getBrokerServiceUrl());
    ClientConfigurationData conf = new ClientConfigurationData();
    conf.setConnectionsPerBroker(20);
    EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(20, false, new DefaultThreadFactory("test-pool", Thread.currentThread().isDaemon()));
    ExecutorService executor = Executors.newFixedThreadPool(10);
    try (ConnectionPool pool = new ConnectionPool(conf, eventLoop)) {
        final int totalConsumers = 20;
        List<Future<?>> futures = new ArrayList<>();
        // test for partitionMetadataRequest
        for (int i = 0; i < totalConsumers; i++) {
            long reqId = 0xdeadbeef + i;
            Future<?> f = executor.submit(() -> {
                ByteBuf request = Commands.newPartitionMetadataRequest(topicName, reqId);
                pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request, reqId)).get();
                return null;
            });
            futures.add(f);
        }
        int rejects = 0;
        for (Future<?> f : futures) {
            try {
                f.get();
            } catch (ExecutionException e) {
                Throwable rootCause = e;
                while (rootCause instanceof ExecutionException) {
                    rootCause = rootCause.getCause();
                }
                if (rootCause instanceof org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException) {
                    rejects++;
                } else {
                    throw e;
                }
            }
        }
        assertTrue(rejects > 0);
        futures.clear();
        // test for lookup
        for (int i = 0; i < totalConsumers; i++) {
            long reqId = 0xdeadfeef + i;
            Future<?> f = executor.submit(() -> {
                ByteBuf request = Commands.newLookup(topicName, true, reqId);
                pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request, reqId)).get();
                return null;
            });
            futures.add(f);
        }
        rejects = 0;
        for (Future<?> f : futures) {
            try {
                f.get();
            } catch (ExecutionException e) {
                Throwable rootCause = e;
                while (rootCause instanceof ExecutionException) {
                    rootCause = rootCause.getCause();
                }
                if (rootCause instanceof org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException) {
                    rejects++;
                } else {
                    throw e;
                }
            }
        }
        assertTrue(rejects > 0);
    } finally {
        executor.shutdownNow();
        eventLoop.shutdownNow();
    }
}

Example 2 with PulsarServiceNameResolver

use of org.apache.pulsar.client.impl.PulsarServiceNameResolver in project pulsar by yahoo.

the class PulsarWebResource method getRedirectionUrl.

private URI getRedirectionUrl(ClusterData differentClusterData) throws MalformedURLException {
    try {
        PulsarServiceNameResolver serviceNameResolver = new PulsarServiceNameResolver();
        if (isRequestHttps() && pulsar.getConfiguration().getWebServicePortTls().isPresent() && StringUtils.isNotBlank(differentClusterData.getServiceUrlTls())) {
            serviceNameResolver.updateServiceUrl(differentClusterData.getServiceUrlTls());
        } else {
            serviceNameResolver.updateServiceUrl(differentClusterData.getServiceUrl());
        }
        URL webUrl = new URL(serviceNameResolver.resolveHostUri().toString());
        return UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.getHost()).port(webUrl.getPort()).build();
    } catch (PulsarClientException.InvalidServiceURL exception) {
        throw new MalformedURLException(exception.getMessage());
    }
}

Example 3 with PulsarServiceNameResolver

use of org.apache.pulsar.client.impl.PulsarServiceNameResolver in project incubator-pulsar by apache.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBroker.

/**
 * Verifies: Broker side throttling:
 *
 * <pre>
 * 1. concurrent_consumer_creation > maxConcurrentLookupRequest at broker
 * 2. few of the consumer creation must fail with TooManyLookupRequestException.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBroker() throws Exception {
    final String topicName = "persistent://prop/ns-abc/newTopic";
    @Cleanup PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).statsInterval(0, TimeUnit.SECONDS).ioThreads(20).connectionsPerBroker(20).build();
    int newPermits = 1;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }
    PulsarServiceNameResolver resolver = new PulsarServiceNameResolver();
    resolver.updateServiceUrl(pulsar.getBrokerServiceUrl());
    ClientConfigurationData conf = new ClientConfigurationData();
    conf.setConnectionsPerBroker(20);
    EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(20, false, new DefaultThreadFactory("test-pool", Thread.currentThread().isDaemon()));
    ExecutorService executor = Executors.newFixedThreadPool(10);
    try (ConnectionPool pool = new ConnectionPool(conf, eventLoop)) {
        final int totalConsumers = 20;
        List<Future<?>> futures = new ArrayList<>();
        // test for partitionMetadataRequest
        for (int i = 0; i < totalConsumers; i++) {
            long reqId = 0xdeadbeef + i;
            Future<?> f = executor.submit(() -> {
                ByteBuf request = Commands.newPartitionMetadataRequest(topicName, reqId);
                pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request, reqId)).get();
                return null;
            });
            futures.add(f);
        }
        int rejects = 0;
        for (Future<?> f : futures) {
            try {
                f.get();
            } catch (ExecutionException e) {
                Throwable rootCause = e;
                while (rootCause instanceof ExecutionException) {
                    rootCause = rootCause.getCause();
                }
                if (rootCause instanceof org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException) {
                    rejects++;
                } else {
                    throw e;
                }
            }
        }
        assertTrue(rejects > 0);
        futures.clear();
        // test for lookup
        for (int i = 0; i < totalConsumers; i++) {
            long reqId = 0xdeadfeef + i;
            Future<?> f = executor.submit(() -> {
                ByteBuf request = Commands.newLookup(topicName, true, reqId);
                pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request, reqId)).get();
                return null;
            });
            futures.add(f);
        }
        rejects = 0;
        for (Future<?> f : futures) {
            try {
                f.get();
            } catch (ExecutionException e) {
                Throwable rootCause = e;
                while (rootCause instanceof ExecutionException) {
                    rootCause = rootCause.getCause();
                }
                if (rootCause instanceof org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException) {
                    rejects++;
                } else {
                    throw e;
                }
            }
        }
        assertTrue(rejects > 0);
    } finally {
        executor.shutdownNow();
        eventLoop.shutdownNow();
    }
}

Example 4 with PulsarServiceNameResolver

use of org.apache.pulsar.client.impl.PulsarServiceNameResolver in project incubator-pulsar by apache.

the class PulsarWebResource method getRedirectionUrl.

private URI getRedirectionUrl(ClusterData differentClusterData) throws MalformedURLException {
    try {
        PulsarServiceNameResolver serviceNameResolver = new PulsarServiceNameResolver();
        if (isRequestHttps() && pulsar.getConfiguration().getWebServicePortTls().isPresent() && StringUtils.isNotBlank(differentClusterData.getServiceUrlTls())) {
            serviceNameResolver.updateServiceUrl(differentClusterData.getServiceUrlTls());
        } else {
            serviceNameResolver.updateServiceUrl(differentClusterData.getServiceUrl());
        }
        URL webUrl = new URL(serviceNameResolver.resolveHostUri().toString());
        return UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.getHost()).port(webUrl.getPort()).build();
    } catch (PulsarClientException.InvalidServiceURL exception) {
        throw new MalformedURLException(exception.getMessage());
    }
}

Example 5 with PulsarServiceNameResolver

use of org.apache.pulsar.client.impl.PulsarServiceNameResolver in project incubator-pulsar by apache.

the class BrokerServiceTest method testLookupThrottlingForClientByClient.

/**
 * Verifies: client side throttling.
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByClient() throws Exception {
    // This test looks like it could be flakey, if the broker responds
    // quickly enough, there may never be concurrency in requests
    final String topicName = "persistent://prop/ns-abc/newTopic";
    PulsarServiceNameResolver resolver = new PulsarServiceNameResolver();
    resolver.updateServiceUrl(pulsar.getBrokerServiceUrl());
    ClientConfigurationData conf = new ClientConfigurationData();
    conf.setConcurrentLookupRequest(1);
    conf.setMaxLookupRequest(2);
    EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(20, false, new DefaultThreadFactory("test-pool", Thread.currentThread().isDaemon()));
    long reqId = 0xdeadbeef;
    try (ConnectionPool pool = new ConnectionPool(conf, eventLoop)) {
        // for PMR
        // 2 lookup will succeed
        long reqId1 = reqId++;
        ByteBuf request1 = Commands.newPartitionMetadataRequest(topicName, reqId1);
        CompletableFuture<?> f1 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request1, reqId1));
        long reqId2 = reqId++;
        ByteBuf request2 = Commands.newPartitionMetadataRequest(topicName, reqId2);
        CompletableFuture<?> f2 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request2, reqId2));
        f1.get();
        f2.get();
        // 3 lookup will fail
        long reqId3 = reqId++;
        ByteBuf request3 = Commands.newPartitionMetadataRequest(topicName, reqId3);
        f1 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request3, reqId3));
        long reqId4 = reqId++;
        ByteBuf request4 = Commands.newPartitionMetadataRequest(topicName, reqId4);
        f2 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request4, reqId4));
        long reqId5 = reqId++;
        ByteBuf request5 = Commands.newPartitionMetadataRequest(topicName, reqId5);
        CompletableFuture<?> f3 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request5, reqId5));
        try {
            f1.get();
            f2.get();
            f3.get();
            fail("At least one should fail");
        } catch (ExecutionException e) {
            Throwable rootCause = e;
            while (rootCause instanceof ExecutionException) {
                rootCause = rootCause.getCause();
            }
            if (!(rootCause instanceof org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException)) {
                throw e;
            }
        }
        // for Lookup
        // 2 lookup will succeed
        long reqId6 = reqId++;
        ByteBuf request6 = Commands.newLookup(topicName, true, reqId6);
        f1 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request6, reqId6));
        long reqId7 = reqId++;
        ByteBuf request7 = Commands.newLookup(topicName, true, reqId7);
        f2 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request7, reqId7));
        f1.get();
        f2.get();
        // 3 lookup will fail
        long reqId8 = reqId++;
        ByteBuf request8 = Commands.newLookup(topicName, true, reqId8);
        f1 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request8, reqId8));
        long reqId9 = reqId++;
        ByteBuf request9 = Commands.newLookup(topicName, true, reqId9);
        f2 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request9, reqId9));
        long reqId10 = reqId++;
        ByteBuf request10 = Commands.newLookup(topicName, true, reqId10);
        f3 = pool.getConnection(resolver.resolveHost()).thenCompose(clientCnx -> clientCnx.newLookup(request10, reqId10));
        try {
            f1.get();
            f2.get();
            f3.get();
            fail("At least one should fail");
        } catch (ExecutionException e) {
            Throwable rootCause = e;
            while (rootCause instanceof ExecutionException) {
                rootCause = rootCause.getCause();
            }
            if (!(rootCause instanceof org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException)) {
                throw e;
            }
        }
    }
}