Search in sources :

Example 96 with TopicName

use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testFailReceiveAsyncOnConsumerClose.

@Test(timeOut = 5000)
public void testFailReceiveAsyncOnConsumerClose() throws Exception {
    log.info("-- Starting {} test --", methodName);
    // (1) simple consumers
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/failAsyncReceive", "my-subscriber-name", new ConsumerConfiguration());
    consumer.close();
    // receive messages
    try {
        consumer.receiveAsync().get(1, TimeUnit.SECONDS);
        fail("it should have failed because consumer is already closed");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PulsarClientException.AlreadyClosedException);
    }
    // (2) Partitioned-consumer
    int numPartitions = 4;
    TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/failAsyncReceive");
    admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
    Consumer partitionedConsumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", new ConsumerConfiguration());
    partitionedConsumer.close();
    // receive messages
    try {
        partitionedConsumer.receiveAsync().get(1, TimeUnit.SECONDS);
        fail("it should have failed because consumer is already closed");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PulsarClientException.AlreadyClosedException);
    }
    log.info("-- Exiting {} test --", methodName);
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) ExecutionException(java.util.concurrent.ExecutionException) TopicName(org.apache.pulsar.common.naming.TopicName) Test(org.testng.annotations.Test)

Example 97 with TopicName

use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.

the class BrokerClientIntegrationTest method testResetCursor.

@Test(timeOut = 10000, dataProvider = "subType")
public void testResetCursor(SubscriptionType subType) throws Exception {
    final RetentionPolicies policy = new RetentionPolicies(60, 52 * 1024);
    final TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/unacked-topic");
    final int warmup = 20;
    final int testSize = 150;
    final List<Message<byte[]>> received = new ArrayList<>();
    final String subsId = "sub";
    final NavigableMap<Long, TimestampEntryCount> publishTimeIdMap = new ConcurrentSkipListMap<>();
    // set delay time to start dispatching messages to active consumer in order to avoid message duplication
    conf.setActiveConsumerFailoverDelayTimeMillis(500);
    restartBroker();
    admin.namespaces().setRetention(topicName.getNamespace(), policy);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName(subsId).subscriptionType(subType).messageListener((consumer, msg) -> {
        try {
            synchronized (received) {
                received.add(msg);
            }
            consumer.acknowledge(msg);
            long publishTime = ((MessageImpl<?>) msg).getPublishTime();
            log.info(" publish time is " + publishTime + "," + msg.getMessageId());
            TimestampEntryCount timestampEntryCount = publishTimeIdMap.computeIfAbsent(publishTime, (k) -> new TimestampEntryCount(publishTime));
            timestampEntryCount.incrementAndGet();
        } catch (final PulsarClientException e) {
            log.warn("Failed to ack!");
        }
    });
    Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    final Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString()).create();
    log.info("warm up started for " + topicName.toString());
    // send warmup msgs
    byte[] msgBytes = new byte[1000];
    for (Integer i = 0; i < warmup; i++) {
        producer.send(msgBytes);
    }
    log.info("warm up finished.");
    // sleep to ensure receiving of msgs
    for (int n = 0; n < 10 && received.size() < warmup; n++) {
        Thread.sleep(200);
    }
    // validate received msgs
    Assert.assertEquals(received.size(), warmup);
    received.clear();
    // publish testSize num of msgs
    log.info("Sending more messages.");
    for (Integer n = 0; n < testSize; n++) {
        producer.send(msgBytes);
        Thread.sleep(1);
    }
    log.info("Sending more messages done.");
    Thread.sleep(3000);
    long begints = publishTimeIdMap.firstEntry().getKey();
    long endts = publishTimeIdMap.lastEntry().getKey();
    // find reset timestamp
    long timestamp = (endts - begints) / 2 + begints;
    timestamp = publishTimeIdMap.floorKey(timestamp);
    NavigableMap<Long, TimestampEntryCount> expectedMessages = new ConcurrentSkipListMap<>();
    expectedMessages.putAll(publishTimeIdMap.tailMap(timestamp, true));
    received.clear();
    log.info("reset cursor to " + timestamp + " for topic " + topicName.toString() + " for subs " + subsId);
    log.info("issuing admin operation on " + admin.getServiceUrl().toString());
    List<String> subList = admin.persistentTopics().getSubscriptions(topicName.toString());
    for (String subs : subList) {
        log.info("got sub " + subs);
    }
    publishTimeIdMap.clear();
    // reset the cursor to this timestamp
    Assert.assertTrue(subList.contains(subsId));
    admin.persistentTopics().resetCursor(topicName.toString(), subsId, timestamp);
    Thread.sleep(3000);
    int totalExpected = 0;
    for (TimestampEntryCount tec : expectedMessages.values()) {
        totalExpected += tec.numMessages;
    }
    // validate that replay happens after the timestamp
    Assert.assertTrue(publishTimeIdMap.firstEntry().getKey() >= timestamp);
    consumer1.close();
    consumer2.close();
    producer.close();
    // validate that expected and received counts match
    int totalReceived = 0;
    for (TimestampEntryCount tec : publishTimeIdMap.values()) {
        totalReceived += tec.numMessages;
    }
    Assert.assertEquals(totalReceived, totalExpected, "did not receive all messages on replay after reset");
    resetConfig();
    restartBroker();
}
Also used : RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) Message(org.apache.pulsar.client.api.Message) ArrayList(java.util.ArrayList) TopicName(org.apache.pulsar.common.naming.TopicName) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) Test(org.testng.annotations.Test)

Example 98 with TopicName

use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testPartitionedMetadataWithDeprecatedVersion.

@Test
public void testPartitionedMetadataWithDeprecatedVersion() throws Exception {
    final String cluster = "use2";
    final String property = "my-property2";
    final String namespace = "my-ns";
    final String topicName = "my-partitioned";
    final int totalPartitions = 10;
    final TopicName dest = TopicName.get("persistent", property, cluster, namespace, topicName);
    admin.clusters().createCluster(cluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, null, null));
    admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(cluster)));
    admin.namespaces().createNamespace(property + "/" + cluster + "/" + namespace);
    admin.persistentTopics().createPartitionedTopic(dest.toString(), totalPartitions);
    stopBroker();
    conf.setClientLibraryVersionCheckEnabled(true);
    startBroker();
    URI brokerServiceUrl = new URI(pulsar.getWebServiceAddress());
    URL url = brokerServiceUrl.toURL();
    String path = String.format("admin/%s/partitions", dest.getLookupName());
    AsyncHttpClient httpClient = getHttpClient("Pulsar-Java-1.20");
    PartitionedTopicMetadata metadata = getPartitionedMetadata(httpClient, url, path);
    assertEquals(metadata.partitions, totalPartitions);
    httpClient.close();
    httpClient = getHttpClient("Pulsar-CPP-v1.21");
    metadata = getPartitionedMetadata(httpClient, url, path);
    assertEquals(metadata.partitions, totalPartitions);
    httpClient.close();
    httpClient = getHttpClient("Pulsar-CPP-v1.21-SNAPSHOT");
    metadata = getPartitionedMetadata(httpClient, url, path);
    assertEquals(metadata.partitions, totalPartitions);
    httpClient.close();
    httpClient = getHttpClient("");
    try {
        metadata = getPartitionedMetadata(httpClient, url, path);
        fail("should have failed due to invalid version");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PulsarClientException);
    }
    httpClient.close();
    httpClient = getHttpClient("Pulsar-CPP-v1.20-SNAPSHOT");
    try {
        metadata = getPartitionedMetadata(httpClient, url, path);
        fail("should have failed due to invalid version");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PulsarClientException);
    }
    httpClient.close();
    httpClient = getHttpClient("Pulsar-CPP-v1.20");
    try {
        metadata = getPartitionedMetadata(httpClient, url, path);
        fail("should have failed due to invalid version");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PulsarClientException);
    }
    httpClient.close();
}
Also used : ClusterData(org.apache.pulsar.common.policies.data.ClusterData) PropertyAdmin(org.apache.pulsar.common.policies.data.PropertyAdmin) ExecutionException(java.util.concurrent.ExecutionException) URI(java.net.URI) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) URL(java.net.URL) TopicName(org.apache.pulsar.common.naming.TopicName) AsyncHttpClient(org.asynchttpclient.AsyncHttpClient) DefaultAsyncHttpClient(org.asynchttpclient.DefaultAsyncHttpClient) Test(org.testng.annotations.Test)

Example 99 with TopicName

use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testSplitUnloadLookupTest.

/**
 * <pre>
 * When broker-1's load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testSplitUnloadLookupTest() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/my-ns";
    // (1) Start broker-1
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // (2) Make sure broker-2 always assign bundle to broker1
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    // mock: return Broker1 as a Least-loaded broker when leader receies request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
    URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
    // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
    final String topic1 = "persistent://" + namespace + "/topic1";
    Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream().map(nb -> nb.toString()).collect(Collectors.toSet());
    // (4) Broker-1 will own topic-1
    final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
    assertTrue(serviceUnits1.contains(unsplitBundle));
    // broker-2 should have this bundle into the cache
    TopicName topicName = TopicName.get(topic1);
    NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
    assertEquals(bundleInBroker2.toString(), unsplitBundle);
    // (5) Split the bundle for topic-1
    admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true);
    // (6) Broker-2 should get the watch and update bundle cache
    final int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
            Thread.sleep(200);
        } else {
            break;
        }
    }
    // (7) Make lookup request again to Broker-2 which should succeed.
    final String topic2 = "persistent://" + namespace + "/topic2";
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name").subscribe();
    NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
    assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
    consumer1.close();
    consumer2.close();
    pulsarClient2.close();
    pulsar2.close();
}
Also used : HttpURLConnection(java.net.HttpURLConnection) SSLContext(javax.net.ssl.SSLContext) URL(java.net.URL) DiscoveryService(org.apache.pulsar.discovery.service.DiscoveryService) Request(org.asynchttpclient.Request) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) LoggerFactory(org.slf4j.LoggerFactory) TrustManager(javax.net.ssl.TrustManager) Cleanup(lombok.Cleanup) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) PortManager(org.apache.bookkeeper.test.PortManager) AfterMethod(org.testng.annotations.AfterMethod) AuthenticationException(javax.naming.AuthenticationException) SecureRandom(java.security.SecureRandom) AsyncHttpClientConfig(org.asynchttpclient.AsyncHttpClientConfig) Map(java.util.Map) DefaultAsyncHttpClientConfig(org.asynchttpclient.DefaultAsyncHttpClientConfig) URI(java.net.URI) Mockito.doReturn(org.mockito.Mockito.doReturn) Assert.assertFalse(org.testng.Assert.assertFalse) Method(java.lang.reflect.Method) DefaultKeepAliveStrategy(org.asynchttpclient.channel.DefaultKeepAliveStrategy) HttpRequest(io.netty.handler.codec.http.HttpRequest) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) KeyStore(java.security.KeyStore) Assert.assertNotNull(org.testng.Assert.assertNotNull) Collectors(java.util.stream.Collectors) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) Sets(com.google.common.collect.Sets) Matchers.any(org.mockito.Matchers.any) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) Certificate(java.security.cert.Certificate) PrivateKey(java.security.PrivateKey) BoundRequestBuilder(org.asynchttpclient.BoundRequestBuilder) ModularLoadManagerWrapper(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper) ServiceConfig(org.apache.pulsar.discovery.service.server.ServiceConfig) Optional(java.util.Optional) HttpResponse(io.netty.handler.codec.http.HttpResponse) PortManager.nextFreePort(org.apache.bookkeeper.test.PortManager.nextFreePort) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) TopicName(org.apache.pulsar.common.naming.TopicName) AsyncHttpClient(org.asynchttpclient.AsyncHttpClient) SecurityUtility(org.apache.pulsar.common.util.SecurityUtility) Assert.assertEquals(org.testng.Assert.assertEquals) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ModularLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) Mockito.spy(org.mockito.Mockito.spy) AtomicReference(java.util.concurrent.atomic.AtomicReference) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) InsecureTrustManagerFactory(io.netty.handler.ssl.util.InsecureTrustManagerFactory) Lists(com.google.common.collect.Lists) ListenableFuture(org.asynchttpclient.ListenableFuture) URLConnection(java.net.URLConnection) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) DefaultAsyncHttpClient(org.asynchttpclient.DefaultAsyncHttpClient) LeaderElectionService(org.apache.pulsar.broker.loadbalance.LeaderElectionService) HttpsURLConnection(javax.net.ssl.HttpsURLConnection) PropertyAdmin(org.apache.pulsar.common.policies.data.PropertyAdmin) Logger(org.slf4j.Logger) AuthenticationProvider(org.apache.pulsar.broker.authentication.AuthenticationProvider) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Assert.fail(org.testng.Assert.fail) KeyManagerFactory(javax.net.ssl.KeyManagerFactory) IOException(java.io.IOException) Response(org.asynchttpclient.Response) Field(java.lang.reflect.Field) PulsarService(org.apache.pulsar.broker.PulsarService) KeyManager(javax.net.ssl.KeyManager) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Assert.assertTrue(org.testng.Assert.assertTrue) AsyncCompletionHandler(org.asynchttpclient.AsyncCompletionHandler) InputStream(java.io.InputStream) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) URI(java.net.URI) TopicName(org.apache.pulsar.common.naming.TopicName) Field(java.lang.reflect.Field) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarService(org.apache.pulsar.broker.PulsarService) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) Test(org.testng.annotations.Test)

Example 100 with TopicName

use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testModularLoadManagerSplitBundle.

/**
 * <pre>
 * When broker-1's Modular-load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Broker-2 will be a leader and trigger Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testModularLoadManagerSplitBundle() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String loadBalancerName = conf.getLoadManagerClassName();
    try {
        final String namespace = "my-property/use/my-ns";
        // (1) Start broker-1
        ServiceConfiguration conf2 = new ServiceConfiguration();
        conf2.setAdvertisedAddress("localhost");
        conf2.setBrokerServicePort(PortManager.nextFreePort());
        conf2.setBrokerServicePortTls(PortManager.nextFreePort());
        conf2.setWebServicePort(PortManager.nextFreePort());
        conf2.setWebServicePortTls(PortManager.nextFreePort());
        conf2.setAdvertisedAddress("localhost");
        conf2.setClusterName(conf.getClusterName());
        conf2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
        conf2.setZookeeperServers("localhost:2181");
        PulsarService pulsar2 = startBroker(conf2);
        // configure broker-1 with ModularLoadlManager
        stopBroker();
        conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
        startBroker();
        pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
        pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
        LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
        LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
        Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
        loadManagerField.setAccessible(true);
        // (2) Make sure broker-2 always assign bundle to broker1
        // mock: redirect request to leader [2]
        doReturn(true).when(loadManager2).isCentralized();
        loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
        // mock: return Broker1 as a Least-loaded broker when leader receies request [3]
        doReturn(true).when(loadManager1).isCentralized();
        SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
        Optional<ResourceUnit> res = Optional.of(resourceUnit);
        doReturn(res).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
        loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
        URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
        PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
        // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
        final String topic1 = "persistent://" + namespace + "/topic1";
        Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
        Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream().map(nb -> nb.toString()).collect(Collectors.toSet());
        // (4) Broker-1 will own topic-1
        final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
        assertTrue(serviceUnits1.contains(unsplitBundle));
        // broker-2 should have this bundle into the cache
        TopicName topicName = TopicName.get(topic1);
        NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
        assertEquals(bundleInBroker2.toString(), unsplitBundle);
        // update broker-1 bundle report to zk
        pulsar.getBrokerService().updateRates();
        pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
        // this will create znode for bundle-data
        pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper();
        pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
        // (5) Modular-load-manager will split the bundle due to max-topic threshold reached
        Field leaderField = LeaderElectionService.class.getDeclaredField("isLeader");
        Method updateAllMethod = ModularLoadManagerImpl.class.getDeclaredMethod("updateAll");
        updateAllMethod.setAccessible(true);
        leaderField.setAccessible(true);
        AtomicBoolean isLeader = (AtomicBoolean) leaderField.get(pulsar2.getLeaderElectionService());
        isLeader.set(true);
        ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar2.getLoadManager().get()).getLoadManager();
        // broker-2 loadManager is a leader and let it refresh load-report from all the brokers
        updateAllMethod.invoke(loadManager);
        conf2.setLoadBalancerAutoBundleSplitEnabled(true);
        conf2.setLoadBalancerAutoUnloadSplitBundlesEnabled(true);
        conf2.setLoadBalancerNamespaceBundleMaxTopics(0);
        loadManager.checkNamespaceBundleSplit();
        // (6) Broker-2 should get the watch and update bundle cache
        final int retry = 5;
        for (int i = 0; i < retry; i++) {
            if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
                Thread.sleep(200);
            } else {
                break;
            }
        }
        // (7) Make lookup request again to Broker-2 which should succeed.
        final String topic2 = "persistent://" + namespace + "/topic2";
        Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name").subscribe();
        NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
        assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
        consumer1.close();
        consumer2.close();
        pulsarClient2.close();
        pulsar2.close();
    } finally {
        conf.setLoadManagerClassName(loadBalancerName);
    }
}
Also used : HttpURLConnection(java.net.HttpURLConnection) SSLContext(javax.net.ssl.SSLContext) URL(java.net.URL) DiscoveryService(org.apache.pulsar.discovery.service.DiscoveryService) Request(org.asynchttpclient.Request) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) LoggerFactory(org.slf4j.LoggerFactory) TrustManager(javax.net.ssl.TrustManager) Cleanup(lombok.Cleanup) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) PortManager(org.apache.bookkeeper.test.PortManager) AfterMethod(org.testng.annotations.AfterMethod) AuthenticationException(javax.naming.AuthenticationException) SecureRandom(java.security.SecureRandom) AsyncHttpClientConfig(org.asynchttpclient.AsyncHttpClientConfig) Map(java.util.Map) DefaultAsyncHttpClientConfig(org.asynchttpclient.DefaultAsyncHttpClientConfig) URI(java.net.URI) Mockito.doReturn(org.mockito.Mockito.doReturn) Assert.assertFalse(org.testng.Assert.assertFalse) Method(java.lang.reflect.Method) DefaultKeepAliveStrategy(org.asynchttpclient.channel.DefaultKeepAliveStrategy) HttpRequest(io.netty.handler.codec.http.HttpRequest) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) KeyStore(java.security.KeyStore) Assert.assertNotNull(org.testng.Assert.assertNotNull) Collectors(java.util.stream.Collectors) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) Sets(com.google.common.collect.Sets) Matchers.any(org.mockito.Matchers.any) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) Certificate(java.security.cert.Certificate) PrivateKey(java.security.PrivateKey) BoundRequestBuilder(org.asynchttpclient.BoundRequestBuilder) ModularLoadManagerWrapper(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper) ServiceConfig(org.apache.pulsar.discovery.service.server.ServiceConfig) Optional(java.util.Optional) HttpResponse(io.netty.handler.codec.http.HttpResponse) PortManager.nextFreePort(org.apache.bookkeeper.test.PortManager.nextFreePort) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) TopicName(org.apache.pulsar.common.naming.TopicName) AsyncHttpClient(org.asynchttpclient.AsyncHttpClient) SecurityUtility(org.apache.pulsar.common.util.SecurityUtility) Assert.assertEquals(org.testng.Assert.assertEquals) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ModularLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) Mockito.spy(org.mockito.Mockito.spy) AtomicReference(java.util.concurrent.atomic.AtomicReference) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) InsecureTrustManagerFactory(io.netty.handler.ssl.util.InsecureTrustManagerFactory) Lists(com.google.common.collect.Lists) ListenableFuture(org.asynchttpclient.ListenableFuture) URLConnection(java.net.URLConnection) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) DefaultAsyncHttpClient(org.asynchttpclient.DefaultAsyncHttpClient) LeaderElectionService(org.apache.pulsar.broker.loadbalance.LeaderElectionService) HttpsURLConnection(javax.net.ssl.HttpsURLConnection) PropertyAdmin(org.apache.pulsar.common.policies.data.PropertyAdmin) Logger(org.slf4j.Logger) AuthenticationProvider(org.apache.pulsar.broker.authentication.AuthenticationProvider) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Assert.fail(org.testng.Assert.fail) KeyManagerFactory(javax.net.ssl.KeyManagerFactory) IOException(java.io.IOException) Response(org.asynchttpclient.Response) Field(java.lang.reflect.Field) PulsarService(org.apache.pulsar.broker.PulsarService) KeyManager(javax.net.ssl.KeyManager) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Assert.assertTrue(org.testng.Assert.assertTrue) AsyncCompletionHandler(org.asynchttpclient.AsyncCompletionHandler) InputStream(java.io.InputStream) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) AfterMethod(org.testng.annotations.AfterMethod) Method(java.lang.reflect.Method) BeforeMethod(org.testng.annotations.BeforeMethod) URI(java.net.URI) ModularLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl) TopicName(org.apache.pulsar.common.naming.TopicName) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Field(java.lang.reflect.Field) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarService(org.apache.pulsar.broker.PulsarService) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) Test(org.testng.annotations.Test)

Aggregations

TopicName (org.apache.pulsar.common.naming.TopicName)127 Test (org.testng.annotations.Test)54 CompletableFuture (java.util.concurrent.CompletableFuture)43 WebTarget (javax.ws.rs.client.WebTarget)32 NamespaceBundle (org.apache.pulsar.common.naming.NamespaceBundle)23 NamespaceName (org.apache.pulsar.common.naming.NamespaceName)23 Logger (org.slf4j.Logger)23 LoggerFactory (org.slf4j.LoggerFactory)23 List (java.util.List)22 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)22 Map (java.util.Map)20 ExecutionException (java.util.concurrent.ExecutionException)20 TimeUnit (java.util.concurrent.TimeUnit)20 NamingException (org.apache.pulsar.broker.service.BrokerServiceException.NamingException)18 Field (java.lang.reflect.Field)17 PersistentTopic (org.apache.pulsar.broker.service.persistent.PersistentTopic)17 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)17 ByteBuf (io.netty.buffer.ByteBuf)15 Set (java.util.Set)15 ServerMetadataException (org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException)14