Examples with NamespaceBundle org.apache.pulsar.common.naming.NamespaceBundle used on opensource projects

Example 36 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class ReplicatorTest method testConfigChangeNegativeCases.

@Test(enabled = false, timeOut = 30000)
public void testConfigChangeNegativeCases() throws Exception {
    log.info("--- Starting ReplicatorTest::testConfigChangeNegativeCases ---");
    // Negative test cases for global namespace config change. Verify that the namespace config change can not be
    // updated when the namespace is being unloaded.
    // Set up field access to internal namespace state in NamespaceService
    Field ownershipCacheField = NamespaceService.class.getDeclaredField("ownershipCache");
    ownershipCacheField.setAccessible(true);
    OwnershipCache ownerCache = (OwnershipCache) ownershipCacheField.get(pulsar1.getNamespaceService());
    Assert.assertNotNull(pulsar1, "pulsar1 is null");
    Assert.assertNotNull(pulsar1.getNamespaceService(), "pulsar1.getNamespaceService() is null");
    NamespaceBundle globalNsBundle = pulsar1.getNamespaceService().getNamespaceBundleFactory().getFullBundle(NamespaceName.get("pulsar/global/ns"));
    ownerCache.tryAcquiringOwnership(globalNsBundle);
    Assert.assertNotNull(ownerCache.getOwnedBundle(globalNsBundle), "pulsar1.getNamespaceService().getOwnedServiceUnit(NamespaceName.get(\"pulsar/global/ns\")) is null");
    Field stateField = OwnedBundle.class.getDeclaredField("isActive");
    stateField.setAccessible(true);
    // set the namespace to be disabled
    ownerCache.disableOwnership(globalNsBundle);
    // Make sure the namespace config update failed
    try {
        admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1"));
        fail("Should have raised exception");
    } catch (PreconditionFailedException pfe) {
    // OK
    }
    // restore the namespace state
    ownerCache.removeOwnership(globalNsBundle).get();
    ownerCache.tryAcquiringOwnership(globalNsBundle);
}

Example 37 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class BrokerClientIntegrationTest method testDisconnectClientWithoutClosingConnection.

/**
 * Verifies unload namespace-bundle doesn't close shared connection used by other namespace-bundle.
 *
 * <pre>
 * 1. after disabling broker fron loadbalancer
 * 2. unload namespace-bundle "my-ns1" which disconnects client (producer/consumer) connected on that namespacebundle
 * 3. but doesn't close the connection for namesapce-bundle "my-ns2" and clients are still connected
 * 4. verifies unloaded "my-ns1" should not connected again with the broker as broker is disabled
 * 5. unload "my-ns2" which closes the connection as broker doesn't have any more client connected on that connection
 * 6. all namespace-bundles are in "connecting" state and waiting for available broker
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testDisconnectClientWithoutClosingConnection() throws Exception {
    final String ns1 = "my-property/use/con-ns1";
    final String ns2 = "my-property/use/con-ns2";
    admin.namespaces().createNamespace(ns1);
    admin.namespaces().createNamespace(ns2);
    final String topic1 = "persistent://" + ns1 + "/my-topic";
    final String topic2 = "persistent://" + ns2 + "/my-topic";
    ConsumerImpl<byte[]> cons1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    ProducerImpl<byte[]> prod1 = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topic1).create();
    ProducerImpl<byte[]> prod2 = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topic2).create();
    ConsumerImpl<byte[]> consumer1 = spy(cons1);
    doAnswer(invocationOnMock -> cons1.getState()).when(consumer1).getState();
    doAnswer(invocationOnMock -> cons1.getClientCnx()).when(consumer1).getClientCnx();
    doAnswer(invocationOnMock -> cons1.cnx()).when(consumer1).cnx();
    doAnswer(invocationOnMock -> {
        cons1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
        return null;
    }).when(consumer1).connectionClosed(anyObject());
    ProducerImpl<byte[]> producer1 = spy(prod1);
    doAnswer(invocationOnMock -> prod1.getState()).when(producer1).getState();
    doAnswer(invocationOnMock -> prod1.getClientCnx()).when(producer1).getClientCnx();
    doAnswer(invocationOnMock -> prod1.cnx()).when(producer1).cnx();
    doAnswer(invocationOnMock -> {
        prod1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
        return null;
    }).when(producer1).connectionClosed(anyObject());
    ProducerImpl<byte[]> producer2 = spy(prod2);
    doAnswer(invocationOnMock -> prod2.getState()).when(producer2).getState();
    doAnswer(invocationOnMock -> prod2.getClientCnx()).when(producer2).getClientCnx();
    doAnswer(invocationOnMock -> prod2.cnx()).when(producer2).cnx();
    doAnswer(invocationOnMock -> {
        prod2.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
        return null;
    }).when(producer2).connectionClosed(anyObject());
    ClientCnx clientCnx = producer1.getClientCnx();
    Field pfield = ClientCnx.class.getDeclaredField("producers");
    pfield.setAccessible(true);
    Field cfield = ClientCnx.class.getDeclaredField("consumers");
    cfield.setAccessible(true);
    ConcurrentLongHashMap<ProducerImpl<byte[]>> producers = (ConcurrentLongHashMap) pfield.get(clientCnx);
    ConcurrentLongHashMap<ConsumerImpl<byte[]>> consumers = (ConcurrentLongHashMap) cfield.get(clientCnx);
    producers.put(2, producers.get(0));
    producers.put(3, producers.get(1));
    consumers.put(1, consumers.get(0));
    producers.put(0, producer1);
    producers.put(1, producer2);
    consumers.put(0, consumer1);
    // disable this broker to avoid any new requests
    pulsar.getLoadManager().get().disableBroker();
    NamespaceBundle bundle1 = pulsar.getNamespaceService().getBundle(TopicName.get(topic1));
    NamespaceBundle bundle2 = pulsar.getNamespaceService().getBundle(TopicName.get(topic2));
    // unload ns-bundle:1
    pulsar.getNamespaceService().unloadNamespaceBundle((NamespaceBundle) bundle1);
    // let server send signal to close-connection and client close the connection
    Thread.sleep(1000);
    // [1] Verify: producer1 must get connectionClosed signal
    verify(producer1, atLeastOnce()).connectionClosed(anyObject());
    // [2] Verify: consumer1 must get connectionClosed signal
    verify(consumer1, atLeastOnce()).connectionClosed(anyObject());
    // [3] Verify: producer2 should have not received connectionClosed signal
    verify(producer2, never()).connectionClosed(anyObject());
    // sleep for sometime to let other disconnected producer and consumer connect again: but they should not get
    // connected with same broker as that broker is already out from active-broker list
    Thread.sleep(200);
    // producer1 must not be able to connect again
    assertTrue(prod1.getClientCnx() == null);
    assertTrue(prod1.getState().equals(State.Connecting));
    // consumer1 must not be able to connect again
    assertTrue(cons1.getClientCnx() == null);
    assertTrue(cons1.getState().equals(State.Connecting));
    // producer2 must have live connection
    assertTrue(prod2.getClientCnx() != null);
    assertTrue(prod2.getState().equals(State.Ready));
    // unload ns-bundle2 as well
    pulsar.getNamespaceService().unloadNamespaceBundle((NamespaceBundle) bundle2);
    // let producer2 give some time to get disconnect signal and get disconencted
    Thread.sleep(200);
    verify(producer2, atLeastOnce()).connectionClosed(anyObject());
    // producer1 must not be able to connect again
    assertTrue(prod1.getClientCnx() == null);
    assertTrue(prod1.getState().equals(State.Connecting));
    // consumer1 must not be able to connect again
    assertTrue(cons1.getClientCnx() == null);
    assertTrue(cons1.getState().equals(State.Connecting));
    // producer2 must not be able to connect again
    assertTrue(prod2.getClientCnx() == null);
    assertTrue(prod2.getState().equals(State.Connecting));
    producer1.close();
    producer2.close();
    consumer1.close();
    prod1.close();
    prod2.close();
    cons1.close();
}

Example 38 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testSplitUnloadLookupTest.

/**
 * <pre>
 * When broker-1's load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testSplitUnloadLookupTest() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/my-ns";
    // (1) Start broker-1
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // (2) Make sure broker-2 always assign bundle to broker1
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    // mock: return Broker1 as a Least-loaded broker when leader receies request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
    URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
    // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
    final String topic1 = "persistent://" + namespace + "/topic1";
    Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream().map(nb -> nb.toString()).collect(Collectors.toSet());
    // (4) Broker-1 will own topic-1
    final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
    assertTrue(serviceUnits1.contains(unsplitBundle));
    // broker-2 should have this bundle into the cache
    TopicName topicName = TopicName.get(topic1);
    NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
    assertEquals(bundleInBroker2.toString(), unsplitBundle);
    // (5) Split the bundle for topic-1
    admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true);
    // (6) Broker-2 should get the watch and update bundle cache
    final int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
            Thread.sleep(200);
        } else {
            break;
        }
    }
    // (7) Make lookup request again to Broker-2 which should succeed.
    final String topic2 = "persistent://" + namespace + "/topic2";
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name").subscribe();
    NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
    assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
    consumer1.close();
    consumer2.close();
    pulsarClient2.close();
    pulsar2.close();
}

Example 39 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testModularLoadManagerSplitBundle.

/**
 * <pre>
 * When broker-1's Modular-load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Broker-2 will be a leader and trigger Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testModularLoadManagerSplitBundle() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String loadBalancerName = conf.getLoadManagerClassName();
    try {
        final String namespace = "my-property/use/my-ns";
        // (1) Start broker-1
        ServiceConfiguration conf2 = new ServiceConfiguration();
        conf2.setAdvertisedAddress("localhost");
        conf2.setBrokerServicePort(PortManager.nextFreePort());
        conf2.setBrokerServicePortTls(PortManager.nextFreePort());
        conf2.setWebServicePort(PortManager.nextFreePort());
        conf2.setWebServicePortTls(PortManager.nextFreePort());
        conf2.setAdvertisedAddress("localhost");
        conf2.setClusterName(conf.getClusterName());
        conf2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
        conf2.setZookeeperServers("localhost:2181");
        PulsarService pulsar2 = startBroker(conf2);
        // configure broker-1 with ModularLoadlManager
        stopBroker();
        conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
        startBroker();
        pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
        pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
        LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
        LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
        Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
        loadManagerField.setAccessible(true);
        // (2) Make sure broker-2 always assign bundle to broker1
        // mock: redirect request to leader [2]
        doReturn(true).when(loadManager2).isCentralized();
        loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
        // mock: return Broker1 as a Least-loaded broker when leader receies request [3]
        doReturn(true).when(loadManager1).isCentralized();
        SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
        Optional<ResourceUnit> res = Optional.of(resourceUnit);
        doReturn(res).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
        loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
        URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
        PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
        // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
        final String topic1 = "persistent://" + namespace + "/topic1";
        Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
        Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream().map(nb -> nb.toString()).collect(Collectors.toSet());
        // (4) Broker-1 will own topic-1
        final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
        assertTrue(serviceUnits1.contains(unsplitBundle));
        // broker-2 should have this bundle into the cache
        TopicName topicName = TopicName.get(topic1);
        NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
        assertEquals(bundleInBroker2.toString(), unsplitBundle);
        // update broker-1 bundle report to zk
        pulsar.getBrokerService().updateRates();
        pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
        // this will create znode for bundle-data
        pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper();
        pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
        // (5) Modular-load-manager will split the bundle due to max-topic threshold reached
        Field leaderField = LeaderElectionService.class.getDeclaredField("isLeader");
        Method updateAllMethod = ModularLoadManagerImpl.class.getDeclaredMethod("updateAll");
        updateAllMethod.setAccessible(true);
        leaderField.setAccessible(true);
        AtomicBoolean isLeader = (AtomicBoolean) leaderField.get(pulsar2.getLeaderElectionService());
        isLeader.set(true);
        ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar2.getLoadManager().get()).getLoadManager();
        // broker-2 loadManager is a leader and let it refresh load-report from all the brokers
        updateAllMethod.invoke(loadManager);
        conf2.setLoadBalancerAutoBundleSplitEnabled(true);
        conf2.setLoadBalancerAutoUnloadSplitBundlesEnabled(true);
        conf2.setLoadBalancerNamespaceBundleMaxTopics(0);
        loadManager.checkNamespaceBundleSplit();
        // (6) Broker-2 should get the watch and update bundle cache
        final int retry = 5;
        for (int i = 0; i < retry; i++) {
            if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
                Thread.sleep(200);
            } else {
                break;
            }
        }
        // (7) Make lookup request again to Broker-2 which should succeed.
        final String topic2 = "persistent://" + namespace + "/topic2";
        Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name").subscribe();
        NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
        assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
        consumer1.close();
        consumer2.close();
        pulsarClient2.close();
        pulsar2.close();
    } finally {
        conf.setLoadManagerClassName(loadBalancerName);
    }
}

Example 40 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class BrokerServiceTest method testBrokerServiceNamespaceStats.

@Test
public void testBrokerServiceNamespaceStats() throws Exception {
    final int numBundles = 4;
    final String ns1 = "prop/use/stats1";
    final String ns2 = "prop/use/stats2";
    List<String> nsList = Lists.newArrayList(ns1, ns2);
    List<Producer<byte[]>> producerList = Lists.newArrayList();
    BrokerStats brokerStatsClient = admin.brokerStats();
    for (String ns : nsList) {
        admin.namespaces().createNamespace(ns, numBundles);
        String topic1 = String.format("persistent://%s/topic1", ns);
        producerList.add(pulsarClient.newProducer().topic(topic1).create());
        String topic2 = String.format("persistent://%s/topic2", ns);
        producerList.add(pulsarClient.newProducer().topic(topic2).create());
    }
    rolloverPerIntervalStats();
    JsonObject topicStats = brokerStatsClient.getTopics();
    assertEquals(topicStats.size(), 2, topicStats.toString());
    for (String ns : nsList) {
        JsonObject nsObject = topicStats.getAsJsonObject(ns);
        List<String> topicList = admin.namespaces().getTopics(ns);
        for (String topic : topicList) {
            NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService().getBundle(TopicName.get(topic));
            JsonObject bundleObject = nsObject.getAsJsonObject(bundle.getBundleRange());
            JsonObject topicObject = bundleObject.getAsJsonObject("persistent");
            AtomicBoolean topicPresent = new AtomicBoolean();
            topicObject.entrySet().iterator().forEachRemaining(persistentTopic -> {
                if (persistentTopic.getKey().equals(topic)) {
                    topicPresent.set(true);
                }
            });
            assertTrue(topicPresent.get());
        }
    }
    for (Producer<?> producer : producerList) {
        producer.close();
    }
    for (String ns : nsList) {
        List<String> topics = admin.namespaces().getTopics(ns);
        for (String dest : topics) {
            admin.persistentTopics().delete(dest);
        }
        admin.namespaces().deleteNamespace(ns);
    }
}