Examples with NamespaceBundle com.yahoo.pulsar.common.naming.NamespaceBundle used on opensource projects

Example 36 with NamespaceBundle

use of com.yahoo.pulsar.common.naming.NamespaceBundle in project pulsar by yahoo.

the class BrokerServiceTest method testBrokerServiceNamespaceStats.

@SuppressWarnings("unchecked")
@Test
public void testBrokerServiceNamespaceStats() throws Exception {
    final int numBundles = 4;
    final String ns1 = "prop/use/stats1";
    final String ns2 = "prop/use/stats2";
    List<String> nsList = Lists.newArrayList(ns1, ns2);
    List<Producer> producerList = Lists.newArrayList();
    BrokerStats brokerStatsClient = admin.brokerStats();
    for (String ns : nsList) {
        admin.namespaces().createNamespace(ns, numBundles);
        String topic1 = String.format("persistent://%s/topic1", ns);
        producerList.add(pulsarClient.createProducer(topic1));
        String topic2 = String.format("persistent://%s/topic2", ns);
        producerList.add(pulsarClient.createProducer(topic2));
    }
    rolloverPerIntervalStats();
    JsonObject destinationStats = brokerStatsClient.getDestinations();
    assertEquals(destinationStats.size(), 2, destinationStats.toString());
    for (String ns : nsList) {
        JsonObject nsObject = destinationStats.getAsJsonObject(ns);
        List<String> topicList = admin.namespaces().getDestinations(ns);
        for (String topic : topicList) {
            NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService().getBundle(DestinationName.get(topic));
            JsonObject bundleObject = nsObject.getAsJsonObject(bundle.getBundleRange());
            JsonObject topicObject = bundleObject.getAsJsonObject("persistent");
            AtomicBoolean topicPresent = new AtomicBoolean();
            topicObject.entrySet().iterator().forEachRemaining(persistentTopic -> {
                if (persistentTopic.getKey().equals(topic)) {
                    topicPresent.set(true);
                }
            });
            assertTrue(topicPresent.get());
        }
    }
    for (Producer producer : producerList) {
        producer.close();
    }
    for (String ns : nsList) {
        List<String> destinations = admin.namespaces().getDestinations(ns);
        for (String dest : destinations) {
            admin.persistentTopics().delete(dest);
        }
        admin.namespaces().deleteNamespace(ns);
    }
}

Example 37 with NamespaceBundle

use of com.yahoo.pulsar.common.naming.NamespaceBundle in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitMapWithRefreshedStatMap.

@Test
public void testSplitMapWithRefreshedStatMap() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    ManagedLedger ledger = mock(ManagedLedger.class);
    when(ledger.getCursors()).thenReturn(Lists.newArrayList());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    PersistentTopic topic = new PersistentTopic(dn.toString(), ledger, pulsar.getBrokerService());
    Method method = pulsar.getBrokerService().getClass().getDeclaredMethod("addTopicToStatsMaps", DestinationName.class, PersistentTopic.class);
    method.setAccessible(true);
    method.invoke(pulsar.getBrokerService(), dn, topic);
    String nspace = originalBundle.getNamespaceObject().toString();
    List<PersistentTopic> list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertNotNull(list);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    try {
        // old bundle should be removed from status-map
        list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
        fail();
    } catch (NullPointerException ne) {
    // OK
    }
    // status-map should be updated with new split bundles
    NamespaceBundle splitBundle = pulsar.getNamespaceService().getBundle(dn);
    assertTrue(!CollectionUtils.isEmpty(this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, splitBundle.toString())));
}

Example 38 with NamespaceBundle

use of com.yahoo.pulsar.common.naming.NamespaceBundle in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitAndOwnBundles.

@Test
public void testSplitAndOwnBundles() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    NamespaceBundleFactory bundleFactory = this.pulsar.getNamespaceService().getNamespaceBundleFactory();
    NamespaceBundles updatedNsBundles = bundleFactory.getBundles(nsname);
    // new updated bundles shouldn't be null
    assertNotNull(updatedNsBundles);
    List<NamespaceBundle> bundleList = updatedNsBundles.getBundles();
    assertNotNull(bundles);
    NamespaceBundleFactory utilityFactory = NamespaceBundleFactory.createFactory(Hashing.crc32());
    // (1) validate bundleFactory-cache has newly split bundles and removed old parent bundle
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = splitBundles(utilityFactory, nsname, bundles, originalBundle);
    assertNotNull(splitBundles);
    Set<NamespaceBundle> splitBundleSet = new HashSet<>(splitBundles.getRight());
    splitBundleSet.removeAll(bundleList);
    assertTrue(splitBundleSet.isEmpty());
    // (2) validate LocalZookeeper policies updated with newly created split
    // bundles
    String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString());
    byte[] content = this.pulsar.getLocalZkCache().getZooKeeper().getData(path, null, new Stat());
    Policies policies = ObjectMapperFactory.getThreadLocal().readValue(content, Policies.class);
    NamespaceBundles localZkBundles = bundleFactory.getBundles(nsname, policies.bundles);
    assertTrue(updatedNsBundles.equals(localZkBundles));
    log.info("Policies: {}", policies);
    // (3) validate ownership of new split bundles by local owner
    bundleList.stream().forEach(b -> {
        try {
            byte[] data = this.pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(b), null, new Stat());
            NamespaceEphemeralData node = ObjectMapperFactory.getThreadLocal().readValue(data, NamespaceEphemeralData.class);
            Assert.assertEquals(node.getNativeUrl(), this.pulsar.getBrokerServiceUrl());
        } catch (Exception e) {
            fail("failed to setup ownership", e);
        }
    });
}

Example 39 with NamespaceBundle

use of com.yahoo.pulsar.common.naming.NamespaceBundle in project pulsar by yahoo.

the class NamespaceServiceTest method testremoveOwnershipNamespaceBundle.

@Test
public void testremoveOwnershipNamespaceBundle() throws Exception {
    OwnershipCache ownershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    ManagedLedger ledger = mock(ManagedLedger.class);
    when(ledger.getCursors()).thenReturn(Lists.newArrayList());
    doNothing().when(ownershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), ownershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("prop/use/ns1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle bundle = bundles.getBundles().get(0);
    assertNotNull(ownershipCache.tryAcquiringOwnership(bundle));
    assertNotNull(ownershipCache.getOwnedBundle(bundle));
    ownershipCache.removeOwnership(bundles).get();
    assertNull(ownershipCache.getOwnedBundle(bundle));
}

Example 40 with NamespaceBundle

use of com.yahoo.pulsar.common.naming.NamespaceBundle in project pulsar by yahoo.

the class OwnershipCacheTest method testGetOrSetOwner.

@Test
public void testGetOrSetOwner() throws Exception {
    OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory);
    NamespaceBundle testFullBundle = bundleFactory.getFullBundle(new NamespaceName("pulsar/test/ns-2"));
    // case 1: no one owns the namespace
    assertFalse(cache.getOwnerAsync(testFullBundle).get().isPresent());
    NamespaceEphemeralData data1 = cache.tryAcquiringOwnership(testFullBundle).get();
    assertEquals(data1.getNativeUrl(), selfBrokerUrl);
    assertTrue(!data1.isDisabled());
    // case 2: the local broker owned the namespace and disabled, getOrSetOwner() should not change it
    OwnedBundle nsObj = cache.getOwnedBundle(testFullBundle);
    // this would disable the ownership
    doReturn(cache).when(nsService).getOwnershipCache();
    nsObj.handleUnloadRequest(pulsar);
    Thread.sleep(1000);
    // case 3: some other broker owned the namespace, getOrSetOwner() should return other broker's URL
    // The only chance that we lost an already existing ephemeral node is when the broker dies or unload has
    // succeeded in both cases, the ownerInfoCache will be updated (i.e. invalidated the entry)
    localCache.ownerInfoCache().invalidate(ServiceUnitZkUtils.path(testFullBundle));
    ServiceUnitZkUtils.acquireNameSpace(zkCache.getZooKeeper(), ServiceUnitZkUtils.path(testFullBundle), new NamespaceEphemeralData("pulsar://otherhost:8881", "pulsar://otherhost:8884", "http://localhost:8080", "https://localhost:4443", false));
    data1 = cache.tryAcquiringOwnership(testFullBundle).get();
    assertEquals(data1.getNativeUrl(), "pulsar://otherhost:8881");
    assertEquals(data1.getNativeUrlTls(), "pulsar://otherhost:8884");
    assertTrue(!data1.isDisabled());
}