Examples with NamespaceBundleFactory org.apache.pulsar.common.naming.NamespaceBundleFactory used on opensource projects

Example 1 with NamespaceBundleFactory

use of org.apache.pulsar.common.naming.NamespaceBundleFactory in project incubator-pulsar by apache.

the class NamespaceBundlesTest method testSplitBundleInTwo.

@Test
public void testSplitBundleInTwo() throws Exception {
    final int NO_BUNDLES = 2;
    NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
    TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = factory.getBundles(nsname);
    NamespaceBundle bundle = bundles.findBundle(topicName);
    // (1) split : [0x00000000,0xffffffff] => [0x00000000_0x7fffffff,0x7fffffff_0xffffffff]
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = factory.splitBundles(bundle, NO_BUNDLES);
    assertNotNull(splitBundles);
    assertBundleDivideInTwo(bundle, splitBundles.getRight(), NO_BUNDLES);
    // (2) split: [0x00000000,0x7fffffff] => [0x00000000_0x3fffffff,0x3fffffff_0x7fffffff],
    // [0x7fffffff,0xffffffff] => [0x7fffffff_0xbfffffff,0xbfffffff_0xffffffff]
    NamespaceBundleFactory utilityFactory = getNamespaceBundleFactory();
    assertBundles(utilityFactory, nsname, bundle, splitBundles, NO_BUNDLES);
    // (3) split: [0x00000000,0x3fffffff] => [0x00000000_0x1fffffff,0x1fffffff_0x3fffffff],
    // [0x3fffffff,0x7fffffff] => [0x3fffffff_0x5fffffff,0x5fffffff_0x7fffffff]
    Pair<NamespaceBundles, List<NamespaceBundle>> splitChildBundles = splitBundlesUtilFactory(utilityFactory, nsname, splitBundles.getLeft(), splitBundles.getRight().get(0), NO_BUNDLES);
    assertBundles(utilityFactory, nsname, splitBundles.getRight().get(0), splitChildBundles, NO_BUNDLES);
    // (4) split: [0x7fffffff,0xbfffffff] => [0x7fffffff_0x9fffffff,0x9fffffff_0xbfffffff],
    // [0xbfffffff,0xffffffff] => [0xbfffffff_0xdfffffff,0xdfffffff_0xffffffff]
    splitChildBundles = splitBundlesUtilFactory(utilityFactory, nsname, splitBundles.getLeft(), splitBundles.getRight().get(1), NO_BUNDLES);
    assertBundles(utilityFactory, nsname, splitBundles.getRight().get(1), splitChildBundles, NO_BUNDLES);
}

Example 2 with NamespaceBundleFactory

use of org.apache.pulsar.common.naming.NamespaceBundleFactory in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method checkNamespaceBundleSplit.

/**
 * As the leader broker, attempt to automatically detect and split hot namespace bundles.
 */
@Override
public void checkNamespaceBundleSplit() {
    if (!conf.isLoadBalancerAutoBundleSplitEnabled() || pulsar.getLeaderElectionService() == null || !pulsar.getLeaderElectionService().isLeader()) {
        return;
    }
    final boolean unloadSplitBundles = pulsar.getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled();
    synchronized (bundleSplitStrategy) {
        final Set<String> bundlesToBeSplit = bundleSplitStrategy.findBundlesToSplit(loadData, pulsar);
        NamespaceBundleFactory namespaceBundleFactory = pulsar.getNamespaceService().getNamespaceBundleFactory();
        for (String bundleName : bundlesToBeSplit) {
            try {
                final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundleName);
                final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundleName);
                if (!namespaceBundleFactory.canSplitBundle(namespaceBundleFactory.getBundle(namespaceName, bundleRange))) {
                    continue;
                }
                log.info("Load-manager splitting budnle {} and unloading {}", bundleName, unloadSplitBundles);
                pulsar.getAdminClient().namespaces().splitNamespaceBundle(namespaceName, bundleRange, unloadSplitBundles);
                // Make sure the same bundle is not selected again.
                loadData.getBundleData().remove(bundleName);
                localData.getLastStats().remove(bundleName);
                // Clear namespace bundle-cache
                this.pulsar.getNamespaceService().getNamespaceBundleFactory().invalidateBundleCache(NamespaceName.get(namespaceName));
                deleteBundleDataFromZookeeper(bundleName);
                log.info("Successfully split namespace bundle {}", bundleName);
            } catch (Exception e) {
                log.error("Failed to split namespace bundle {}", bundleName, e);
            }
        }
    }
}

Example 3 with NamespaceBundleFactory

use of org.apache.pulsar.common.naming.NamespaceBundleFactory in project incubator-pulsar by apache.

the class NamespaceServiceTest method testCreateNamespaceWithDefaultNumberOfBundles.

@Test
public void testCreateNamespaceWithDefaultNumberOfBundles() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
    TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(topicName);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle, false);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    NamespaceBundleFactory bundleFactory = this.pulsar.getNamespaceService().getNamespaceBundleFactory();
    NamespaceBundles updatedNsBundles = bundleFactory.getBundles(nsname);
    // new updated bundles shouldn't be null
    assertNotNull(updatedNsBundles);
    List<NamespaceBundle> bundleList = updatedNsBundles.getBundles();
    assertNotNull(bundles);
    NamespaceBundleFactory utilityFactory = NamespaceBundleFactory.createFactory(pulsar, Hashing.crc32());
    // (1) validate bundleFactory-cache has newly split bundles and removed old parent bundle
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = splitBundles(utilityFactory, nsname, bundles, originalBundle);
    assertNotNull(splitBundles);
    Set<NamespaceBundle> splitBundleSet = new HashSet<>(splitBundles.getRight());
    splitBundleSet.removeAll(bundleList);
    assertTrue(splitBundleSet.isEmpty());
    // (2) validate LocalZookeeper policies updated with newly created split
    // bundles
    String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString());
    byte[] content = this.pulsar.getLocalZkCache().getZooKeeper().getData(path, null, new Stat());
    Policies policies = ObjectMapperFactory.getThreadLocal().readValue(content, Policies.class);
    NamespaceBundles localZkBundles = bundleFactory.getBundles(nsname, policies.bundles);
    assertTrue(updatedNsBundles.equals(localZkBundles));
    log.info("Policies: {}", policies);
    // (3) validate ownership of new split bundles by local owner
    bundleList.stream().forEach(b -> {
        try {
            byte[] data = this.pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(b), null, new Stat());
            NamespaceEphemeralData node = ObjectMapperFactory.getThreadLocal().readValue(data, NamespaceEphemeralData.class);
            Assert.assertEquals(node.getNativeUrl(), this.pulsar.getBrokerServiceUrl());
        } catch (Exception e) {
            fail("failed to setup ownership", e);
        }
    });
}

Example 4 with NamespaceBundleFactory

use of org.apache.pulsar.common.naming.NamespaceBundleFactory in project incubator-pulsar by apache.

the class NamespaceServiceTest method testSplitAndOwnBundles.

@Test
public void testSplitAndOwnBundles() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
    TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(topicName);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle, false);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    NamespaceBundleFactory bundleFactory = this.pulsar.getNamespaceService().getNamespaceBundleFactory();
    NamespaceBundles updatedNsBundles = bundleFactory.getBundles(nsname);
    // new updated bundles shouldn't be null
    assertNotNull(updatedNsBundles);
    List<NamespaceBundle> bundleList = updatedNsBundles.getBundles();
    assertNotNull(bundles);
    NamespaceBundleFactory utilityFactory = NamespaceBundleFactory.createFactory(pulsar, Hashing.crc32());
    // (1) validate bundleFactory-cache has newly split bundles and removed old parent bundle
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = splitBundles(utilityFactory, nsname, bundles, originalBundle);
    assertNotNull(splitBundles);
    Set<NamespaceBundle> splitBundleSet = new HashSet<>(splitBundles.getRight());
    splitBundleSet.removeAll(bundleList);
    assertTrue(splitBundleSet.isEmpty());
    // (2) validate LocalZookeeper policies updated with newly created split
    // bundles
    String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString());
    byte[] content = this.pulsar.getLocalZkCache().getZooKeeper().getData(path, null, new Stat());
    Policies policies = ObjectMapperFactory.getThreadLocal().readValue(content, Policies.class);
    NamespaceBundles localZkBundles = bundleFactory.getBundles(nsname, policies.bundles);
    assertTrue(updatedNsBundles.equals(localZkBundles));
    log.info("Policies: {}", policies);
    // (3) validate ownership of new split bundles by local owner
    bundleList.stream().forEach(b -> {
        try {
            byte[] data = this.pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(b), null, new Stat());
            NamespaceEphemeralData node = ObjectMapperFactory.getThreadLocal().readValue(data, NamespaceEphemeralData.class);
            Assert.assertEquals(node.getNativeUrl(), this.pulsar.getBrokerServiceUrl());
        } catch (Exception e) {
            fail("failed to setup ownership", e);
        }
    });
}

Example 5 with NamespaceBundleFactory

use of org.apache.pulsar.common.naming.NamespaceBundleFactory in project incubator-pulsar by apache.

the class OwnershipCacheTest method setup.

@BeforeMethod
public void setup() throws Exception {
    final int port = 8080;
    selfBrokerUrl = "tcp://localhost:" + port;
    pulsar = mock(PulsarService.class);
    config = mock(ServiceConfiguration.class);
    executor = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("test").build();
    scheduledExecutor = Executors.newScheduledThreadPool(2);
    zkCache = new LocalZooKeeperCache(MockZooKeeper.newInstance(), executor, scheduledExecutor);
    localCache = spy(new LocalZooKeeperCacheService(zkCache, null));
    ZooKeeperDataCache<LocalPolicies> poilciesCache = mock(ZooKeeperDataCache.class);
    when(pulsar.getLocalZkCacheService()).thenReturn(localCache);
    when(localCache.policiesCache()).thenReturn(poilciesCache);
    doNothing().when(poilciesCache).registerListener(any());
    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
    nsService = mock(NamespaceService.class);
    brokerService = mock(BrokerService.class);
    doReturn(CompletableFuture.completedFuture(1)).when(brokerService).unloadServiceUnit(anyObject());
    doReturn(zkCache).when(pulsar).getLocalZkCache();
    doReturn(localCache).when(pulsar).getLocalZkCacheService();
    doReturn(config).when(pulsar).getConfiguration();
    doReturn(nsService).when(pulsar).getNamespaceService();
    doReturn(port).when(config).getBrokerServicePort();
    doReturn(brokerService).when(pulsar).getBrokerService();
    doReturn(webAddress(config)).when(pulsar).getWebServiceAddress();
    doReturn(selfBrokerUrl).when(pulsar).getBrokerServiceUrl();
}