Examples with NamespaceName org.apache.pulsar.common.naming.NamespaceName used on opensource projects

Example 86 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method doNamespaceBundleSplit.

/**
 * Detect and split hot namespace bundles
 */
@Override
public void doNamespaceBundleSplit() throws Exception {
    int maxBundleCount = pulsar.getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * MBytes;
    log.info("Running namespace bundle split with thresholds: topics {}, sessions {}, msgRate {}, bandwidth {}, maxBundles {}", maxBundleTopics, maxBundleSessions, maxBundleMsgRate, maxBundleBandwidth, maxBundleCount);
    if (this.lastLoadReport == null || this.lastLoadReport.getBundleStats() == null) {
        return;
    }
    Map<String, NamespaceBundleStats> bundleStats = this.lastLoadReport.getBundleStats();
    Set<String> bundlesToBeSplit = new HashSet<>();
    for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
        String bundleName = statsEntry.getKey();
        NamespaceBundleStats stats = statsEntry.getValue();
        long totalSessions = stats.consumerCount + stats.producerCount;
        double totalMsgRate = stats.msgRateIn + stats.msgRateOut;
        double totalBandwidth = stats.msgThroughputIn + stats.msgThroughputOut;
        boolean needSplit = false;
        if (stats.topics > maxBundleTopics || totalSessions > maxBundleSessions || totalMsgRate > maxBundleMsgRate || totalBandwidth > maxBundleBandwidth) {
            if (stats.topics <= 1) {
                log.info("Unable to split hot namespace bundle {} since there is only one topic.", bundleName);
            } else {
                NamespaceName namespaceName = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundleName));
                int numBundles = pulsar.getNamespaceService().getBundleCount(namespaceName);
                if (numBundles >= maxBundleCount) {
                    log.info("Unable to split hot namespace bundle {} since the namespace has too many bundles.", bundleName);
                } else {
                    needSplit = true;
                }
            }
        }
        if (needSplit) {
            if (this.getLoadBalancerAutoBundleSplitEnabled()) {
                log.info("Will split hot namespace bundle {}, topics {}, producers+consumers {}, msgRate in+out {}, bandwidth in+out {}", bundleName, stats.topics, totalSessions, totalMsgRate, totalBandwidth);
                bundlesToBeSplit.add(bundleName);
            } else {
                log.info("DRY RUN - split hot namespace bundle {}, topics {}, producers+consumers {}, msgRate in+out {}, bandwidth in+out {}", bundleName, stats.topics, totalSessions, totalMsgRate, totalBandwidth);
            }
        }
    }
    if (bundlesToBeSplit.size() > 0) {
        for (String bundleName : bundlesToBeSplit) {
            try {
                pulsar.getAdminClient().namespaces().splitNamespaceBundle(LoadManagerShared.getNamespaceNameFromBundleName(bundleName), LoadManagerShared.getBundleRangeFromBundleName(bundleName), pulsar.getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled());
                log.info("Successfully split namespace bundle {}", bundleName);
            } catch (Exception e) {
                log.error("Failed to split namespace bundle {}", bundleName, e);
            }
        }
        this.setLoadReportForceUpdateFlag();
    }
}

Example 87 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method selectBrokerForAssignment.

/**
 * As the leader broker, find a suitable broker for the assignment of the given bundle.
 *
 * @param serviceUnit
 *            ServiceUnitId for the bundle.
 * @return The name of the selected broker, as it appears on ZooKeeper.
 */
@Override
public Optional<String> selectBrokerForAssignment(final ServiceUnitId serviceUnit) {
    // Use brokerCandidateCache as a lock to reduce synchronization.
    synchronized (brokerCandidateCache) {
        final String bundle = serviceUnit.toString();
        if (preallocatedBundleToBroker.containsKey(bundle)) {
            // If the given bundle is already in preallocated, return the selected broker.
            return Optional.of(preallocatedBundleToBroker.get(bundle));
        }
        final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, key -> getBundleDataOrDefault(bundle));
        brokerCandidateCache.clear();
        LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), brokerTopicLoadingPredicate);
        // filter brokers which owns topic higher than threshold
        LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, conf.getLoadBalancerBrokerMaxTopics());
        // distribute namespaces to domain and brokers according to anti-affinity-group
        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, serviceUnit.toString(), brokerCandidateCache, brokerToNamespaceToBundleRange, brokerToFailureDomainMap);
        // distribute bundles evenly to candidate-brokers
        LoadManagerShared.removeMostServicingBrokersForNamespace(serviceUnit.toString(), brokerCandidateCache, brokerToNamespaceToBundleRange);
        log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle);
        // Use the filter pipeline to finalize broker candidates.
        try {
            for (BrokerFilter filter : filterPipeline) {
                filter.filter(brokerCandidateCache, data, loadData, conf);
            }
        } catch (BrokerFilterException x) {
            // restore the list of brokers to the full set
            LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), brokerTopicLoadingPredicate);
        }
        if (brokerCandidateCache.isEmpty()) {
            // restore the list of brokers to the full set
            LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), brokerTopicLoadingPredicate);
        }
        // Choose a broker among the potentially smaller filtered list, when possible
        Optional<String> broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
        if (log.isDebugEnabled()) {
            log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache);
        }
        if (!broker.isPresent()) {
            // No brokers available
            return broker;
        }
        final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0;
        final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage();
        if (maxUsage > overloadThreshold) {
            // All brokers that were in the filtered list were overloaded, so check if there is a better broker
            LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), brokerTopicLoadingPredicate);
            broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
        }
        // Add new bundle to preallocated.
        loadData.getBrokerData().get(broker.get()).getPreallocatedBundleData().put(bundle, data);
        preallocatedBundleToBroker.put(bundle, broker.get());
        final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
        final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
        brokerToNamespaceToBundleRange.get(broker.get()).computeIfAbsent(namespaceName, k -> new HashSet<>()).add(bundleRange);
        return broker;
    }
}

Example 88 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class LoadBalancerTestingUtils method makeBundles.

public static NamespaceBundle[] makeBundles(final NamespaceBundleFactory nsFactory, final String property, final String cluster, final String namespace, final int numBundles) {
    final NamespaceBundle[] result = new NamespaceBundle[numBundles];
    final NamespaceName namespaceName = NamespaceName.get(property, cluster, namespace);
    for (int i = 0; i < numBundles - 1; ++i) {
        final long lower = NamespaceBundles.FULL_UPPER_BOUND * i / numBundles;
        final long upper = NamespaceBundles.FULL_UPPER_BOUND * (i + 1) / numBundles;
        result[i] = nsFactory.getBundle(namespaceName, Range.range(lower, BoundType.CLOSED, upper, BoundType.OPEN));
    }
    result[numBundles - 1] = nsFactory.getBundle(namespaceName, Range.range(NamespaceBundles.FULL_UPPER_BOUND * (numBundles - 1) / numBundles, BoundType.CLOSED, NamespaceBundles.FULL_UPPER_BOUND, BoundType.CLOSED));
    return result;
}

Example 89 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class NamespaceServiceTest method testSplitMapWithRefreshedStatMap.

@Test
public void testSplitMapWithRefreshedStatMap() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    ManagedLedger ledger = mock(ManagedLedger.class);
    when(ledger.getCursors()).thenReturn(Lists.newArrayList());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
    TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(topicName);
    PersistentTopic topic = new PersistentTopic(topicName.toString(), ledger, pulsar.getBrokerService());
    Method method = pulsar.getBrokerService().getClass().getDeclaredMethod("addTopicToStatsMaps", TopicName.class, Topic.class);
    method.setAccessible(true);
    method.invoke(pulsar.getBrokerService(), topicName, topic);
    String nspace = originalBundle.getNamespaceObject().toString();
    List<Topic> list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertNotNull(list);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle, false);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    // old bundle should be removed from status-map
    list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertTrue(list.isEmpty());
    // status-map should be updated with new split bundles
    NamespaceBundle splitBundle = pulsar.getNamespaceService().getBundle(topicName);
    assertTrue(!CollectionUtils.isEmpty(this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, splitBundle.toString())));
}

Example 90 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class OwnershipCacheTest method testGetOwnedServiceUnits.

@Test
public void testGetOwnedServiceUnits() throws Exception {
    OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory);
    NamespaceName testNs = NamespaceName.get("pulsar/test/ns-6");
    NamespaceBundle testBundle = bundleFactory.getFullBundle(testNs);
    // case 1: no one owns the namespace
    assertFalse(cache.getOwnerAsync(testBundle).get().isPresent());
    assertTrue(cache.getOwnedBundles().isEmpty());
    // case 2: someone else owns the namespace
    ServiceUnitZkUtils.acquireNameSpace(zkCache.getZooKeeper(), ServiceUnitZkUtils.path(testBundle), new NamespaceEphemeralData("pulsar://otherhost:8881", "pulsar://otherhost:8884", "http://otherhost:8080", "https://otherhost:4443", false));
    assertTrue(cache.getOwnedBundles().isEmpty());
    Thread.sleep(500);
    // try to acquire, which will load the read-only cache
    NamespaceEphemeralData data1 = cache.tryAcquiringOwnership(testBundle).get();
    assertEquals(data1.getNativeUrl(), "pulsar://otherhost:8881");
    assertEquals(data1.getNativeUrlTls(), "pulsar://otherhost:8884");
    assertTrue(!data1.isDisabled());
    assertTrue(cache.getOwnedBundles().isEmpty());
    // case 3: this broker owns the namespace
    // delete the ephemeral node by others
    zkCache.getZooKeeper().delete(ServiceUnitZkUtils.path(testBundle), -1);
    // force to read directly from ZK
    localCache.ownerInfoCache().invalidate(ServiceUnitZkUtils.path(testBundle));
    data1 = cache.tryAcquiringOwnership(testBundle).get();
    assertEquals(data1.getNativeUrl(), selfBrokerUrl);
    assertTrue(!data1.isDisabled());
    assertTrue(cache.getOwnedBundles().size() == 1);
}