Examples with NamespaceBundle org.apache.pulsar.common.naming.NamespaceBundle used on opensource projects

Example 11 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class NamespaceService method registerNamespace.

/**
 * Tried to registers a namespace to this instance
 *
 * @param namespace
 * @param ensureOwned
 * @return
 * @throws PulsarServerException
 * @throws Exception
 */
private boolean registerNamespace(String namespace, boolean ensureOwned) throws PulsarServerException {
    String myUrl = pulsar.getBrokerServiceUrl();
    try {
        NamespaceName nsname = NamespaceName.get(namespace);
        String otherUrl = null;
        NamespaceBundle nsFullBundle = null;
        // all pre-registered namespace is assumed to have bundles disabled
        nsFullBundle = bundleFactory.getFullBundle(nsname);
        // v2 namespace will always use full bundle object
        otherUrl = ownershipCache.tryAcquiringOwnership(nsFullBundle).get().getNativeUrl();
        if (myUrl.equals(otherUrl)) {
            if (nsFullBundle != null) {
                // preload heartbeat namespace
                pulsar.loadNamespaceTopics(nsFullBundle);
            }
            return true;
        }
        String msg = String.format("namespace already owned by other broker : ns=%s expected=%s actual=%s", namespace, myUrl, otherUrl);
        // ignore if not be owned for now
        if (!ensureOwned) {
            LOG.info(msg);
            return false;
        }
        // should not happen
        throw new IllegalStateException(msg);
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new PulsarServerException(e);
    }
}

Example 12 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class NamespaceService method searchForCandidateBroker.

private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture<Optional<LookupResult>> lookupFuture, boolean authoritative) {
    String candidateBroker = null;
    try {
        // check if this is Heartbeat or SLAMonitor namespace
        candidateBroker = checkHeartbeatNamespace(bundle);
        if (candidateBroker == null) {
            String broker = getSLAMonitorBrokerName(bundle);
            // checking if the broker is up and running
            if (broker != null && isBrokerActive(broker)) {
                candidateBroker = broker;
            }
        }
        if (candidateBroker == null) {
            if (!this.loadManager.get().isCentralized() || pulsar.getLeaderElectionService().isLeader()) {
                Optional<String> availableBroker = getLeastLoadedFromLoadManager(bundle);
                if (!availableBroker.isPresent()) {
                    lookupFuture.complete(Optional.empty());
                    return;
                }
                candidateBroker = availableBroker.get();
            } else {
                if (authoritative) {
                    // leader broker already assigned the current broker as owner
                    candidateBroker = pulsar.getWebServiceAddress();
                } else {
                    // forward to leader broker to make assignment
                    candidateBroker = pulsar.getLeaderElectionService().getCurrentLeader().getServiceUrl();
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
        return;
    }
    try {
        checkNotNull(candidateBroker);
        if (pulsar.getWebServiceAddress().equals(candidateBroker)) {
            // Load manager decided that the local broker should try to become the owner
            ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> {
                if (ownerInfo.isDisabled()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Namespace bundle {} is currently being unloaded", bundle);
                    }
                    lookupFuture.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is currently being unloaded", bundle)));
                } else {
                    // Found owner for the namespace bundle
                    // Schedule the task to pre-load topics
                    pulsar.loadNamespaceTopics(bundle);
                    lookupFuture.complete(Optional.of(new LookupResult(ownerInfo)));
                }
            }).exceptionally(exception -> {
                LOG.warn("Failed to acquire ownership for namespace bundle {}: ", bundle, exception.getMessage(), exception);
                lookupFuture.completeExceptionally(new PulsarServerException("Failed to acquire ownership for namespace bundle " + bundle, exception));
                return null;
            });
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Redirecting to broker {} to acquire ownership of bundle {}", candidateBroker, bundle);
            }
            // Now setting the redirect url
            createLookupResult(candidateBroker).thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))).exceptionally(ex -> {
                lookupFuture.completeExceptionally(ex);
                return null;
            });
        }
    } catch (Exception e) {
        LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
    }
}

Example 13 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class LoadManagerShared method applyNamespacePolicies.

// Determines the brokers available for the given service unit according to the given policies.
// The brokers are put into brokerCandidateCache.
public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, final Set<String> brokerCandidateCache, final Set<String> availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) {
    Set<String> primariesCache = localPrimariesCache.get();
    primariesCache.clear();
    Set<String> secondaryCache = localSecondaryCache.get();
    secondaryCache.clear();
    NamespaceName namespace = serviceUnit.getNamespaceObject();
    boolean isIsolationPoliciesPresent = policies.areIsolationPoliciesPresent(namespace);
    boolean isNonPersistentTopic = (serviceUnit instanceof NamespaceBundle) ? ((NamespaceBundle) serviceUnit).hasNonPersistentTopic() : false;
    if (isIsolationPoliciesPresent) {
        log.debug("Isolation Policies Present for namespace - [{}]", namespace.toString());
    }
    for (final String broker : availableBrokers) {
        final String brokerUrlString = String.format("http://%s", broker);
        URL brokerUrl;
        try {
            brokerUrl = new URL(brokerUrlString);
        } catch (MalformedURLException e) {
            log.error("Unable to parse brokerUrl from ResourceUnitId - [{}]", e);
            continue;
        }
        // the namespace
        if (isIsolationPoliciesPresent) {
            // brokerName
            if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) {
                primariesCache.add(broker);
                if (log.isDebugEnabled()) {
                    log.debug("Added Primary Broker - [{}] as possible Candidates for" + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString());
                }
            } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) {
                secondaryCache.add(broker);
                if (log.isDebugEnabled()) {
                    log.debug("Added Shared Broker - [{}] as possible " + "Candidates for namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString());
                }
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("Skipping Broker - [{}] not primary broker and not shared" + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString());
                }
            }
        } else {
            // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic
            if (isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) {
                if (log.isDebugEnabled()) {
                    log.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            } else if (!isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) {
                // persistent topic can be assigned to only brokers that enabled for persistent-topic
                if (log.isDebugEnabled()) {
                    log.debug("Filter broker- [{}] because broker only supports non-persistent namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            } else if (policies.isSharedBroker(brokerUrl.getHost())) {
                secondaryCache.add(broker);
                if (log.isDebugEnabled()) {
                    log.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            }
        }
    }
    if (isIsolationPoliciesPresent) {
        brokerCandidateCache.addAll(primariesCache);
        if (policies.shouldFailoverToSecondaries(namespace, primariesCache.size())) {
            log.debug("Not enough of primaries [{}] available for namespace - [{}], " + "adding shared [{}] as possible candidate owners", primariesCache.size(), namespace.toString(), secondaryCache.size());
            brokerCandidateCache.addAll(secondaryCache);
        }
    } else {
        log.debug("Policies not present for namespace - [{}] so only " + "considering shared [{}] brokers for possible owner", namespace.toString(), secondaryCache.size());
        brokerCandidateCache.addAll(secondaryCache);
    }
}

Example 14 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testEvenBundleDistribution.

// Test that bundles belonging to the same namespace are distributed evenly among brokers.
// Test disabled since it's depending on CPU usage in the machine
@Test(enabled = false)
public void testEvenBundleDistribution() throws Exception {
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(nsFactory, "test", "test", "test", 16);
    int numAssignedToPrimary = 0;
    int numAssignedToSecondary = 0;
    final BundleData bundleData = new BundleData(10, 1000);
    final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000);
    longTermMessageData.setMsgRateIn(1000);
    bundleData.setLongTermData(longTermMessageData);
    final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_ZPATH, bundles[0]);
    // Write long message rate for first bundle to ensure that even bundle distribution is not a coincidence of
    // balancing by message rate. If we were balancing by message rate, one of the brokers should only have this
    // one bundle.
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), firstBundleDataPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    for (final NamespaceBundle bundle : bundles) {
        if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryHost)) {
            ++numAssignedToPrimary;
        } else {
            ++numAssignedToSecondary;
        }
        if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) {
            // On even number of assignments, assert that an equal number of bundles have been assigned between
            // them.
            assertEquals(numAssignedToPrimary, numAssignedToSecondary);
        }
    }
}

Example 15 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testMaxTopicDistributionToBroker.

/**
 * It verifies that once broker owns max-number of topics: load-manager doesn't allocates new bundles to that broker
 * unless all the brokers are in same state.
 *
 * <pre>
 * 1. Create a bundle whose bundle-resource-quota will contain max-topics
 * 2. Load-manager assigns broker to this bundle so, assigned broker is overloaded with max-topics
 * 3. For any new further bundles: broker assigns different brokers.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testMaxTopicDistributionToBroker() throws Exception {
    final int totalBundles = 50;
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(nsFactory, "test", "test", "test", totalBundles);
    final BundleData bundleData = new BundleData(10, 1000);
    // it sets max topics under this bundle so, owner of this broker reaches max-topic threshold
    bundleData.setTopics(pulsar1.getConfiguration().getLoadBalancerBrokerMaxTopics() + 10);
    final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000);
    longTermMessageData.setMsgRateIn(1000);
    bundleData.setLongTermData(longTermMessageData);
    final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_ZPATH, bundles[0]);
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), firstBundleDataPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    String maxTopicOwnedBroker = primaryLoadManager.selectBrokerForAssignment(bundles[0]).get();
    for (int i = 1; i < totalBundles; i++) {
        assertNotEquals(primaryLoadManager.selectBrokerForAssignment(bundles[i]), maxTopicOwnedBroker);
    }
}