Examples with NamespaceName org.apache.pulsar.common.naming.NamespaceName used on opensource projects

Example 36 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class LoadManagerShared method applyNamespacePolicies.

// Determines the brokers available for the given service unit according to the given policies.
// The brokers are put into brokerCandidateCache.
public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, final Set<String> brokerCandidateCache, final Set<String> availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) {
    Set<String> primariesCache = localPrimariesCache.get();
    primariesCache.clear();
    Set<String> secondaryCache = localSecondaryCache.get();
    secondaryCache.clear();
    NamespaceName namespace = serviceUnit.getNamespaceObject();
    boolean isIsolationPoliciesPresent = policies.areIsolationPoliciesPresent(namespace);
    boolean isNonPersistentTopic = (serviceUnit instanceof NamespaceBundle) ? ((NamespaceBundle) serviceUnit).hasNonPersistentTopic() : false;
    if (isIsolationPoliciesPresent) {
        log.debug("Isolation Policies Present for namespace - [{}]", namespace.toString());
    }
    for (final String broker : availableBrokers) {
        final String brokerUrlString = String.format("http://%s", broker);
        URL brokerUrl;
        try {
            brokerUrl = new URL(brokerUrlString);
        } catch (MalformedURLException e) {
            log.error("Unable to parse brokerUrl from ResourceUnitId - [{}]", e);
            continue;
        }
        // the namespace
        if (isIsolationPoliciesPresent) {
            // brokerName
            if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) {
                primariesCache.add(broker);
                if (log.isDebugEnabled()) {
                    log.debug("Added Primary Broker - [{}] as possible Candidates for" + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString());
                }
            } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) {
                secondaryCache.add(broker);
                if (log.isDebugEnabled()) {
                    log.debug("Added Shared Broker - [{}] as possible " + "Candidates for namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString());
                }
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("Skipping Broker - [{}] not primary broker and not shared" + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString());
                }
            }
        } else {
            // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic
            if (isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) {
                if (log.isDebugEnabled()) {
                    log.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            } else if (!isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) {
                // persistent topic can be assigned to only brokers that enabled for persistent-topic
                if (log.isDebugEnabled()) {
                    log.debug("Filter broker- [{}] because broker only supports non-persistent namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            } else if (policies.isSharedBroker(brokerUrl.getHost())) {
                secondaryCache.add(broker);
                if (log.isDebugEnabled()) {
                    log.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            }
        }
    }
    if (isIsolationPoliciesPresent) {
        brokerCandidateCache.addAll(primariesCache);
        if (policies.shouldFailoverToSecondaries(namespace, primariesCache.size())) {
            log.debug("Not enough of primaries [{}] available for namespace - [{}], " + "adding shared [{}] as possible candidate owners", primariesCache.size(), namespace.toString(), secondaryCache.size());
            brokerCandidateCache.addAll(secondaryCache);
        }
    } else {
        log.debug("Policies not present for namespace - [{}] so only " + "considering shared [{}] brokers for possible owner", namespace.toString(), secondaryCache.size());
        brokerCandidateCache.addAll(secondaryCache);
    }
}

Example 37 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method findBrokerForPlacement.

/**
 * Assign owner for specified ServiceUnit from the given candidates, following the the principles: 1) Optimum
 * distribution: fill up one broker till its load reaches optimum level (defined by underload threshold) before pull
 * another idle broker in; 2) Even distribution: once all brokers' load are above optimum level, maintain all
 * brokers to have even load; 3) Set the underload threshold to small value (like 1) for pure even distribution, and
 * high value (like 80) for pure optimum distribution;
 *
 * Strategy to select broker: 1) The first choice is the least loaded broker which is underload but not idle; 2) The
 * second choice is idle broker (if there is any); 3) Othewise simply select the least loaded broker if it is NOT
 * overloaded; 4) If all brokers are overloaded, select the broker with maximum available capacity (considering
 * brokers could have different hardware configuration, this usually means to select the broker with more hardware
 * resource);
 *
 * Broker's load level: 1) Load ranking (triggered by LoadReport update) estimate the load level according to the
 * resourse usage and namespace bundles already loaded by each broker; 2) When leader broker decide the owner for a
 * new namespace bundle, it may take time for the real owner to actually load the bundle and refresh LoadReport,
 * leader broker will store the bundle in a list called preAllocatedBundles, and the quota of all
 * preAllocatedBundles in preAllocatedQuotas, and re-estimate the broker's load level by putting the
 * preAllocatedQuota into calculation; 3) Everything (preAllocatedBundles and preAllocatedQuotas) will get reset in
 * load ranking.
 */
private synchronized ResourceUnit findBrokerForPlacement(Multimap<Long, ResourceUnit> candidates, ServiceUnitId serviceUnit) {
    long underloadThreshold = this.getLoadBalancerBrokerUnderloadedThresholdPercentage();
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    ResourceQuota defaultQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
    double minLoadPercentage = 101.0;
    long maxAvailability = -1;
    ResourceUnit idleRU = null;
    ResourceUnit maxAvailableRU = null;
    ResourceUnit randomRU = null;
    ResourceUnit selectedRU = null;
    ResourceUnitRanking selectedRanking = null;
    String serviceUnitId = serviceUnit.toString();
    // If the ranking is expected to be in the range [0,100] (which is the case for LOADBALANCER_STRATEGY_LLS),
    // the ranks are bounded. Otherwise (as is the case in LOADBALANCER_STRATEGY_LEAST_MSG, the ranks are simply
    // the total message rate which is in the range [0,Infinity) so they are unbounded. The
    // "boundedness" affects how two ranks are compared to see which one is better
    boolean unboundedRanks = getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LEAST_MSG);
    long randomBrokerIndex = (candidates.size() > 0) ? (this.brokerRotationCursor % candidates.size()) : 0;
    // find the least loaded & not-idle broker
    for (Map.Entry<Long, ResourceUnit> candidateOwner : candidates.entries()) {
        ResourceUnit candidate = candidateOwner.getValue();
        randomBrokerIndex--;
        // skip broker which is not ranked. this should never happen except in unit test
        if (!resourceUnitRankings.containsKey(candidate)) {
            continue;
        }
        String resourceUnitId = candidate.getResourceId();
        ResourceUnitRanking ranking = resourceUnitRankings.get(candidate);
        // check if this ServiceUnit is already loaded
        if (ranking.isServiceUnitLoaded(serviceUnitId)) {
            ranking.removeLoadedServiceUnit(serviceUnitId, this.getResourceQuota(serviceUnitId));
        }
        // record a random broker
        if (randomBrokerIndex < 0 && randomRU == null) {
            randomRU = candidate;
        }
        // check the available capacity
        double loadPercentage = ranking.getEstimatedLoadPercentage();
        double availablePercentage = Math.max(0, (100 - loadPercentage) / 100);
        long availability = (long) (ranking.estimateMaxCapacity(defaultQuota) * availablePercentage);
        if (availability > maxAvailability) {
            maxAvailability = availability;
            maxAvailableRU = candidate;
        }
        // check the load percentage
        if (ranking.isIdle()) {
            if (idleRU == null) {
                idleRU = candidate;
            }
        } else {
            if (selectedRU == null) {
                selectedRU = candidate;
                selectedRanking = ranking;
                minLoadPercentage = loadPercentage;
            } else {
                if ((unboundedRanks ? ranking.compareMessageRateTo(selectedRanking) : ranking.compareTo(selectedRanking)) < 0) {
                    minLoadPercentage = loadPercentage;
                    selectedRU = candidate;
                    selectedRanking = ranking;
                }
            }
        }
    }
    if ((minLoadPercentage > underloadThreshold && idleRU != null) || selectedRU == null) {
        // assigned to idle broker is the least loaded broker already have optimum load (which means NOT
        // underloaded), or all brokers are idle
        selectedRU = idleRU;
    } else if (minLoadPercentage >= 100.0 && randomRU != null && !unboundedRanks) {
        // all brokers are full, assign to a random one
        selectedRU = randomRU;
    } else if (minLoadPercentage > overloadThreshold && !unboundedRanks) {
        // assign to the broker with maximum available capacity if all brokers are overloaded
        selectedRU = maxAvailableRU;
    }
    // re-calculate load level for selected broker
    if (selectedRU != null) {
        this.brokerRotationCursor = (this.brokerRotationCursor + 1) % 1000000;
        ResourceUnitRanking ranking = resourceUnitRankings.get(selectedRU);
        String loadPercentageDesc = ranking.getEstimatedLoadPercentageString();
        log.info("Assign {} to {} with ({}).", serviceUnitId, selectedRU.getResourceId(), loadPercentageDesc);
        if (!ranking.isServiceUnitPreAllocated(serviceUnitId)) {
            final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(serviceUnitId);
            final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(serviceUnitId);
            ResourceQuota quota = this.getResourceQuota(serviceUnitId);
            // Add preallocated bundle range so incoming bundles from the same namespace are not assigned to the
            // same broker.
            brokerToNamespaceToBundleRange.computeIfAbsent(selectedRU.getResourceId().replace("http://", ""), k -> new HashMap<>()).computeIfAbsent(namespaceName, k -> new HashSet<>()).add(bundleRange);
            ranking.addPreAllocatedServiceUnit(serviceUnitId, quota);
            resourceUnitRankings.put(selectedRU, ranking);
        }
    }
    return selectedRU;
}

Example 38 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class NamespaceNameTest method namespace.

@Test
void namespace() {
    try {
        NamespaceName.get("namespace");
        fail("Should have caused exception");
    } catch (IllegalArgumentException e) {
    // expected
    }
    try {
        NamespaceName.get("property.namespace");
        fail("Should have caused exception");
    } catch (IllegalArgumentException e) {
    // expected
    }
    try {
        NamespaceName.get("0.0.0.0");
        fail("Should have caused exception");
    } catch (IllegalArgumentException e) {
    // expected
    }
    try {
        NamespaceName.get("property.namespace:topic");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("property/cluster/namespace/topic");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get(null);
    } catch (IllegalArgumentException e) {
    // OK
    }
    try {
        NamespaceName.get(null, "use", "ns1");
    } catch (IllegalArgumentException e) {
    // OK
    }
    assertEquals(NamespaceName.get("prop/cluster/ns").getPersistentTopicName("ds"), "persistent://prop/cluster/ns/ds");
    try {
        NamespaceName.get("prop/cluster/ns").getTopicName(null, "ds");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    assertEquals(NamespaceName.get("prop/cluster/ns").getTopicName(TopicDomain.persistent, "ds"), "persistent://prop/cluster/ns/ds");
    assertEquals(NamespaceName.get("prop/cluster/ns"), NamespaceName.get("prop/cluster/ns"));
    assertEquals(NamespaceName.get("prop/cluster/ns").toString(), "prop/cluster/ns");
    assertFalse(NamespaceName.get("prop/cluster/ns").equals("prop/cluster/ns"));
    assertEquals(NamespaceName.get("prop", "cluster", "ns"), NamespaceName.get("prop/cluster/ns"));
    assertEquals(NamespaceName.get("prop/cluster/ns").getProperty(), "prop");
    assertEquals(NamespaceName.get("prop/cluster/ns").getCluster(), "cluster");
    assertEquals(NamespaceName.get("prop/cluster/ns").getLocalName(), "ns");
    try {
        NamespaceName.get("ns").getProperty();
        fail("old style namespace");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("ns").getCluster();
        fail("old style namespace");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("ns").getLocalName();
        fail("old style namespace");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get(null, "cluster", "namespace");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("", "cluster", "namespace");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("/cluster/namespace");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("pulsar//namespace");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("pulsar", null, "namespace");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("pulsar", "", "namespace");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("pulsar", "cluster", null);
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    try {
        NamespaceName.get("pulsar", "cluster", "");
        fail("Should have raised exception");
    } catch (IllegalArgumentException e) {
    // Ok
    }
    NamespaceName v2Namespace = NamespaceName.get("pulsar/colo1/testns-1");
    assertEquals(v2Namespace.getProperty(), "pulsar");
    assertEquals(v2Namespace.getCluster(), "colo1");
    assertEquals(v2Namespace.getLocalName(), "testns-1");
}

Example 39 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class NamespaceNameTest method testNewScheme.

@Test
void testNewScheme() {
    NamespaceName ns = NamespaceName.get("my-tenant/my-namespace");
    assertEquals(ns.getProperty(), "my-tenant");
    assertEquals(ns.getLocalName(), "my-namespace");
    assertEquals(ns.isGlobal(), true);
    assertEquals(ns.getCluster(), null);
    assertEquals(ns.getPersistentTopicName("my-topic"), "persistent://my-tenant/my-namespace/my-topic");
}

Example 40 with NamespaceName

use of org.apache.pulsar.common.naming.NamespaceName in project incubator-pulsar by apache.

the class AbstractWebSocketHandler method extractTopicName.

private TopicName extractTopicName(HttpServletRequest request) {
    String uri = request.getRequestURI();
    List<String> parts = Splitter.on("/").splitToList(uri);
    // V1 Format must be like :
    // /ws/producer/persistent/my-property/my-cluster/my-ns/my-topic
    // or
    // /ws/consumer/persistent/my-property/my-cluster/my-ns/my-topic/my-subscription
    // or
    // /ws/reader/persistent/my-property/my-cluster/my-ns/my-topic
    // V2 Format must be like :
    // /ws/v2/producer/persistent/my-property/my-ns/my-topic
    // or
    // /ws/v2/consumer/persistent/my-property/my-ns/my-topic/my-subscription
    // or
    // /ws/v2/reader/persistent/my-property/my-ns/my-topic
    checkArgument(parts.size() >= 8, "Invalid topic name format");
    checkArgument(parts.get(1).equals("ws"));
    final boolean isV2Format = parts.get(2).equals("v2");
    final int domainIndex = isV2Format ? 4 : 3;
    checkArgument(parts.get(domainIndex).equals("persistent") || parts.get(domainIndex).equals("non-persistent"));
    final String domain = parts.get(domainIndex);
    final NamespaceName namespace = isV2Format ? NamespaceName.get(parts.get(5), parts.get(6)) : NamespaceName.get(parts.get(4), parts.get(5), parts.get(6));
    final String name = parts.get(7);
    return TopicName.get(domain, namespace, name);
}