Search in sources :

Example 1 with LookupResult

use of org.apache.pulsar.broker.lookup.LookupResult in project incubator-pulsar by apache.

the class NamespaceService method searchForCandidateBroker.

private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture<Optional<LookupResult>> lookupFuture, boolean authoritative) {
    String candidateBroker = null;
    try {
        // check if this is Heartbeat or SLAMonitor namespace
        candidateBroker = checkHeartbeatNamespace(bundle);
        if (candidateBroker == null) {
            String broker = getSLAMonitorBrokerName(bundle);
            // checking if the broker is up and running
            if (broker != null && isBrokerActive(broker)) {
                candidateBroker = broker;
            }
        }
        if (candidateBroker == null) {
            if (!this.loadManager.get().isCentralized() || pulsar.getLeaderElectionService().isLeader()) {
                Optional<String> availableBroker = getLeastLoadedFromLoadManager(bundle);
                if (!availableBroker.isPresent()) {
                    lookupFuture.complete(Optional.empty());
                    return;
                }
                candidateBroker = availableBroker.get();
            } else {
                if (authoritative) {
                    // leader broker already assigned the current broker as owner
                    candidateBroker = pulsar.getWebServiceAddress();
                } else {
                    // forward to leader broker to make assignment
                    candidateBroker = pulsar.getLeaderElectionService().getCurrentLeader().getServiceUrl();
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
        return;
    }
    try {
        checkNotNull(candidateBroker);
        if (pulsar.getWebServiceAddress().equals(candidateBroker)) {
            // Load manager decided that the local broker should try to become the owner
            ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> {
                if (ownerInfo.isDisabled()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Namespace bundle {} is currently being unloaded", bundle);
                    }
                    lookupFuture.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is currently being unloaded", bundle)));
                } else {
                    // Found owner for the namespace bundle
                    // Schedule the task to pre-load topics
                    pulsar.loadNamespaceTopics(bundle);
                    lookupFuture.complete(Optional.of(new LookupResult(ownerInfo)));
                }
            }).exceptionally(exception -> {
                LOG.warn("Failed to acquire ownership for namespace bundle {}: ", bundle, exception.getMessage(), exception);
                lookupFuture.completeExceptionally(new PulsarServerException("Failed to acquire ownership for namespace bundle " + bundle, exception));
                return null;
            });
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Redirecting to broker {} to acquire ownership of bundle {}", candidateBroker, bundle);
            }
            // Now setting the redirect url
            createLookupResult(candidateBroker).thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))).exceptionally(ex -> {
                lookupFuture.completeExceptionally(ex);
                return null;
            });
        }
    } catch (Exception e) {
        LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
    }
}
Also used : ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) URL(java.net.URL) AdminResource(org.apache.pulsar.broker.admin.AdminResource) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) LoggerFactory(org.slf4j.LoggerFactory) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) StringUtils(org.apache.commons.lang3.StringUtils) NamespaceBundles(org.apache.pulsar.common.naming.NamespaceBundles) NamespaceIsolationPolicies(org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Matcher(java.util.regex.Matcher) Pair(org.apache.commons.lang3.tuple.Pair) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NamespaceOwnershipStatus(org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus) Map(java.util.Map) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) URI(java.net.URI) NamespaceIsolationPolicy(org.apache.pulsar.common.policies.NamespaceIsolationPolicy) Set(java.util.Set) LocalPolicies(org.apache.pulsar.common.policies.data.LocalPolicies) PulsarWebResource.joinPath(org.apache.pulsar.broker.web.PulsarWebResource.joinPath) StatCallback(org.apache.zookeeper.AsyncCallback.StatCallback) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) NamespaceBundleFactory(org.apache.pulsar.common.naming.NamespaceBundleFactory) List(java.util.List) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) NamespaceBundleFactory.getBundlesData(org.apache.pulsar.common.naming.NamespaceBundleFactory.getBundlesData) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) Code(org.apache.zookeeper.KeeperException.Code) TopicName(org.apache.pulsar.common.naming.TopicName) LOCAL_POLICIES_ROOT(org.apache.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) Hashing(com.google.common.hash.Hashing) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) AtomicReference(java.util.concurrent.atomic.AtomicReference) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) Lists(com.google.common.collect.Lists) BundlesData(org.apache.pulsar.common.policies.data.BundlesData) ZooKeeperCache.cacheTimeOutInSec(org.apache.pulsar.zookeeper.ZooKeeperCache.cacheTimeOutInSec) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) LookupData(org.apache.pulsar.common.lookup.data.LookupData) PulsarService(org.apache.pulsar.broker.PulsarService) ServiceLookupData(org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData) TimeUnit(java.util.concurrent.TimeUnit) LookupResult(org.apache.pulsar.broker.lookup.LookupResult) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SECONDS(java.util.concurrent.TimeUnit.SECONDS) BrokerAssignment(org.apache.pulsar.common.policies.data.BrokerAssignment) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) LookupResult(org.apache.pulsar.broker.lookup.LookupResult) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) KeeperException(org.apache.zookeeper.KeeperException) PulsarServerException(org.apache.pulsar.broker.PulsarServerException)

Example 2 with LookupResult

use of org.apache.pulsar.broker.lookup.LookupResult in project incubator-pulsar by apache.

the class NamespaceService method createLookupResult.

protected CompletableFuture<LookupResult> createLookupResult(String candidateBroker) throws Exception {
    CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
    try {
        checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null " + candidateBroker);
        URI uri = new URI(candidateBroker);
        String path = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri.getHost(), uri.getPort());
        pulsar.getLocalZkCache().getDataAsync(path, pulsar.getLoadManager().get().getLoadReportDeserializer()).thenAccept(reportData -> {
            if (reportData.isPresent()) {
                ServiceLookupData lookupData = reportData.get();
                lookupFuture.complete(new LookupResult(lookupData.getWebServiceUrl(), lookupData.getWebServiceUrlTls(), lookupData.getPulsarServiceUrl(), lookupData.getPulsarServiceUrlTls()));
            } else {
                lookupFuture.completeExceptionally(new KeeperException.NoNodeException(path));
            }
        }).exceptionally(ex -> {
            lookupFuture.completeExceptionally(ex);
            return null;
        });
    } catch (Exception e) {
        lookupFuture.completeExceptionally(e);
    }
    return lookupFuture;
}
Also used : ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) URL(java.net.URL) AdminResource(org.apache.pulsar.broker.admin.AdminResource) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) LoggerFactory(org.slf4j.LoggerFactory) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) StringUtils(org.apache.commons.lang3.StringUtils) NamespaceBundles(org.apache.pulsar.common.naming.NamespaceBundles) NamespaceIsolationPolicies(org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Matcher(java.util.regex.Matcher) Pair(org.apache.commons.lang3.tuple.Pair) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NamespaceOwnershipStatus(org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus) Map(java.util.Map) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) URI(java.net.URI) NamespaceIsolationPolicy(org.apache.pulsar.common.policies.NamespaceIsolationPolicy) Set(java.util.Set) LocalPolicies(org.apache.pulsar.common.policies.data.LocalPolicies) PulsarWebResource.joinPath(org.apache.pulsar.broker.web.PulsarWebResource.joinPath) StatCallback(org.apache.zookeeper.AsyncCallback.StatCallback) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) NamespaceBundleFactory(org.apache.pulsar.common.naming.NamespaceBundleFactory) List(java.util.List) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) NamespaceBundleFactory.getBundlesData(org.apache.pulsar.common.naming.NamespaceBundleFactory.getBundlesData) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) Code(org.apache.zookeeper.KeeperException.Code) TopicName(org.apache.pulsar.common.naming.TopicName) LOCAL_POLICIES_ROOT(org.apache.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) Hashing(com.google.common.hash.Hashing) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) AtomicReference(java.util.concurrent.atomic.AtomicReference) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) Lists(com.google.common.collect.Lists) BundlesData(org.apache.pulsar.common.policies.data.BundlesData) ZooKeeperCache.cacheTimeOutInSec(org.apache.pulsar.zookeeper.ZooKeeperCache.cacheTimeOutInSec) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) LookupData(org.apache.pulsar.common.lookup.data.LookupData) PulsarService(org.apache.pulsar.broker.PulsarService) ServiceLookupData(org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData) TimeUnit(java.util.concurrent.TimeUnit) LookupResult(org.apache.pulsar.broker.lookup.LookupResult) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SECONDS(java.util.concurrent.TimeUnit.SECONDS) BrokerAssignment(org.apache.pulsar.common.policies.data.BrokerAssignment) CompletableFuture(java.util.concurrent.CompletableFuture) LookupResult(org.apache.pulsar.broker.lookup.LookupResult) ServiceLookupData(org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData) URI(java.net.URI) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) KeeperException(org.apache.zookeeper.KeeperException) PulsarServerException(org.apache.pulsar.broker.PulsarServerException)

Example 3 with LookupResult

use of org.apache.pulsar.broker.lookup.LookupResult in project incubator-pulsar by apache.

the class NamespaceServiceTest method testLoadReportDeserialize.

/**
 * <pre>
 *  It verifies that namespace service deserialize the load-report based on load-manager which active.
 *  1. write candidate1- load report using {@link LoadReport} which is used by SimpleLoadManagerImpl
 *  2. Write candidate2- load report using {@link LocalBrokerData} which is used by ModularLoadManagerImpl
 *  3. try to get Lookup Result based on active load-manager
 * </pre>
 * @throws Exception
 */
@Test
public void testLoadReportDeserialize() throws Exception {
    final String candidateBroker1 = "http://localhost:8000";
    final String candidateBroker2 = "http://localhost:3000";
    LoadReport lr = new LoadReport(null, null, candidateBroker1, null);
    LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker2, null);
    URI uri1 = new URI(candidateBroker1);
    URI uri2 = new URI(candidateBroker2);
    String path1 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri1.getHost(), uri1.getPort());
    String path2 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri2.getHost(), uri2.getPort());
    ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), path1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(lr), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), path2, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(ld), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    LookupResult result1 = pulsar.getNamespaceService().createLookupResult(candidateBroker1).get();
    // update to new load mananger
    pulsar.getLoadManager().set(new ModularLoadManagerWrapper(new ModularLoadManagerImpl()));
    LookupResult result2 = pulsar.getNamespaceService().createLookupResult(candidateBroker2).get();
    Assert.assertEquals(result1.getLookupData().getBrokerUrl(), candidateBroker1);
    Assert.assertEquals(result2.getLookupData().getBrokerUrl(), candidateBroker2);
    System.out.println(result2);
}
Also used : LocalBrokerData(org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData) LoadReport(org.apache.pulsar.policies.data.loadbalancer.LoadReport) LookupResult(org.apache.pulsar.broker.lookup.LookupResult) URI(java.net.URI) ModularLoadManagerWrapper(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper) ModularLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl) Test(org.testng.annotations.Test)

Aggregations

URI (java.net.URI)3 LookupResult (org.apache.pulsar.broker.lookup.LookupResult)3 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)2 Lists (com.google.common.collect.Lists)2 Hashing (com.google.common.hash.Hashing)2 String.format (java.lang.String.format)2 URL (java.net.URL)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2 Optional (java.util.Optional)2 Set (java.util.Set)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 TimeUnit (java.util.concurrent.TimeUnit)2 SECONDS (java.util.concurrent.TimeUnit.SECONDS)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 Matcher (java.util.regex.Matcher)2 Pattern (java.util.regex.Pattern)2