Search in sources :

Example 1 with LookupResult

use of com.yahoo.pulsar.broker.lookup.LookupResult in project pulsar by yahoo.

the class NamespaceService method searchForCandidateBroker.

private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture<LookupResult> lookupFuture, boolean authoritative) {
    String candidateBroker = null;
    try {
        // check if this is Heartbeat or SLAMonitor namespace
        candidateBroker = checkHeartbeatNamespace(bundle);
        if (candidateBroker == null) {
            String broker = getSLAMonitorBrokerName(bundle);
            // checking if the broker is up and running
            if (broker != null && isBrokerActive(broker)) {
                candidateBroker = broker;
            }
        }
        if (candidateBroker == null) {
            if (!this.loadManager.isCentralized() || pulsar.getLeaderElectionService().isLeader()) {
                candidateBroker = getLeastLoadedFromLoadManager(bundle);
            } else {
                if (authoritative) {
                    // leader broker already assigned the current broker as owner
                    candidateBroker = pulsar.getWebServiceAddress();
                } else {
                    // forward to leader broker to make assignment
                    candidateBroker = pulsar.getLeaderElectionService().getCurrentLeader().getServiceUrl();
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
        return;
    }
    try {
        checkNotNull(candidateBroker);
        if (pulsar.getWebServiceAddress().equals(candidateBroker)) {
            // Load manager decided that the local broker should try to become the owner
            ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> {
                if (ownerInfo.isDisabled()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Namespace bundle {} is currently being unloaded", bundle);
                    }
                    lookupFuture.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is currently being unloaded", bundle)));
                } else {
                    pulsar.loadNamespaceDestinations(bundle);
                    lookupFuture.complete(new LookupResult(ownerInfo));
                }
            }).exceptionally(exception -> {
                LOG.warn("Failed to acquire ownership for namespace bundle {}: ", bundle, exception.getMessage(), exception);
                lookupFuture.completeExceptionally(new PulsarServerException("Failed to acquire ownership for namespace bundle " + bundle, exception));
                return null;
            });
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Redirecting to broker {} to acquire ownership of bundle {}", candidateBroker, bundle);
            }
            // Now setting the redirect url
            createLookupResult(candidateBroker).thenAccept(lookupResult -> lookupFuture.complete(lookupResult)).exceptionally(ex -> {
                lookupFuture.completeExceptionally(ex);
                return null;
            });
        }
    } catch (Exception e) {
        LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
    }
}
Also used : URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) StringUtils(org.apache.commons.lang3.StringUtils) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Matcher(java.util.regex.Matcher) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) PulsarService(com.yahoo.pulsar.broker.PulsarService) URI(java.net.URI) LocalPolicies(com.yahoo.pulsar.common.policies.data.LocalPolicies) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) NamespaceBundleFactory.getBundlesData(com.yahoo.pulsar.common.naming.NamespaceBundleFactory.getBundlesData) Set(java.util.Set) StatCallback(org.apache.zookeeper.AsyncCallback.StatCallback) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ObjectMapperFactory(com.yahoo.pulsar.common.util.ObjectMapperFactory) NamespaceIsolationPolicies(com.yahoo.pulsar.common.policies.impl.NamespaceIsolationPolicies) List(java.util.List) NamespaceBundles(com.yahoo.pulsar.common.naming.NamespaceBundles) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) LOCAL_POLICIES_ROOT(com.yahoo.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT) PulsarWebResource.joinPath(com.yahoo.pulsar.broker.web.PulsarWebResource.joinPath) AdminResource.jsonMapper(com.yahoo.pulsar.broker.admin.AdminResource.jsonMapper) LoadManager(com.yahoo.pulsar.broker.loadbalance.LoadManager) BundlesData(com.yahoo.pulsar.common.policies.data.BundlesData) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Hashing(com.google.common.hash.Hashing) LookupData(com.yahoo.pulsar.common.lookup.data.LookupData) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) NamespaceIsolationPolicy(com.yahoo.pulsar.common.policies.NamespaceIsolationPolicy) Lists(com.google.common.collect.Lists) LoadReport(com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport) Deserializer(com.yahoo.pulsar.zookeeper.ZooKeeperCache.Deserializer) Codec(com.yahoo.pulsar.common.util.Codec) NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) BrokerAssignment(com.yahoo.pulsar.common.policies.data.BrokerAssignment) ServiceUnitId(com.yahoo.pulsar.common.naming.ServiceUnitId) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) LookupResult(com.yahoo.pulsar.broker.lookup.LookupResult) PulsarAdmin(com.yahoo.pulsar.client.admin.PulsarAdmin) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) NamespaceOwnershipStatus(com.yahoo.pulsar.common.policies.data.NamespaceOwnershipStatus) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException) LookupResult(com.yahoo.pulsar.broker.lookup.LookupResult) KeeperException(org.apache.zookeeper.KeeperException) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException)

Example 2 with LookupResult

use of com.yahoo.pulsar.broker.lookup.LookupResult in project pulsar by yahoo.

the class NamespaceService method findBrokerServiceUrl.

/**
     * Main internal method to lookup and setup ownership of service unit to a broker
     *
     * @param bundle
     * @param authoritative
     * @param readOnly
     * @return
     * @throws PulsarServerException
     */
private CompletableFuture<LookupResult> findBrokerServiceUrl(NamespaceBundle bundle, boolean authoritative, boolean readOnly) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("findBrokerServiceUrl: {} - read-only: {}", bundle, readOnly);
    }
    CompletableFuture<LookupResult> future = new CompletableFuture<>();
    // First check if we or someone else already owns the bundle
    ownershipCache.getOwnerAsync(bundle).thenAccept(nsData -> {
        if (!nsData.isPresent()) {
            if (readOnly) {
                future.completeExceptionally(new IllegalStateException(String.format("Can't find owner of ServiceUnit: %s", bundle)));
            } else {
                pulsar.getExecutor().execute(() -> {
                    searchForCandidateBroker(bundle, future, authoritative);
                });
            }
        } else if (nsData.get().isDisabled()) {
            future.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is being unloaded", bundle)));
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Namespace bundle {} already owned by {} ", bundle, nsData);
            }
            future.complete(new LookupResult(nsData.get()));
        }
    }).exceptionally(exception -> {
        LOG.warn("Failed to check owner for bundle {}: {}", bundle, exception.getMessage(), exception);
        future.completeExceptionally(exception);
        return null;
    });
    return future;
}
Also used : URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) StringUtils(org.apache.commons.lang3.StringUtils) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Matcher(java.util.regex.Matcher) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) PulsarService(com.yahoo.pulsar.broker.PulsarService) URI(java.net.URI) LocalPolicies(com.yahoo.pulsar.common.policies.data.LocalPolicies) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) NamespaceBundleFactory.getBundlesData(com.yahoo.pulsar.common.naming.NamespaceBundleFactory.getBundlesData) Set(java.util.Set) StatCallback(org.apache.zookeeper.AsyncCallback.StatCallback) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ObjectMapperFactory(com.yahoo.pulsar.common.util.ObjectMapperFactory) NamespaceIsolationPolicies(com.yahoo.pulsar.common.policies.impl.NamespaceIsolationPolicies) List(java.util.List) NamespaceBundles(com.yahoo.pulsar.common.naming.NamespaceBundles) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) LOCAL_POLICIES_ROOT(com.yahoo.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT) PulsarWebResource.joinPath(com.yahoo.pulsar.broker.web.PulsarWebResource.joinPath) AdminResource.jsonMapper(com.yahoo.pulsar.broker.admin.AdminResource.jsonMapper) LoadManager(com.yahoo.pulsar.broker.loadbalance.LoadManager) BundlesData(com.yahoo.pulsar.common.policies.data.BundlesData) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Hashing(com.google.common.hash.Hashing) LookupData(com.yahoo.pulsar.common.lookup.data.LookupData) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) NamespaceIsolationPolicy(com.yahoo.pulsar.common.policies.NamespaceIsolationPolicy) Lists(com.google.common.collect.Lists) LoadReport(com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport) Deserializer(com.yahoo.pulsar.zookeeper.ZooKeeperCache.Deserializer) Codec(com.yahoo.pulsar.common.util.Codec) NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) BrokerAssignment(com.yahoo.pulsar.common.policies.data.BrokerAssignment) ServiceUnitId(com.yahoo.pulsar.common.naming.ServiceUnitId) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) LookupResult(com.yahoo.pulsar.broker.lookup.LookupResult) PulsarAdmin(com.yahoo.pulsar.client.admin.PulsarAdmin) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) NamespaceOwnershipStatus(com.yahoo.pulsar.common.policies.data.NamespaceOwnershipStatus) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException) CompletableFuture(java.util.concurrent.CompletableFuture) LookupResult(com.yahoo.pulsar.broker.lookup.LookupResult)

Example 3 with LookupResult

use of com.yahoo.pulsar.broker.lookup.LookupResult in project pulsar by yahoo.

the class NamespaceService method createLookupResult.

private CompletableFuture<LookupResult> createLookupResult(String candidateBroker) throws Exception {
    CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
    try {
        checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null " + candidateBroker);
        URI uri = new URI(candidateBroker);
        String path = String.format("%s/%s:%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, uri.getHost(), uri.getPort());
        pulsar.getLocalZkCache().getDataAsync(path, loadReportDeserializer).thenAccept(reportData -> {
            if (reportData.isPresent()) {
                LoadReport report = reportData.get();
                lookupFuture.complete(new LookupResult(report.getWebServiceUrl(), report.getWebServiceUrlTls(), report.getPulsarServiceUrl(), report.getPulsarServieUrlTls()));
            } else {
                lookupFuture.completeExceptionally(new KeeperException.NoNodeException(path));
            }
        }).exceptionally(ex -> {
            lookupFuture.completeExceptionally(ex);
            return null;
        });
    } catch (Exception e) {
        lookupFuture.completeExceptionally(e);
    }
    return lookupFuture;
}
Also used : URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) StringUtils(org.apache.commons.lang3.StringUtils) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Matcher(java.util.regex.Matcher) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) PulsarService(com.yahoo.pulsar.broker.PulsarService) URI(java.net.URI) LocalPolicies(com.yahoo.pulsar.common.policies.data.LocalPolicies) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) NamespaceBundleFactory.getBundlesData(com.yahoo.pulsar.common.naming.NamespaceBundleFactory.getBundlesData) Set(java.util.Set) StatCallback(org.apache.zookeeper.AsyncCallback.StatCallback) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ObjectMapperFactory(com.yahoo.pulsar.common.util.ObjectMapperFactory) NamespaceIsolationPolicies(com.yahoo.pulsar.common.policies.impl.NamespaceIsolationPolicies) List(java.util.List) NamespaceBundles(com.yahoo.pulsar.common.naming.NamespaceBundles) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) LOCAL_POLICIES_ROOT(com.yahoo.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT) PulsarWebResource.joinPath(com.yahoo.pulsar.broker.web.PulsarWebResource.joinPath) AdminResource.jsonMapper(com.yahoo.pulsar.broker.admin.AdminResource.jsonMapper) LoadManager(com.yahoo.pulsar.broker.loadbalance.LoadManager) BundlesData(com.yahoo.pulsar.common.policies.data.BundlesData) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Hashing(com.google.common.hash.Hashing) LookupData(com.yahoo.pulsar.common.lookup.data.LookupData) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) NamespaceIsolationPolicy(com.yahoo.pulsar.common.policies.NamespaceIsolationPolicy) Lists(com.google.common.collect.Lists) LoadReport(com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport) Deserializer(com.yahoo.pulsar.zookeeper.ZooKeeperCache.Deserializer) Codec(com.yahoo.pulsar.common.util.Codec) NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) BrokerAssignment(com.yahoo.pulsar.common.policies.data.BrokerAssignment) ServiceUnitId(com.yahoo.pulsar.common.naming.ServiceUnitId) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) LookupResult(com.yahoo.pulsar.broker.lookup.LookupResult) PulsarAdmin(com.yahoo.pulsar.client.admin.PulsarAdmin) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) NamespaceOwnershipStatus(com.yahoo.pulsar.common.policies.data.NamespaceOwnershipStatus) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException) CompletableFuture(java.util.concurrent.CompletableFuture) LoadReport(com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport) LookupResult(com.yahoo.pulsar.broker.lookup.LookupResult) URI(java.net.URI) KeeperException(org.apache.zookeeper.KeeperException) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException)

Aggregations

Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)3 Lists (com.google.common.collect.Lists)3 Hashing (com.google.common.hash.Hashing)3 PulsarServerException (com.yahoo.pulsar.broker.PulsarServerException)3 PulsarService (com.yahoo.pulsar.broker.PulsarService)3 ServiceConfiguration (com.yahoo.pulsar.broker.ServiceConfiguration)3 AdminResource (com.yahoo.pulsar.broker.admin.AdminResource)3 AdminResource.jsonMapper (com.yahoo.pulsar.broker.admin.AdminResource.jsonMapper)3 LOCAL_POLICIES_ROOT (com.yahoo.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT)3 LoadManager (com.yahoo.pulsar.broker.loadbalance.LoadManager)3 SimpleLoadManagerImpl (com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl)3 LookupResult (com.yahoo.pulsar.broker.lookup.LookupResult)3 ServiceUnitNotReadyException (com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException)3 PulsarWebResource.joinPath (com.yahoo.pulsar.broker.web.PulsarWebResource.joinPath)3 PulsarAdmin (com.yahoo.pulsar.client.admin.PulsarAdmin)3 LookupData (com.yahoo.pulsar.common.lookup.data.LookupData)3 DestinationName (com.yahoo.pulsar.common.naming.DestinationName)3 NamespaceBundle (com.yahoo.pulsar.common.naming.NamespaceBundle)3 NamespaceBundleFactory (com.yahoo.pulsar.common.naming.NamespaceBundleFactory)3