Examples with LeaderBroker org.apache.pulsar.broker.loadbalance.LeaderBroker used on opensource projects

Example 1 with LeaderBroker

use of org.apache.pulsar.broker.loadbalance.LeaderBroker in project pulsar by apache.

the class NamespaceService method searchForCandidateBroker.

private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture<Optional<LookupResult>> lookupFuture, LookupOptions options) {
    if (null == pulsar.getLeaderElectionService()) {
        LOG.warn("The leader election has not yet been completed! NamespaceBundle[{}]", bundle);
        lookupFuture.completeExceptionally(new IllegalStateException("The leader election has not yet been completed!"));
        return;
    }
    String candidateBroker = null;
    LeaderElectionService les = pulsar.getLeaderElectionService();
    if (les == null) {
        // The leader election service was not initialized yet. This can happen because the broker service is
        // initialized first and it might start receiving lookup requests before the leader election service is
        // fully initialized.
        LOG.warn("Leader election service isn't initialized yet. " + "Returning empty result to lookup. NamespaceBundle[{}]", bundle);
        lookupFuture.complete(Optional.empty());
        return;
    }
    boolean authoritativeRedirect = les.isLeader();
    try {
        // check if this is Heartbeat or SLAMonitor namespace
        candidateBroker = checkHeartbeatNamespace(bundle);
        if (candidateBroker == null) {
            candidateBroker = checkHeartbeatNamespaceV2(bundle);
        }
        if (candidateBroker == null) {
            String broker = getSLAMonitorBrokerName(bundle);
            // checking if the broker is up and running
            if (broker != null && isBrokerActive(broker)) {
                candidateBroker = broker;
            }
        }
        if (candidateBroker == null) {
            Optional<LeaderBroker> currentLeader = pulsar.getLeaderElectionService().getCurrentLeader();
            if (options.isAuthoritative()) {
                // leader broker already assigned the current broker as owner
                candidateBroker = pulsar.getSafeWebServiceAddress();
            } else {
                LoadManager loadManager = this.loadManager.get();
                boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader();
                if (!makeLoadManagerDecisionOnThisBroker) {
                    // If leader is not active, fallback to pick the least loaded from current broker loadmanager
                    boolean leaderBrokerActive = currentLeader.isPresent() && isBrokerActive(currentLeader.get().getServiceUrl());
                    if (!leaderBrokerActive) {
                        makeLoadManagerDecisionOnThisBroker = true;
                        if (!currentLeader.isPresent()) {
                            LOG.warn("The information about the current leader broker wasn't available. " + "Handling load manager decisions in a decentralized way. " + "NamespaceBundle[{}]", bundle);
                        } else {
                            LOG.warn("The current leader broker {} isn't active. " + "Handling load manager decisions in a decentralized way. " + "NamespaceBundle[{}]", currentLeader.get(), bundle);
                        }
                    }
                }
                if (makeLoadManagerDecisionOnThisBroker) {
                    Optional<String> availableBroker = getLeastLoadedFromLoadManager(bundle);
                    if (!availableBroker.isPresent()) {
                        LOG.warn("Load manager didn't return any available broker. " + "Returning empty result to lookup. NamespaceBundle[{}]", bundle);
                        lookupFuture.complete(Optional.empty());
                        return;
                    }
                    candidateBroker = availableBroker.get();
                    authoritativeRedirect = true;
                } else {
                    // forward to leader broker to make assignment
                    candidateBroker = currentLeader.get().getServiceUrl();
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
        return;
    }
    try {
        checkNotNull(candidateBroker);
        if (candidateBroker.equals(pulsar.getSafeWebServiceAddress())) {
            // Load manager decided that the local broker should try to become the owner
            ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> {
                if (ownerInfo.isDisabled()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Namespace bundle {} is currently being unloaded", bundle);
                    }
                    lookupFuture.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is currently being unloaded", bundle)));
                } else {
                    if (options.isLoadTopicsInBundle()) {
                        // Schedule the task to pre-load topics
                        pulsar.loadNamespaceTopics(bundle);
                    }
                    // find the target
                    if (options.hasAdvertisedListenerName()) {
                        AdvertisedListener listener = ownerInfo.getAdvertisedListeners().get(options.getAdvertisedListenerName());
                        if (listener == null) {
                            lookupFuture.completeExceptionally(new PulsarServerException("the broker do not have " + options.getAdvertisedListenerName() + " listener"));
                            return;
                        } else {
                            URI url = listener.getBrokerServiceUrl();
                            URI urlTls = listener.getBrokerServiceUrlTls();
                            lookupFuture.complete(Optional.of(new LookupResult(ownerInfo, url == null ? null : url.toString(), urlTls == null ? null : urlTls.toString())));
                            return;
                        }
                    } else {
                        lookupFuture.complete(Optional.of(new LookupResult(ownerInfo)));
                        return;
                    }
                }
            }).exceptionally(exception -> {
                LOG.warn("Failed to acquire ownership for namespace bundle {}: {}", bundle, exception);
                lookupFuture.completeExceptionally(new PulsarServerException("Failed to acquire ownership for namespace bundle " + bundle, exception));
                return null;
            });
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Redirecting to broker {} to acquire ownership of bundle {}", candidateBroker, bundle);
            }
            // Now setting the redirect url
            createLookupResult(candidateBroker, authoritativeRedirect, options.getAdvertisedListenerName()).thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))).exceptionally(ex -> {
                lookupFuture.completeExceptionally(ex);
                return null;
            });
        }
    } catch (Exception e) {
        LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
    }
}

Example 2 with LeaderBroker

use of org.apache.pulsar.broker.loadbalance.LeaderBroker in project pulsar by yahoo.

the class AdminApiTest method brokers.

@Test
public void brokers() throws Exception {
    List<String> list = admin.brokers().getActiveBrokers("test");
    Assert.assertNotNull(list);
    Assert.assertEquals(list.size(), 1);
    List<String> list2 = otheradmin.brokers().getActiveBrokers("test");
    Assert.assertNotNull(list2);
    Assert.assertEquals(list2.size(), 1);
    BrokerInfo leaderBroker = admin.brokers().getLeaderBroker();
    Assert.assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get());
    Map<String, NamespaceOwnershipStatus> nsMap = admin.brokers().getOwnedNamespaces("test", list.get(0));
    // since sla-monitor ns is not created nsMap.size() == 1 (for HeartBeat Namespace)
    Assert.assertEquals(nsMap.size(), 2);
    for (String ns : nsMap.keySet()) {
        NamespaceOwnershipStatus nsStatus = nsMap.get(ns);
        if (ns.equals(NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) {
            assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared);
            assertFalse(nsStatus.is_controlled);
            assertTrue(nsStatus.is_active);
        }
    }
    String[] parts = list.get(0).split(":");
    Assert.assertEquals(parts.length, 2);
    Map<String, NamespaceOwnershipStatus> nsMap2 = adminTls.brokers().getOwnedNamespaces("test", String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get()));
    Assert.assertEquals(nsMap2.size(), 2);
    admin.namespaces().deleteNamespace("prop-xyz/ns1");
    admin.clusters().deleteCluster("test");
    assertEquals(admin.clusters().getClusters(), Lists.newArrayList());
}

Example 3 with LeaderBroker

use of org.apache.pulsar.broker.loadbalance.LeaderBroker in project pulsar by yahoo.

the class AdminTest method brokers.

@Test
@SuppressWarnings("unchecked")
public void brokers() throws Exception {
    asyncRequests(ctx -> clusters.createCluster(ctx, "use", ClusterDataImpl.builder().serviceUrl("http://broker.messaging.use.example.com").serviceUrlTls("https://broker.messaging.use.example.com:4443").build()));
    URI requestUri = new URI("http://broker.messaging.use.example.com:8080/admin/brokers/use");
    UriInfo mockUri = mock(UriInfo.class);
    doReturn(requestUri).when(mockUri).getRequestUri();
    Field uriField = PulsarWebResource.class.getDeclaredField("uri");
    uriField.setAccessible(true);
    uriField.set(brokers, mockUri);
    Object res = asyncRequests(ctx -> brokers.getActiveBrokers(ctx, "use"));
    assertTrue(res instanceof Set);
    Set<String> activeBrokers = (Set<String>) res;
    assertEquals(activeBrokers.size(), 1);
    assertEquals(activeBrokers, Sets.newHashSet(pulsar.getAdvertisedAddress() + ":" + pulsar.getListenPortHTTP().get()));
    Object leaderBrokerRes = asyncRequests(ctx -> brokers.getLeaderBroker(ctx));
    assertTrue(leaderBrokerRes instanceof BrokerInfo);
    BrokerInfo leaderBroker = (BrokerInfo) leaderBrokerRes;
    assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get());
}

Example 4 with LeaderBroker

use of org.apache.pulsar.broker.loadbalance.LeaderBroker in project pulsar by yahoo.

the class AdminApiMultiBrokersTest method testGetLeaderBroker.

@Test(timeOut = 30 * 1000)
public void testGetLeaderBroker() throws ExecutionException, InterruptedException, PulsarAdminException {
    List<PulsarService> allBrokers = getAllBrokers();
    Optional<LeaderBroker> leaderBroker = allBrokers.get(0).getLeaderElectionService().readCurrentLeader().get();
    assertTrue(leaderBroker.isPresent());
    log.info("Leader broker is {}", leaderBroker);
    for (PulsarAdmin admin : getAllAdmins()) {
        String serviceUrl = admin.brokers().getLeaderBroker().getServiceUrl();
        log.info("Pulsar admin get leader broker is {}", serviceUrl);
        assertEquals(leaderBroker.get().getServiceUrl(), serviceUrl);
    }
}

Example 5 with LeaderBroker

use of org.apache.pulsar.broker.loadbalance.LeaderBroker in project incubator-pulsar by apache.

the class AdminApiTest method brokers.

@Test
public void brokers() throws Exception {
    List<String> list = admin.brokers().getActiveBrokers("test");
    Assert.assertNotNull(list);
    Assert.assertEquals(list.size(), 1);
    List<String> list2 = otheradmin.brokers().getActiveBrokers("test");
    Assert.assertNotNull(list2);
    Assert.assertEquals(list2.size(), 1);
    BrokerInfo leaderBroker = admin.brokers().getLeaderBroker();
    Assert.assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get());
    Map<String, NamespaceOwnershipStatus> nsMap = admin.brokers().getOwnedNamespaces("test", list.get(0));
    // since sla-monitor ns is not created nsMap.size() == 1 (for HeartBeat Namespace)
    Assert.assertEquals(nsMap.size(), 2);
    for (String ns : nsMap.keySet()) {
        NamespaceOwnershipStatus nsStatus = nsMap.get(ns);
        if (ns.equals(NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) {
            assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared);
            assertFalse(nsStatus.is_controlled);
            assertTrue(nsStatus.is_active);
        }
    }
    String[] parts = list.get(0).split(":");
    Assert.assertEquals(parts.length, 2);
    Map<String, NamespaceOwnershipStatus> nsMap2 = adminTls.brokers().getOwnedNamespaces("test", String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get()));
    Assert.assertEquals(nsMap2.size(), 2);
    admin.namespaces().deleteNamespace("prop-xyz/ns1");
    admin.clusters().deleteCluster("test");
    assertEquals(admin.clusters().getClusters(), Lists.newArrayList());
}