Examples with AdvertisedListener org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener used on opensource projects

Example 1 with AdvertisedListener

use of org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener in project pulsar by apache.

the class PulsarStandalone method start.

public void start() throws Exception {
    if (config == null) {
        log.error("Failed to load configuration");
        System.exit(1);
    }
    log.debug("--- setup PulsarStandaloneStarter ---");
    if (!this.isOnlyBroker()) {
        ServerConfiguration bkServerConf = new ServerConfiguration();
        bkServerConf.loadConf(new File(configFile).toURI().toURL());
        // Start LocalBookKeeper
        bkEnsemble = new LocalBookkeeperEnsemble(this.getNumOfBk(), this.getZkPort(), this.getBkPort(), this.getStreamStoragePort(), this.getZkDir(), this.getBkDir(), this.isWipeData(), "127.0.0.1");
        bkEnsemble.startStandalone(bkServerConf, !this.isNoStreamStorage());
    }
    if (this.isNoBroker()) {
        return;
    }
    // initialize the functions worker
    if (!this.isNoFunctionsWorker()) {
        workerConfig = PulsarService.initializeWorkerConfigFromBrokerConfig(config, this.getFnWorkerConfigFile());
        // worker talks to local broker
        if (this.isNoStreamStorage()) {
            // only set the state storage service url when state is enabled.
            workerConfig.setStateStorageServiceUrl(null);
        } else if (workerConfig.getStateStorageServiceUrl() == null) {
            workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" + this.getStreamStoragePort());
        }
        fnWorkerService = WorkerServiceLoader.load(workerConfig);
    } else {
        workerConfig = new WorkerConfig();
    }
    // Start Broker
    broker = new PulsarService(config, workerConfig, Optional.ofNullable(fnWorkerService), (exitCode) -> {
        log.info("Halting standalone process with code {}", exitCode);
        LogManager.shutdown();
        Runtime.getRuntime().halt(exitCode);
    });
    broker.start();
    final String cluster = config.getClusterName();
    final AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config);
    if (!config.isTlsEnabled()) {
        checkArgument(config.getWebServicePort().isPresent(), "webServicePort must be present");
        checkArgument(internalListener.getBrokerServiceUrl() != null, "plaintext must be configured on internal listener");
        URL webServiceUrl = new URL(String.format("http://%s:%d", ServiceConfigurationUtils.getWebServiceAddress(config), config.getWebServicePort().get()));
        admin = PulsarAdmin.builder().serviceHttpUrl(webServiceUrl.toString()).authentication(config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters()).build();
        ClusterData clusterData = ClusterData.builder().serviceUrl(webServiceUrl.toString()).brokerServiceUrl(internalListener.getBrokerServiceUrl().toString()).build();
        createSampleNameSpace(clusterData, cluster);
    } else {
        checkArgument(config.getWebServicePortTls().isPresent(), "webServicePortTls must be present");
        checkArgument(internalListener.getBrokerServiceUrlTls() != null, "TLS must be configured on internal listener");
        URL webServiceUrlTls = new URL(String.format("https://%s:%d", ServiceConfigurationUtils.getWebServiceAddress(config), config.getWebServicePortTls().get()));
        PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(webServiceUrlTls.toString()).authentication(config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters());
        // set trust store if needed.
        if (config.isBrokerClientTlsEnabled()) {
            if (config.isBrokerClientTlsEnabledWithKeyStore()) {
                builder.useKeyStoreTls(true).tlsTrustStoreType(config.getBrokerClientTlsTrustStoreType()).tlsTrustStorePath(config.getBrokerClientTlsTrustStore()).tlsTrustStorePassword(config.getBrokerClientTlsTrustStorePassword());
            } else {
                builder.tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath());
            }
            builder.allowTlsInsecureConnection(config.isTlsAllowInsecureConnection());
        }
        admin = builder.build();
        ClusterData clusterData = ClusterData.builder().serviceUrlTls(webServiceUrlTls.toString()).brokerServiceUrlTls(internalListener.getBrokerServiceUrlTls().toString()).build();
        createSampleNameSpace(clusterData, cluster);
    }
    // create default namespace
    createNameSpace(cluster, TopicName.PUBLIC_TENANT, TopicName.PUBLIC_TENANT + "/" + TopicName.DEFAULT_NAMESPACE);
    // create pulsar system namespace
    createNameSpace(cluster, SYSTEM_NAMESPACE.getTenant(), SYSTEM_NAMESPACE.toString());
    if (config.isTransactionCoordinatorEnabled() && !admin.namespaces().getTopics(SYSTEM_NAMESPACE.toString()).contains(TRANSACTION_COORDINATOR_ASSIGN.getPartition(0).toString())) {
        admin.topics().createPartitionedTopic(TRANSACTION_COORDINATOR_ASSIGN.toString(), 1);
    }
    log.debug("--- setup completed ---");
}

Example 2 with AdvertisedListener

use of org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener in project pulsar by apache.

the class ServiceConfigurationUtils method getAppliedAdvertisedAddress.

/**
 * Get the address of Broker, first try to get it from AdvertisedAddress.
 * If it is not set, try to get the address set by advertisedListener.
 * If it is still not set, get it through InetAddress.getLocalHost().
 * @param configuration
 * @param ignoreAdvertisedListener Sometimes we can’t use the default key of AdvertisedListener,
 *                                 setting it to true can ignore AdvertisedListener.
 * @return
 */
@Deprecated
public static String getAppliedAdvertisedAddress(ServiceConfiguration configuration, boolean ignoreAdvertisedListener) {
    Map<String, AdvertisedListener> result = MultipleListenerValidator.validateAndAnalysisAdvertisedListener(configuration);
    String advertisedAddress = configuration.getAdvertisedAddress();
    if (advertisedAddress != null) {
        return advertisedAddress;
    }
    AdvertisedListener advertisedListener = result.get(configuration.getInternalListenerName());
    if (advertisedListener != null && !ignoreAdvertisedListener) {
        String address = advertisedListener.getBrokerServiceUrl().getHost();
        if (address != null) {
            return address;
        }
    }
    return getDefaultOrConfiguredAddress(advertisedAddress);
}

Example 3 with AdvertisedListener

use of org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener in project pulsar by apache.

the class ServiceConfigurationUtils method getInternalListener.

/**
 * Gets the internal advertised listener for broker-to-broker communication.
 * @return a non-null advertised listener
 */
public static AdvertisedListener getInternalListener(ServiceConfiguration config) {
    Map<String, AdvertisedListener> result = MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config);
    AdvertisedListener internal = result.get(config.getInternalListenerName());
    if (internal == null) {
        // synthesize an advertised listener based on legacy configuration properties
        String host = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
        internal = AdvertisedListener.builder().brokerServiceUrl(createUriOrNull("pulsar", host, config.getBrokerServicePort())).brokerServiceUrlTls(createUriOrNull("pulsar+ssl", host, config.getBrokerServicePortTls())).build();
    }
    return internal;
}

Example 4 with AdvertisedListener

use of org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener in project pulsar by apache.

the class NamespaceService method searchForCandidateBroker.

private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture<Optional<LookupResult>> lookupFuture, LookupOptions options) {
    if (null == pulsar.getLeaderElectionService()) {
        LOG.warn("The leader election has not yet been completed! NamespaceBundle[{}]", bundle);
        lookupFuture.completeExceptionally(new IllegalStateException("The leader election has not yet been completed!"));
        return;
    }
    String candidateBroker = null;
    LeaderElectionService les = pulsar.getLeaderElectionService();
    if (les == null) {
        // The leader election service was not initialized yet. This can happen because the broker service is
        // initialized first and it might start receiving lookup requests before the leader election service is
        // fully initialized.
        LOG.warn("Leader election service isn't initialized yet. " + "Returning empty result to lookup. NamespaceBundle[{}]", bundle);
        lookupFuture.complete(Optional.empty());
        return;
    }
    boolean authoritativeRedirect = les.isLeader();
    try {
        // check if this is Heartbeat or SLAMonitor namespace
        candidateBroker = checkHeartbeatNamespace(bundle);
        if (candidateBroker == null) {
            candidateBroker = checkHeartbeatNamespaceV2(bundle);
        }
        if (candidateBroker == null) {
            String broker = getSLAMonitorBrokerName(bundle);
            // checking if the broker is up and running
            if (broker != null && isBrokerActive(broker)) {
                candidateBroker = broker;
            }
        }
        if (candidateBroker == null) {
            Optional<LeaderBroker> currentLeader = pulsar.getLeaderElectionService().getCurrentLeader();
            if (options.isAuthoritative()) {
                // leader broker already assigned the current broker as owner
                candidateBroker = pulsar.getSafeWebServiceAddress();
            } else {
                LoadManager loadManager = this.loadManager.get();
                boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader();
                if (!makeLoadManagerDecisionOnThisBroker) {
                    // If leader is not active, fallback to pick the least loaded from current broker loadmanager
                    boolean leaderBrokerActive = currentLeader.isPresent() && isBrokerActive(currentLeader.get().getServiceUrl());
                    if (!leaderBrokerActive) {
                        makeLoadManagerDecisionOnThisBroker = true;
                        if (!currentLeader.isPresent()) {
                            LOG.warn("The information about the current leader broker wasn't available. " + "Handling load manager decisions in a decentralized way. " + "NamespaceBundle[{}]", bundle);
                        } else {
                            LOG.warn("The current leader broker {} isn't active. " + "Handling load manager decisions in a decentralized way. " + "NamespaceBundle[{}]", currentLeader.get(), bundle);
                        }
                    }
                }
                if (makeLoadManagerDecisionOnThisBroker) {
                    Optional<String> availableBroker = getLeastLoadedFromLoadManager(bundle);
                    if (!availableBroker.isPresent()) {
                        LOG.warn("Load manager didn't return any available broker. " + "Returning empty result to lookup. NamespaceBundle[{}]", bundle);
                        lookupFuture.complete(Optional.empty());
                        return;
                    }
                    candidateBroker = availableBroker.get();
                    authoritativeRedirect = true;
                } else {
                    // forward to leader broker to make assignment
                    candidateBroker = currentLeader.get().getServiceUrl();
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
        return;
    }
    try {
        checkNotNull(candidateBroker);
        if (candidateBroker.equals(pulsar.getSafeWebServiceAddress())) {
            // Load manager decided that the local broker should try to become the owner
            ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> {
                if (ownerInfo.isDisabled()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Namespace bundle {} is currently being unloaded", bundle);
                    }
                    lookupFuture.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is currently being unloaded", bundle)));
                } else {
                    if (options.isLoadTopicsInBundle()) {
                        // Schedule the task to pre-load topics
                        pulsar.loadNamespaceTopics(bundle);
                    }
                    // find the target
                    if (options.hasAdvertisedListenerName()) {
                        AdvertisedListener listener = ownerInfo.getAdvertisedListeners().get(options.getAdvertisedListenerName());
                        if (listener == null) {
                            lookupFuture.completeExceptionally(new PulsarServerException("the broker do not have " + options.getAdvertisedListenerName() + " listener"));
                            return;
                        } else {
                            URI url = listener.getBrokerServiceUrl();
                            URI urlTls = listener.getBrokerServiceUrlTls();
                            lookupFuture.complete(Optional.of(new LookupResult(ownerInfo, url == null ? null : url.toString(), urlTls == null ? null : urlTls.toString())));
                            return;
                        }
                    } else {
                        lookupFuture.complete(Optional.of(new LookupResult(ownerInfo)));
                        return;
                    }
                }
            }).exceptionally(exception -> {
                LOG.warn("Failed to acquire ownership for namespace bundle {}: {}", bundle, exception);
                lookupFuture.completeExceptionally(new PulsarServerException("Failed to acquire ownership for namespace bundle " + bundle, exception));
                return null;
            });
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Redirecting to broker {} to acquire ownership of bundle {}", candidateBroker, bundle);
            }
            // Now setting the redirect url
            createLookupResult(candidateBroker, authoritativeRedirect, options.getAdvertisedListenerName()).thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))).exceptionally(ex -> {
                lookupFuture.completeExceptionally(ex);
                return null;
            });
        }
    } catch (Exception e) {
        LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
    }
}

Example 5 with AdvertisedListener

use of org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener in project pulsar by yahoo.

the class MultipleListenerValidator method validateAndAnalysisAdvertisedListener.

/**
 * Validate the configuration of `advertisedListeners`, `internalListenerName`.
 * 1. `advertisedListeners` consists of a comma-separated list of endpoints.
 * 2. Each endpoint consists of a listener name and an associated address (`listener:scheme://host:port`).
 * 3. A listener name may be repeated to define both a non-TLS and a TLS endpoint.
 * 4. Duplicate definitions are disallowed.
 * 5. If `internalListenerName` is absent, set it to the first listener defined in `advertisedListeners`.
 * @param config the pulsar broker configure.
 * @return
 */
public static Map<String, AdvertisedListener> validateAndAnalysisAdvertisedListener(ServiceConfiguration config) {
    if (StringUtils.isBlank(config.getAdvertisedListeners())) {
        return Collections.emptyMap();
    }
    Optional<String> firstListenerName = Optional.empty();
    Map<String, List<String>> listeners = new LinkedHashMap<>();
    for (final String str : StringUtils.split(config.getAdvertisedListeners(), ",")) {
        int index = str.indexOf(":");
        if (index <= 0) {
            throw new IllegalArgumentException("the configure entry `advertisedListeners` is invalid. because " + str + " do not contain listener name");
        }
        String listenerName = StringUtils.trim(str.substring(0, index));
        if (!firstListenerName.isPresent()) {
            firstListenerName = Optional.of(listenerName);
        }
        String value = StringUtils.trim(str.substring(index + 1));
        listeners.computeIfAbsent(listenerName, k -> new ArrayList<>(2));
        listeners.get(listenerName).add(value);
    }
    if (StringUtils.isBlank(config.getInternalListenerName())) {
        config.setInternalListenerName(firstListenerName.get());
    }
    if (!listeners.containsKey(config.getInternalListenerName())) {
        throw new IllegalArgumentException("the `advertisedListeners` configure do not contain " + "`internalListenerName` entry");
    }
    final Map<String, AdvertisedListener> result = new LinkedHashMap<>();
    final Map<String, Set<String>> reverseMappings = new LinkedHashMap<>();
    for (final Map.Entry<String, List<String>> entry : listeners.entrySet()) {
        if (entry.getValue().size() > 2) {
            throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`");
        }
        URI pulsarAddress = null, pulsarSslAddress = null, pulsarHttpAddress = null, pulsarHttpsAddress = null;
        for (final String strUri : entry.getValue()) {
            try {
                URI uri = URI.create(strUri);
                if (StringUtils.equalsIgnoreCase(uri.getScheme(), "pulsar")) {
                    if (pulsarAddress == null) {
                        pulsarAddress = uri;
                    } else {
                        throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`");
                    }
                } else if (StringUtils.equalsIgnoreCase(uri.getScheme(), "pulsar+ssl")) {
                    if (pulsarSslAddress == null) {
                        pulsarSslAddress = uri;
                    } else {
                        throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`");
                    }
                } else if (StringUtils.equalsIgnoreCase(uri.getScheme(), "http")) {
                    if (pulsarHttpAddress == null) {
                        pulsarHttpAddress = uri;
                    } else {
                        throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`");
                    }
                } else if (StringUtils.equalsIgnoreCase(uri.getScheme(), "https")) {
                    if (pulsarHttpsAddress == null) {
                        pulsarHttpsAddress = uri;
                    } else {
                        throw new IllegalArgumentException("there are redundant configure for listener `" + entry.getKey() + "`");
                    }
                }
                String hostPort = String.format("%s:%d", uri.getHost(), uri.getPort());
                Set<String> sets = reverseMappings.computeIfAbsent(hostPort, k -> new TreeSet<>());
                sets.add(entry.getKey());
                if (sets.size() > 1) {
                    throw new IllegalArgumentException("must not specify `" + hostPort + "` to different listener.");
                }
            } catch (Throwable cause) {
                throw new IllegalArgumentException("the value " + strUri + " in the `advertisedListeners` " + "configure is invalid", cause);
            }
        }
        result.put(entry.getKey(), AdvertisedListener.builder().brokerServiceUrl(pulsarAddress).brokerServiceUrlTls(pulsarSslAddress).brokerHttpUrl(pulsarHttpAddress).brokerHttpsUrl(pulsarHttpsAddress).build());
    }
    return result;
}