Examples with ServiceConfiguration org.apache.pulsar.broker.ServiceConfiguration used on opensource projects

Example 46 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class ServiceConfigurationTest method testOptionalSettingPresent.

@Test
public void testOptionalSettingPresent() throws Exception {
    String confFile = "loadBalancerOverrideBrokerNicSpeedGbps=5\n";
    InputStream stream = new ByteArrayInputStream(confFile.getBytes());
    final ServiceConfiguration config = PulsarConfigurationLoader.create(stream, ServiceConfiguration.class);
    assertEquals(config.getLoadBalancerOverrideBrokerNicSpeedGbps(), Optional.of(5.0));
}

Example 47 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class BrokerService method getManagedLedgerConfig.

public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(TopicName topicName) {
    CompletableFuture<ManagedLedgerConfig> future = new CompletableFuture<>();
    // Execute in background thread, since getting the policies might block if the z-node wasn't already cached
    pulsar.getOrderedExecutor().submitOrdered(topicName, safeRun(() -> {
        NamespaceName namespace = topicName.getNamespaceObject();
        ServiceConfiguration serviceConfig = pulsar.getConfiguration();
        // Get persistence policy for this topic
        Optional<Policies> policies = Optional.empty();
        try {
            policies = pulsar.getConfigurationCache().policiesCache().get(AdminResource.path(POLICIES, namespace.toString()));
        } catch (Throwable t) {
            // Ignoring since if we don't have policies, we fallback on the default
            log.warn("Got exception when reading persistence policy for {}: {}", topicName, t.getMessage(), t);
            future.completeExceptionally(t);
            return;
        }
        PersistencePolicies persistencePolicies = policies.map(p -> p.persistence).orElseGet(() -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), serviceConfig.getManagedLedgerDefaultWriteQuorum(), serviceConfig.getManagedLedgerDefaultAckQuorum(), serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit()));
        RetentionPolicies retentionPolicies = policies.map(p -> p.retention_policies).orElseGet(() -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), serviceConfig.getDefaultRetentionSizeInMB()));
        ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
        managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble());
        managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
        managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
        managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
        managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType());
        managedLedgerConfig.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
        managedLedgerConfig.setMaxUnackedRangesToPersistInZk(serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInZooKeeper());
        managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
        managedLedgerConfig.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        managedLedgerConfig.setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        managedLedgerConfig.setMaxSizePerLedgerMb(2048);
        managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize());
        managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum());
        managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum());
        managedLedgerConfig.setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger());
        managedLedgerConfig.setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds());
        managedLedgerConfig.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES);
        managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB());
        future.complete(managedLedgerConfig);
    }, (exception) -> future.completeExceptionally(exception)));
    return future;
}

Example 48 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class BrokerService method updateDynamicServiceConfiguration.

/**
 * Updates pulsar.ServiceConfiguration's dynamic field with value persent into zk-dynamic path. It also validates
 * dynamic-value before updating it and throws {@code IllegalArgumentException} if validation fails
 */
private void updateDynamicServiceConfiguration() {
    Optional<Map<String, String>> configCache = Optional.empty();
    try {
        // create dynamic-config znode if not present
        if (pulsar.getZkClient().exists(BROKER_SERVICE_CONFIGURATION_PATH, false) == null) {
            try {
                byte[] data = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(Maps.newHashMap());
                ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), BROKER_SERVICE_CONFIGURATION_PATH, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            } catch (KeeperException.NodeExistsException e) {
            // Ok
            }
        }
        configCache = dynamicConfigurationCache.get(BROKER_SERVICE_CONFIGURATION_PATH);
    } catch (Exception e) {
        log.warn("Failed to read zookeeper path [{}]:", BROKER_SERVICE_CONFIGURATION_PATH, e);
    }
    if (configCache.isPresent()) {
        configCache.get().forEach((key, value) -> {
            // validate field
            if (dynamicConfigurationMap.containsKey(key) && dynamicConfigurationMap.get(key).validator != null) {
                if (!dynamicConfigurationMap.get(key).validator.test(value)) {
                    log.error("Failed to validate dynamic config {} with value {}", key, value);
                    throw new IllegalArgumentException(String.format("Failed to validate dynamic-config %s/%s", key, value));
                }
            }
            // update field value
            try {
                Field field = ServiceConfiguration.class.getDeclaredField(key);
                if (field != null && field.isAnnotationPresent(FieldContext.class)) {
                    field.setAccessible(true);
                    field.set(pulsar().getConfiguration(), FieldParser.value(value, field));
                    log.info("Successfully updated {}/{}", key, value);
                }
            } catch (Exception e) {
                log.warn("Failed to update service configuration {}/{}, {}", key, value, e.getMessage());
            }
        });
    }
    // register a listener: it updates field value and triggers appropriate registered field-listener only if
    // field's value has been changed so, registered doesn't have to update field value in ServiceConfiguration
    dynamicConfigurationCache.registerListener(new ZooKeeperCacheListener<Map<String, String>>() {

        @SuppressWarnings("unchecked")
        @Override
        public void onUpdate(String path, Map<String, String> data, Stat stat) {
            if (BROKER_SERVICE_CONFIGURATION_PATH.equalsIgnoreCase(path) && data != null) {
                data.forEach((configKey, value) -> {
                    Field configField = dynamicConfigurationMap.get(configKey).field;
                    Object newValue = FieldParser.value(data.get(configKey), configField);
                    if (configField != null) {
                        Consumer listener = configRegisteredListeners.get(configKey);
                        try {
                            Object existingValue = configField.get(pulsar.getConfiguration());
                            configField.set(pulsar.getConfiguration(), newValue);
                            log.info("Successfully updated configuration {}/{}", configKey, data.get(configKey));
                            if (listener != null && !existingValue.equals(newValue)) {
                                listener.accept(newValue);
                            }
                        } catch (Exception e) {
                            log.error("Failed to update config {}/{}", configKey, newValue);
                        }
                    } else {
                        log.error("Found non-dynamic field in dynamicConfigMap {}/{}", configKey, newValue);
                    }
                });
            }
        }
    });
}

Example 49 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method start.

@Override
public void start() throws PulsarServerException {
    try {
        // Register the brokers in zk list
        ServiceConfiguration conf = pulsar.getConfiguration();
        if (pulsar.getZkClient().exists(LOADBALANCE_BROKERS_ROOT, false) == null) {
            try {
                ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), LOADBALANCE_BROKERS_ROOT, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            } catch (KeeperException.NodeExistsException e) {
            // ignore the exception, node might be present already
            }
        }
        String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" + conf.getWebServicePort();
        brokerZnodePath = LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress;
        LoadReport loadReport = null;
        try {
            loadReport = generateLoadReport();
            this.lastResourceUsageTimestamp = loadReport.getTimestamp();
        } catch (Exception e) {
            log.warn("Unable to get load report to write it on zookeeper [{}]", e);
        }
        String loadReportJson = "";
        if (loadReport != null) {
            loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(loadReport);
        }
        try {
            ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), brokerZnodePath, loadReportJson.getBytes(Charsets.UTF_8), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
        } catch (KeeperException.NodeExistsException e) {
            long ownerZkSessionId = getBrokerZnodeOwner();
            if (ownerZkSessionId != 0 && ownerZkSessionId != pulsar.getZkClient().getSessionId()) {
                log.error("Broker znode - [{}] is own by different zookeeper-ssession {} ", brokerZnodePath, ownerZkSessionId);
                throw new PulsarServerException("Broker-znode owned by different zk-session " + ownerZkSessionId);
            }
            // Node may already be created by another load manager: in this case update the data.
            if (loadReport != null) {
                pulsar.getZkClient().setData(brokerZnodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
            }
        } catch (Exception e) {
            // Catching excption here to print the right error message
            log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
            throw e;
        }
        // first time, populate the broker ranking
        updateRanking();
        log.info("Created broker ephemeral node on {}", brokerZnodePath);
        // load default resource quota
        this.realtimeAvgResourceQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
        this.lastResourceQuotaUpdateTimestamp = System.currentTimeMillis();
        this.realtimeCpuLoadFactor = getDynamicConfigurationDouble(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_CPU_ZPATH, SETTING_NAME_LOAD_FACTOR_CPU, this.realtimeCpuLoadFactor);
        this.realtimeMemoryLoadFactor = getDynamicConfigurationDouble(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_MEM_ZPATH, SETTING_NAME_LOAD_FACTOR_MEM, this.realtimeMemoryLoadFactor);
    } catch (Exception e) {
        log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
        throw new PulsarServerException(e);
    }
}

Example 50 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class BundleSplitterTask method findBundlesToSplit.

/**
 * Determines which bundles should be split based on various thresholds.
 *
 * @param loadData
 *            Load data to base decisions on (does not have benefit of preallocated data since this may not be the
 *            leader broker).
 * @param localData
 *            Local data for the broker we are splitting on.
 * @param pulsar
 *            Service to use.
 * @return All bundles who have exceeded configured thresholds in number of topics, number of sessions, total
 *         message rates, or total throughput.
 */
@Override
public Set<String> findBundlesToSplit(final LoadData loadData, final PulsarService pulsar) {
    bundleCache.clear();
    final ServiceConfiguration conf = pulsar.getConfiguration();
    int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
    loadData.getBrokerData().forEach((broker, brokerData) -> {
        LocalBrokerData localData = brokerData.getLocalData();
        for (final Map.Entry<String, NamespaceBundleStats> entry : localData.getLastStats().entrySet()) {
            final String bundle = entry.getKey();
            final NamespaceBundleStats stats = entry.getValue();
            double totalMessageRate = 0;
            double totalMessageThroughput = 0;
            // Attempt to consider long-term message data, otherwise effectively ignore.
            if (loadData.getBundleData().containsKey(bundle)) {
                final TimeAverageMessageData longTermData = loadData.getBundleData().get(bundle).getLongTermData();
                totalMessageRate = longTermData.totalMsgRate();
                totalMessageThroughput = longTermData.totalMsgThroughput();
            }
            if (stats.topics > maxBundleTopics || stats.consumerCount + stats.producerCount > maxBundleSessions || totalMessageRate > maxBundleMsgRate || totalMessageThroughput > maxBundleBandwidth) {
                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
                try {
                    final int bundleCount = pulsar.getNamespaceService().getBundleCount(NamespaceName.get(namespace));
                    if (bundleCount < maxBundleCount) {
                        bundleCache.add(bundle);
                    } else {
                        log.warn("Could not split namespace bundle {} because namespace {} has too many bundles: {}", bundle, namespace, bundleCount);
                    }
                } catch (Exception e) {
                    log.warn("Error while getting bundle count for namespace {}", namespace, e);
                }
            }
        }
    });
    return bundleCache;
}