Examples with LoadReport org.apache.pulsar.policies.data.loadbalancer.LoadReport used on opensource projects

Example 21 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method start.

@Override
public void start() throws PulsarServerException {
    try {
        // Register the brokers in zk list
        ServiceConfiguration conf = pulsar.getConfiguration();
        if (pulsar.getZkClient().exists(LOADBALANCE_BROKERS_ROOT, false) == null) {
            try {
                ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), LOADBALANCE_BROKERS_ROOT, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            } catch (KeeperException.NodeExistsException e) {
            // ignore the exception, node might be present already
            }
        }
        String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" + conf.getWebServicePort();
        brokerZnodePath = LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress;
        LoadReport loadReport = null;
        try {
            loadReport = generateLoadReport();
            this.lastResourceUsageTimestamp = loadReport.getTimestamp();
        } catch (Exception e) {
            log.warn("Unable to get load report to write it on zookeeper [{}]", e);
        }
        String loadReportJson = "";
        if (loadReport != null) {
            loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(loadReport);
        }
        try {
            ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), brokerZnodePath, loadReportJson.getBytes(Charsets.UTF_8), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
        } catch (KeeperException.NodeExistsException e) {
            long ownerZkSessionId = getBrokerZnodeOwner();
            if (ownerZkSessionId != 0 && ownerZkSessionId != pulsar.getZkClient().getSessionId()) {
                log.error("Broker znode - [{}] is own by different zookeeper-ssession {} ", brokerZnodePath, ownerZkSessionId);
                throw new PulsarServerException("Broker-znode owned by different zk-session " + ownerZkSessionId);
            }
            // Node may already be created by another load manager: in this case update the data.
            if (loadReport != null) {
                pulsar.getZkClient().setData(brokerZnodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
            }
        } catch (Exception e) {
            // Catching excption here to print the right error message
            log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
            throw e;
        }
        // first time, populate the broker ranking
        updateRanking();
        log.info("Created broker ephemeral node on {}", brokerZnodePath);
        // load default resource quota
        this.realtimeAvgResourceQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
        this.lastResourceQuotaUpdateTimestamp = System.currentTimeMillis();
        this.realtimeCpuLoadFactor = getDynamicConfigurationDouble(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_CPU_ZPATH, SETTING_NAME_LOAD_FACTOR_CPU, this.realtimeCpuLoadFactor);
        this.realtimeMemoryLoadFactor = getDynamicConfigurationDouble(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_MEM_ZPATH, SETTING_NAME_LOAD_FACTOR_MEM, this.realtimeMemoryLoadFactor);
    } catch (Exception e) {
        log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
        throw new PulsarServerException(e);
    }
}

Example 22 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LoadBalancerTest method writeLoadReportsForDynamicQuota.

private void writeLoadReportsForDynamicQuota(long timestamp) throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        lr.setTimestamp(timestamp);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(5000 * (10 + i * 5), 1024000));
        sru.setBandwidthOut(new ResourceUsage(15000 * (10 + i * 5), 1024000));
        sru.setMemory(new ResourceUsage(25 * (10 + i * 5), 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(200, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            stats.msgRateIn = 5 * (i + j);
            stats.msgRateOut = 15 * (i + j);
            stats.msgThroughputIn = 5000 * (i + j);
            stats.msgThroughputOut = 15000 * (i + j);
            stats.topics = 25 * (i + j);
            stats.consumerCount = 50 * (i + j);
            stats.producerCount = 50 * (i + j);
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
}

Example 23 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LoadBalancerTest method testLoadReportsWrittenOnZK.

/*
     * tests that load manager creates its node and writes the initial load report a /loadbalance/brokers tests that
     * those load reports can be deserialized and are in valid format tests if the rankings are populated from the load
     * reports are not, both broker will have zero rank
     */
@SuppressWarnings("unchecked")
@Test
public void testLoadReportsWrittenOnZK() throws Exception {
    ZooKeeper zkc = bkEnsemble.getZkClient();
    try {
        for (int i = 0; i < BROKER_COUNT; i++) {
            String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
            byte[] loadReportData = zkc.getData(znodePath, false, null);
            assert (loadReportData.length > 0);
            log.info("LoadReport {}, {}", lookupAddresses[i], new String(loadReportData));
            LoadReport loadReport = ObjectMapperFactory.getThreadLocal().readValue(loadReportData, LoadReport.class);
            assert (loadReport.getName().equals(lookupAddresses[i]));
            // Check Initial Ranking is populated in both the brokers
            Field ranking = ((SimpleLoadManagerImpl) pulsarServices[i].getLoadManager().get()).getClass().getDeclaredField("sortedRankings");
            ranking.setAccessible(true);
            AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = (AtomicReference<Map<Long, Set<ResourceUnit>>>) ranking.get(pulsarServices[i].getLoadManager().get());
            printSortedRanking(sortedRanking);
            // all brokers have same rank to it would be 0 --> set-of-all-the-brokers
            int brokerCount = 0;
            for (Map.Entry<Long, Set<ResourceUnit>> entry : sortedRanking.get().entrySet()) {
                brokerCount += entry.getValue().size();
            }
            assertEquals(brokerCount, BROKER_COUNT);
            TopicName topicName = TopicName.get("persistent://pulsar/use/primary-ns/test-topic");
            ResourceUnit found = pulsarServices[i].getLoadManager().get().getLeastLoaded(pulsarServices[i].getNamespaceService().getBundle(topicName)).get();
            assertTrue(found != null);
        }
    } catch (InterruptedException | KeeperException e) {
        fail("Unable to read the data from Zookeeper - [{}]", e);
    }
}

Example 24 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LoadBalancerTest method testUpdateLoadReportAndCheckUpdatedRanking.

/*
     * tests rankings get updated when we write write the new load reports to the zookeeper on loadbalance root node
     * tests writing pre-configured load report on the zookeeper translates the pre-calculated rankings
     */
@Test
public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(256, 1024000));
        sru.setBandwidthOut(new ResourceUsage(250, 1024000));
        sru.setMemory(new ResourceUsage(1024, 8192));
        sru.setCpu(new ResourceUsage(5, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait the load ranking be triggered
    Thread.sleep(5000);
    // do lookup for bunch of bundles
    int totalNamespaces = 200;
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        TopicName topicName = TopicName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().get().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(topicName)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    // assert that distribution variation is not more than 10%
    int averageNamespaces = totalNamespaces / BROKER_COUNT;
    int tenPercentOfAverageNamespaces = averageNamespaces / 10;
    int lowerBound = averageNamespaces - tenPercentOfAverageNamespaces;
    int upperBound = averageNamespaces + tenPercentOfAverageNamespaces;
    // assert each broker received ownership of fair amount of namespaces 90%+
    for (Map.Entry<String, Integer> broker : namespaceOwner.entrySet()) {
        log.info("Count of bundles assigned: {}, {}", broker.getKey(), broker.getValue());
        assertTrue(broker.getValue() >= lowerBound && broker.getValue() <= upperBound);
    }
}

Example 25 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LoadBalancerTest method testTopicAssignmentWithExistingBundles.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - already has some bundles
     * assigned Check the distribution of new topics is roughly consistent (with <10% variation) with the ranking
     */
@Test
public void testTopicAssignmentWithExistingBundles() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(87);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(0, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < (i + 1) * 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // print ranking
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
    }
    // check owner of new destiations and verify that the distribution is roughly
    // consistent (variation < 10%) with the broker capacity:
    int totalNamespaces = 250;
    int[] expectedAssignments = new int[] { 17, 34, 51, 68, 85 };
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        TopicName topicName = TopicName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().get().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(topicName)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    double expectedMaxVariation = 10.0;
    for (int i = 0; i < BROKER_COUNT; i++) {
        long actualValue = 0;
        String resourceId = "http://" + lookupAddresses[i];
        if (namespaceOwner.containsKey(resourceId)) {
            actualValue = namespaceOwner.get(resourceId);
        }
        long expectedValue = expectedAssignments[i];
        double variation = Math.abs(actualValue - expectedValue) * 100.0 / expectedValue;
        log.info("Topic assignment - {}, actual: {}, expected baseline: {}, variation: {}/%", lookupAddresses[i], actualValue, expectedValue, String.format("%.2f", variation));
        assertTrue(variation < expectedMaxVariation);
    }
}