Examples with LoadReport org.apache.pulsar.policies.data.loadbalancer.LoadReport used on opensource projects

Example 6 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LookupDataTest method testLoadReportSerialization.

@Test
public void testLoadReportSerialization() throws Exception {
    final String simpleLmBrokerUrl = "simple";
    final String simpleLmReportName = "simpleLoadManager";
    final String modularLmBrokerUrl = "modular";
    final SystemResourceUsage simpleLmSystemResourceUsage = new SystemResourceUsage();
    final ResourceUsage resource = new ResourceUsage();
    final double usage = 55.0;
    resource.usage = usage;
    simpleLmSystemResourceUsage.bandwidthIn = resource;
    LoadReport simpleReport = getSimpleLoadManagerLoadReport(simpleLmBrokerUrl, simpleLmReportName, simpleLmSystemResourceUsage);
    LocalBrokerData modularReport = getModularLoadManagerLoadReport(modularLmBrokerUrl, resource);
    LoadManagerReport simpleLoadReport = ObjectMapperFactory.getThreadLocal().readValue(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(simpleReport), LoadManagerReport.class);
    LoadManagerReport modularLoadReport = ObjectMapperFactory.getThreadLocal().readValue(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(modularReport), LoadManagerReport.class);
    assertEquals(simpleLoadReport.getWebServiceUrl(), simpleLmBrokerUrl);
    assertTrue(simpleLoadReport instanceof LoadReport);
    assertEquals(((LoadReport) simpleLoadReport).getName(), simpleLmReportName);
    assertEquals(((LoadReport) simpleLoadReport).getSystemResourceUsage().bandwidthIn.usage, usage);
    assertEquals(modularLoadReport.getWebServiceUrl(), modularLmBrokerUrl);
    assertTrue(modularLoadReport instanceof LocalBrokerData);
    assertEquals(((LocalBrokerData) modularLoadReport).getBandwidthIn().usage, usage);
}

Example 7 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LookupDataTest method getSimpleLoadManagerLoadReport.

private LoadReport getSimpleLoadManagerLoadReport(String brokerUrl, String reportName, SystemResourceUsage systemResourceUsage) {
    LoadReport report = new LoadReport(brokerUrl, null, null, null);
    report.setName(reportName);
    report.setSystemResourceUsage(systemResourceUsage);
    return report;
}

Example 8 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class BrokerMonitor method printGlobalData.

// Prints out the global load data.
private void printGlobalData() {
    synchronized (loadData) {
        // 1 header row, 1 total row, and loadData.size() rows for brokers.
        Object[][] rows = new Object[loadData.size() + 2][];
        rows[0] = GLOBAL_HEADER;
        int totalBundles = 0;
        double totalThroughput = 0;
        double totalMessageRate = 0;
        double totalLongTermMessageRate = 0;
        double maxMaxUsage = 0;
        int i = 1;
        for (final Map.Entry<String, Object> entry : loadData.entrySet()) {
            final String broker = entry.getKey();
            final Object data = entry.getValue();
            rows[i] = new Object[GLOBAL_HEADER.length];
            rows[i][0] = broker;
            int numBundles;
            double messageRate;
            double longTermMessageRate;
            double messageThroughput;
            double maxUsage;
            if (data instanceof LoadReport) {
                final LoadReport loadReport = (LoadReport) data;
                numBundles = (int) loadReport.getNumBundles();
                messageRate = loadReport.getMsgRateIn() + loadReport.getMsgRateOut();
                longTermMessageRate = loadReport.getAllocatedMsgRateIn() + loadReport.getAllocatedMsgRateOut();
                messageThroughput = (loadReport.getAllocatedBandwidthIn() + loadReport.getAllocatedBandwidthOut()) / 1024;
                final SystemResourceUsage systemResourceUsage = loadReport.getSystemResourceUsage();
                maxUsage = Math.max(Math.max(Math.max(systemResourceUsage.getCpu().percentUsage(), systemResourceUsage.getMemory().percentUsage()), Math.max(systemResourceUsage.getDirectMemory().percentUsage(), systemResourceUsage.getBandwidthIn().percentUsage())), systemResourceUsage.getBandwidthOut().percentUsage());
            } else if (data instanceof LocalBrokerData) {
                final LocalBrokerData localData = (LocalBrokerData) data;
                numBundles = localData.getNumBundles();
                messageRate = localData.getMsgRateIn() + localData.getMsgRateOut();
                final String timeAveragePath = ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH + "/" + broker;
                try {
                    final TimeAverageBrokerData timeAverageData = gson.fromJson(new String(zkClient.getData(timeAveragePath, false, null)), TimeAverageBrokerData.class);
                    longTermMessageRate = timeAverageData.getLongTermMsgRateIn() + timeAverageData.getLongTermMsgRateOut();
                } catch (Exception x) {
                    throw new RuntimeException(x);
                }
                messageThroughput = (localData.getMsgThroughputIn() + localData.getMsgThroughputOut()) / 1024;
                maxUsage = localData.getMaxResourceUsage();
            } else {
                throw new AssertionError("Unreachable code");
            }
            rows[i][1] = numBundles;
            rows[i][2] = messageRate;
            rows[i][3] = messageThroughput;
            rows[i][4] = longTermMessageRate;
            rows[i][5] = maxUsage;
            totalBundles += numBundles;
            totalMessageRate += messageRate;
            totalLongTermMessageRate += longTermMessageRate;
            totalThroughput += messageThroughput;
            maxMaxUsage = Math.max(maxUsage, maxMaxUsage);
            ++i;
        }
        final int finalRow = loadData.size() + 1;
        rows[finalRow] = new Object[GLOBAL_HEADER.length];
        rows[finalRow][0] = "TOTAL";
        rows[finalRow][1] = totalBundles;
        rows[finalRow][2] = totalMessageRate;
        rows[finalRow][3] = totalLongTermMessageRate;
        rows[finalRow][4] = totalThroughput;
        rows[finalRow][5] = maxMaxUsage;
        final String table = globalTableMaker.make(rows);
        log.info("Overall Broker Data:\n{}", table);
    }
}

Example 9 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LoadBalancerTest method testBrokerRanking.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - non-bundles assigned so all
     * idle resources are avaiable for new bundle Check the broker rankings are the load percentage of each broker.
     */
@Test
public void testBrokerRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(1024, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // check the ranking result
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
        // brokers' ranking would be:
        // 50 --> broker 0 ( 1024 / 2048 )
        // 25 --> broker 1 ( 1024 / 4096 )
        // 16 --> broker 2 ( 1024 / 6144 )
        // 15 --> broker 3 ( 60 / 400 )
        // 15 --> broker 4 ( 60 / 400 )
        assertEquals(sortedRanking.get().size(), 4);
        assertEquals(sortedRanking.get().get(50L).size(), 1);
        assertEquals(sortedRanking.get().get(25L).size(), 1);
        assertEquals(sortedRanking.get().get(16L).size(), 1);
        assertEquals(sortedRanking.get().get(15L).size(), 2);
    }
}

Example 10 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class LoadBalancerTest method testNamespaceBundleAutoSplit.

/**
 * Test the namespace bundle auto-split
 */
@Test
public void testNamespaceBundleAutoSplit() throws Exception {
    int maxBundles = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxTopics = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    int maxSessions = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxMsgRate = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBandwidth = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1048576;
    pulsarServices[0].getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
    // create namespaces
    for (int i = 1; i <= 10; i++) {
        int numBundles = (i == 10) ? maxBundles : 2;
        createNamespace(pulsarServices[0], String.format("pulsar/use/primary-ns-%02d", i), numBundles);
    }
    // fake Namespaces Admin
    NamespacesImpl namespaceAdmin = mock(NamespacesImpl.class);
    setObjectField(PulsarAdmin.class, pulsarServices[0].getAdminClient(), "namespaces", namespaceAdmin);
    // create load report
    // namespace 01~09 need to be split
    // namespace 08~10 don't need or cannot be split
    LoadReport lr = new LoadReport();
    lr.setName(lookupAddresses[0]);
    lr.setSystemResourceUsage(new SystemResourceUsage());
    Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
    bundleStats.put("pulsar/use/primary-ns-01/0x00000000_0x80000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-02/0x00000000_0x80000000", newBundleStats(2, maxSessions + 1, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", newBundleStats(2, 0, maxSessions + 1, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-04/0x00000000_0x80000000", newBundleStats(2, 0, 0, maxMsgRate + 1, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-05/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, maxMsgRate + 1, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-06/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, maxBandwidth + 1, 0));
    bundleStats.put("pulsar/use/primary-ns-07/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-08/0x00000000_0x80000000", newBundleStats(maxTopics - 1, maxSessions - 1, 1, maxMsgRate - 1, 1, maxBandwidth - 1, 1));
    bundleStats.put("pulsar/use/primary-ns-09/0x00000000_0x80000000", newBundleStats(1, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-10/0x00000000_0x02000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    lr.setBundleStats(bundleStats);
    setObjectField(SimpleLoadManagerImpl.class, pulsarServices[0].getLoadManager().get(), "lastLoadReport", lr);
    String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[0]);
    String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
    bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    // sleep to wait load ranking be triggered and trigger bundle split
    Thread.sleep(5000);
    pulsarServices[0].getLoadManager().get().doNamespaceBundleSplit();
    boolean isAutoUnooadSplitBundleEnabled = pulsarServices[0].getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled();
    // verify bundles are split
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-01", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-02", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-03", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-04", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-05", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-06", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-07", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-08", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-09", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000", isAutoUnooadSplitBundleEnabled);
}