Examples with SystemResourceUsage com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 1 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class LoadBalancerTest method testBrokerRanking.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - non-bundles assigned so all
     * idle resources are avaiable for new bundle Check the broker rankings are the load percentage of each broker.
     */
@Test
public void testBrokerRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(1024, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // check the ranking result
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
        // brokers' ranking would be:
        // 50 --> broker 0 ( 1024 / 2048 )
        // 25 --> broker 1 ( 1024 / 4096 )
        // 16 --> broker 2 ( 1024 / 6144 )
        // 15 --> broker 3 ( 60 / 400 )
        // 15 --> broker 4 ( 60 / 400 )
        assertEquals(sortedRanking.get().size(), 4);
        assertEquals(sortedRanking.get().get(50L).size(), 1);
        assertEquals(sortedRanking.get().get(25L).size(), 1);
        assertEquals(sortedRanking.get().get(16L).size(), 1);
        assertEquals(sortedRanking.get().get(15L).size(), 2);
    }
}

Example 2 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class LoadBalancerTest method testDestinationAssignmentWithExistingBundles.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - already has some bundles
     * assigned Check the distribution of new destinations is roughly consistent (with <10% variation) with the ranking
     */
@Test
public void testDestinationAssignmentWithExistingBundles() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(87);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(0, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < (i + 1) * 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // print ranking
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
    }
    // check owner of new destiations and verify that the distribution is roughly
    // consistent (variation < 10%) with the broker capacity:
    int totalNamespaces = 250;
    int[] expectedAssignments = new int[] { 17, 34, 51, 68, 85 };
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    double expectedMaxVariation = 10.0;
    for (int i = 0; i < BROKER_COUNT; i++) {
        long actualValue = 0;
        String resourceId = "http://" + lookupAddresses[i];
        if (namespaceOwner.containsKey(resourceId)) {
            actualValue = namespaceOwner.get(resourceId);
        }
        long expectedValue = expectedAssignments[i];
        double variation = Math.abs(actualValue - expectedValue) * 100.0 / expectedValue;
        log.info("Destination assignment - {}, actual: {}, expected baseline: {}, variation: {}/%", lookupAddresses[i], actualValue, expectedValue, String.format("%.2f", variation));
        assertTrue(variation < expectedMaxVariation);
    }
}

Example 3 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class LoadBalancerTest method writeLoadReportsForDynamicQuota.

private void writeLoadReportsForDynamicQuota(long timestamp) throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        lr.setTimestamp(timestamp);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(5000 * (10 + i * 5), 1024000));
        sru.setBandwidthOut(new ResourceUsage(15000 * (10 + i * 5), 1024000));
        sru.setMemory(new ResourceUsage(25 * (10 + i * 5), 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(200, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            stats.msgRateIn = 5 * (i + j);
            stats.msgRateOut = 15 * (i + j);
            stats.msgThroughputIn = 5000 * (i + j);
            stats.msgThroughputOut = 15000 * (i + j);
            stats.topics = 25 * (i + j);
            stats.consumerCount = 50 * (i + j);
            stats.producerCount = 50 * (i + j);
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
}

Example 4 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class GenericBrokerHostUsageImpl method calculateBrokerHostUsage.

private void calculateBrokerHostUsage() {
    SystemResourceUsage usage = new SystemResourceUsage();
    usage.setCpu(getCpuUsage());
    usage.setMemory(getMemUsage());
    this.usage = usage;
}

Example 5 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class LinuxBrokerHostUsageImpl method calculateBrokerHostUsage.

private void calculateBrokerHostUsage() {
    List<String> nics = getNics();
    double totalNicLimit = getTotalNicLimitKbps(nics);
    double totalNicUsageTx = getTotalNicUsageTxKb(nics);
    double totalNicUsageRx = getTotalNicUsageRxKb(nics);
    double totalCpuLimit = getTotalCpuLimit();
    CpuStat cpuStat = getTotalCpuUsage();
    SystemResourceUsage usage = new SystemResourceUsage();
    long now = System.currentTimeMillis();
    if (lastCollection == 0L) {
        usage.setMemory(getMemUsage());
        usage.setBandwidthIn(new ResourceUsage(0d, totalNicLimit));
        usage.setBandwidthOut(new ResourceUsage(0d, totalNicLimit));
        usage.setCpu(new ResourceUsage(0d, totalCpuLimit));
    } else {
        double elapsedSeconds = (now - lastCollection) / 1000d;
        double nicUsageTx = (totalNicUsageTx - lastTotalNicUsageTx) / elapsedSeconds;
        double nicUsageRx = (totalNicUsageRx - lastTotalNicUsageRx) / elapsedSeconds;
        if (cpuStat != null && lastCpuStat != null) {
            // we need two non null stats to get a usage report
            long cpuTimeDiff = cpuStat.getTotalTime() - lastCpuStat.getTotalTime();
            long cpuUsageDiff = cpuStat.getUsage() - lastCpuStat.getUsage();
            double cpuUsage = ((double) cpuUsageDiff / (double) cpuTimeDiff) * totalCpuLimit;
            usage.setCpu(new ResourceUsage(cpuUsage, totalCpuLimit));
        }
        usage.setMemory(getMemUsage());
        usage.setBandwidthIn(new ResourceUsage(nicUsageRx, totalNicLimit));
        usage.setBandwidthOut(new ResourceUsage(nicUsageTx, totalNicLimit));
    }
    lastTotalNicUsageTx = totalNicUsageTx;
    lastTotalNicUsageRx = totalNicUsageRx;
    lastCpuStat = cpuStat;
    lastCollection = System.currentTimeMillis();
    this.usage = usage;
}