Examples with ResourceUsage org.apache.pulsar.policies.data.loadbalancer.ResourceUsage used on opensource projects

Example 1 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class LinuxBrokerHostUsageImpl method calculateBrokerHostUsage.

private void calculateBrokerHostUsage() {
    List<String> nics = getNics();
    double totalNicLimit = getTotalNicLimitKbps(nics);
    double totalNicUsageTx = getTotalNicUsageTxKb(nics);
    double totalNicUsageRx = getTotalNicUsageRxKb(nics);
    double totalCpuLimit = getTotalCpuLimit();
    CpuStat cpuStat = getTotalCpuUsage();
    SystemResourceUsage usage = new SystemResourceUsage();
    long now = System.currentTimeMillis();
    if (lastCollection == 0L) {
        usage.setMemory(getMemUsage());
        usage.setBandwidthIn(new ResourceUsage(0d, totalNicLimit));
        usage.setBandwidthOut(new ResourceUsage(0d, totalNicLimit));
        usage.setCpu(new ResourceUsage(0d, totalCpuLimit));
    } else {
        double elapsedSeconds = (now - lastCollection) / 1000d;
        double nicUsageTx = (totalNicUsageTx - lastTotalNicUsageTx) / elapsedSeconds;
        double nicUsageRx = (totalNicUsageRx - lastTotalNicUsageRx) / elapsedSeconds;
        if (cpuStat != null && lastCpuStat != null) {
            // we need two non null stats to get a usage report
            long cpuTimeDiff = cpuStat.getTotalTime() - lastCpuStat.getTotalTime();
            long cpuUsageDiff = cpuStat.getUsage() - lastCpuStat.getUsage();
            double cpuUsage = ((double) cpuUsageDiff / (double) cpuTimeDiff) * totalCpuLimit;
            usage.setCpu(new ResourceUsage(cpuUsage, totalCpuLimit));
        }
        usage.setMemory(getMemUsage());
        usage.setBandwidthIn(new ResourceUsage(nicUsageRx, totalNicLimit));
        usage.setBandwidthOut(new ResourceUsage(nicUsageTx, totalNicLimit));
    }
    lastTotalNicUsageTx = totalNicUsageTx;
    lastTotalNicUsageRx = totalNicUsageRx;
    lastCpuStat = cpuStat;
    lastCollection = System.currentTimeMillis();
    this.usage = usage;
}

Example 2 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class LookupDataTest method testLoadReportSerialization.

@Test
public void testLoadReportSerialization() throws Exception {
    final String simpleLmBrokerUrl = "simple";
    final String simpleLmReportName = "simpleLoadManager";
    final String modularLmBrokerUrl = "modular";
    final SystemResourceUsage simpleLmSystemResourceUsage = new SystemResourceUsage();
    final ResourceUsage resource = new ResourceUsage();
    final double usage = 55.0;
    resource.usage = usage;
    simpleLmSystemResourceUsage.bandwidthIn = resource;
    LoadReport simpleReport = getSimpleLoadManagerLoadReport(simpleLmBrokerUrl, simpleLmReportName, simpleLmSystemResourceUsage);
    LocalBrokerData modularReport = getModularLoadManagerLoadReport(modularLmBrokerUrl, resource);
    LoadManagerReport simpleLoadReport = ObjectMapperFactory.getThreadLocal().readValue(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(simpleReport), LoadManagerReport.class);
    LoadManagerReport modularLoadReport = ObjectMapperFactory.getThreadLocal().readValue(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(modularReport), LoadManagerReport.class);
    assertEquals(simpleLoadReport.getWebServiceUrl(), simpleLmBrokerUrl);
    assertTrue(simpleLoadReport instanceof LoadReport);
    assertEquals(((LoadReport) simpleLoadReport).getName(), simpleLmReportName);
    assertEquals(((LoadReport) simpleLoadReport).getSystemResourceUsage().bandwidthIn.usage, usage);
    assertEquals(modularLoadReport.getWebServiceUrl(), modularLmBrokerUrl);
    assertTrue(modularLoadReport instanceof LocalBrokerData);
    assertEquals(((LocalBrokerData) modularLoadReport).getBandwidthIn().usage, usage);
}

Example 3 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method testBrokerRanking.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - non-bundles assigned so all
     * idle resources are avaiable for new bundle Check the broker rankings are the load percentage of each broker.
     */
@Test
public void testBrokerRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(1024, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // check the ranking result
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
        // brokers' ranking would be:
        // 50 --> broker 0 ( 1024 / 2048 )
        // 25 --> broker 1 ( 1024 / 4096 )
        // 16 --> broker 2 ( 1024 / 6144 )
        // 15 --> broker 3 ( 60 / 400 )
        // 15 --> broker 4 ( 60 / 400 )
        assertEquals(sortedRanking.get().size(), 4);
        assertEquals(sortedRanking.get().get(50L).size(), 1);
        assertEquals(sortedRanking.get().get(25L).size(), 1);
        assertEquals(sortedRanking.get().get(16L).size(), 1);
        assertEquals(sortedRanking.get().get(15L).size(), 2);
    }
}

Example 4 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method testGetLeastLoadedBasic.

@Test(enabled = false)
public void testGetLeastLoadedBasic() throws Exception {
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[0]);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[1]);
    zkCacheField.set(pulsarServices[0], mockCache);
    zkCacheField.set(pulsarServices[1], mockCache);
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    // TODO move to its own test
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker1.messaging.use.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    assertEquals("http://prod1-broker1.messaging.use.example.com:8080", found.getResourceId());
    zkCacheField.set(pulsarServices[0], originalLZK1);
    zkCacheField.set(pulsarServices[1], originalLZK2);
}

Example 5 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method createResourceDescription.

/*
     * creates a ResourceDescription where the max limits for different parameters are as below
     *
     * Memory = 16GB cpu = 100 percent bandwidthIn = 1Gbps bandwidthOut = 1Gbps threads = 100
     */
private PulsarResourceDescription createResourceDescription(long memoryInMB, long cpuPercentage, long bandwidthInMbps, long bandwidthOutInMbps, long threads) {
    long KB = 1024;
    long MB = 1024 * KB;
    long GB = 1024 * MB;
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(memoryInMB, 4 * GB));
    rd.put("cpu", new ResourceUsage(cpuPercentage, 100));
    rd.put("bandwidthIn", new ResourceUsage(bandwidthInMbps * MB, GB));
    rd.put("bandwidthOut", new ResourceUsage(bandwidthOutInMbps * MB, GB));
    return rd;
}