Examples with SystemResourceUsage org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 16 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testUsage.

@Test
public void testUsage() {
    Map<String, Object> metrics = Maps.newHashMap();
    metrics.put("brk_conn_cnt", new Long(1));
    metrics.put("brk_repl_conn_cnt", new Long(1));
    metrics.put("jvm_thread_cnt", new Long(1));
    BrokerUsage brokerUsage = BrokerUsage.populateFrom(metrics);
    assertEquals(brokerUsage.getConnectionCount(), 1);
    assertEquals(brokerUsage.getReplicationConnectionCount(), 1);
    JvmUsage jvmUage = JvmUsage.populateFrom(metrics);
    assertEquals(jvmUage.getThreadCount(), 1);
    SystemResourceUsage usage = new SystemResourceUsage();
    double usageLimit = 10.0;
    usage.setBandwidthIn(new ResourceUsage(usageLimit, usageLimit));
    assertEquals(usage.getBandwidthIn().usage, usageLimit);
    usage.reset();
    assertNotEquals(usage.getBandwidthIn().usage, usageLimit);
}

Example 17 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method getSystemResourceUsage.

public SystemResourceUsage getSystemResourceUsage() throws IOException {
    SystemResourceUsage systemResourceUsage = LoadManagerShared.getSystemResourceUsage(brokerHostUsage);
    long memoryUsageInMBytes = getAverageJvmHeapUsageMBytes();
    systemResourceUsage.memory.usage = (double) memoryUsageInMBytes;
    return systemResourceUsage;
}

Example 18 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class GenericBrokerHostUsageImpl method calculateBrokerHostUsage.

private void calculateBrokerHostUsage() {
    SystemResourceUsage usage = new SystemResourceUsage();
    usage.setCpu(getCpuUsage());
    usage.setMemory(getMemUsage());
    this.usage = usage;
}

Example 19 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method writeLoadReportsForDynamicQuota.

private void writeLoadReportsForDynamicQuota(long timestamp) throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        lr.setTimestamp(timestamp);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(5000 * (10 + i * 5), 1024000));
        sru.setBandwidthOut(new ResourceUsage(15000 * (10 + i * 5), 1024000));
        sru.setMemory(new ResourceUsage(25 * (10 + i * 5), 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(200, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            stats.msgRateIn = 5 * (i + j);
            stats.msgRateOut = 15 * (i + j);
            stats.msgThroughputIn = 5000 * (i + j);
            stats.msgThroughputOut = 15000 * (i + j);
            stats.topics = 25 * (i + j);
            stats.consumerCount = 50 * (i + j);
            stats.producerCount = 50 * (i + j);
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
}

Example 20 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method testUpdateLoadReportAndCheckUpdatedRanking.

/*
     * tests rankings get updated when we write write the new load reports to the zookeeper on loadbalance root node
     * tests writing pre-configured load report on the zookeeper translates the pre-calculated rankings
     */
@Test
public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(256, 1024000));
        sru.setBandwidthOut(new ResourceUsage(250, 1024000));
        sru.setMemory(new ResourceUsage(1024, 8192));
        sru.setCpu(new ResourceUsage(5, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait the load ranking be triggered
    Thread.sleep(5000);
    // do lookup for bunch of bundles
    int totalNamespaces = 200;
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        TopicName topicName = TopicName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().get().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(topicName)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    // assert that distribution variation is not more than 10%
    int averageNamespaces = totalNamespaces / BROKER_COUNT;
    int tenPercentOfAverageNamespaces = averageNamespaces / 10;
    int lowerBound = averageNamespaces - tenPercentOfAverageNamespaces;
    int upperBound = averageNamespaces + tenPercentOfAverageNamespaces;
    // assert each broker received ownership of fair amount of namespaces 90%+
    for (Map.Entry<String, Integer> broker : namespaceOwner.entrySet()) {
        log.info("Count of bundles assigned: {}, {}", broker.getKey(), broker.getValue());
        assertTrue(broker.getValue() >= lowerBound && broker.getValue() <= upperBound);
    }
}