Examples with SystemResourceUsage com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 11 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImpl method fromLoadReport.

/*
     * temp method, remove it in future, in-place to make this glue code to make load balancing work
     */
private PulsarResourceDescription fromLoadReport(LoadReport report) {
    SystemResourceUsage sru = report.getSystemResourceUsage();
    PulsarResourceDescription resourceDescription = new PulsarResourceDescription();
    if (sru.bandwidthIn != null)
        resourceDescription.put("bandwidthIn", sru.bandwidthIn);
    if (sru.bandwidthOut != null)
        resourceDescription.put("bandwidthOut", sru.bandwidthOut);
    if (sru.memory != null)
        resourceDescription.put("memory", sru.memory);
    if (sru.cpu != null)
        resourceDescription.put("cpu", sru.cpu);
    return resourceDescription;
}

Example 12 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testUsage.

@Test
public void testUsage() {
    Map<String, Object> metrics = Maps.newHashMap();
    metrics.put("brk_conn_cnt", new Long(1));
    metrics.put("brk_repl_conn_cnt", new Long(1));
    metrics.put("jvm_thread_cnt", new Long(1));
    BrokerUsage brokerUsage = BrokerUsage.populateFrom(metrics);
    assertEquals(brokerUsage.getConnectionCount(), 1);
    assertEquals(brokerUsage.getReplicationConnectionCount(), 1);
    JvmUsage jvmUage = JvmUsage.populateFrom(metrics);
    assertEquals(jvmUage.getThreadCount(), 1);
    SystemResourceUsage usage = new SystemResourceUsage();
    double usageLimit = 10.0;
    usage.setBandwidthIn(new ResourceUsage(usageLimit, usageLimit));
    assertEquals(usage.getBandwidthIn().usage, usageLimit);
    usage.reset();
    assertNotEquals(usage.getBandwidthIn().usage, usageLimit);
}

Example 13 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testLoadReportParsing.

@Test
public void testLoadReportParsing() throws Exception {
    ObjectMapper mapper = ObjectMapperFactory.create();
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport reportData = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    reportData.setName("b1");
    SystemResourceUsage resource = new SystemResourceUsage();
    ResourceUsage resourceUsage = new ResourceUsage();
    resource.setBandwidthIn(resourceUsage);
    resource.setBandwidthOut(resourceUsage);
    resource.setMemory(resourceUsage);
    resource.setCpu(resourceUsage);
    reportData.setSystemResourceUsage(resource);
    String loadReportJson = mapper.writeValueAsString(reportData);
    LoadReport loadReport = PulsarLoadReportImpl.parse(loadReportJson);
    assertEquals(loadReport.getResourceUnitDescription().getResourceUsage().get("bandwidthIn").compareTo(resourceUsage), 0);
}

Example 14 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testDoLoadShedding.

@Test(enabled = true)
public void testDoLoadShedding() throws Exception {
    LoadManager loadManager = spy(new SimpleLoadManagerImpl(pulsar1));
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker7.messaging.gq1.yahoo.com:8080", rd);
    ResourceUnit ru2 = new SimpleResourceUnit("http://prod2-broker7.messaging.gq1.yahoo.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    rus.add(ru2);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    // inject the load report and rankings
    SystemResourceUsage systemResource = new SystemResourceUsage();
    systemResource.setBandwidthIn(new ResourceUsage(90, 100));
    Map<String, NamespaceBundleStats> stats = Maps.newHashMap();
    NamespaceBundleStats nsb1 = new NamespaceBundleStats();
    nsb1.msgRateOut = 10000;
    NamespaceBundleStats nsb2 = new NamespaceBundleStats();
    nsb2.msgRateOut = 10000;
    stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1);
    stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2);
    Map<ResourceUnit, com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport1 = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    loadReport1.setSystemResourceUsage(systemResource);
    loadReport1.setBundleStats(stats);
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport2 = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    loadReport2.setSystemResourceUsage(new SystemResourceUsage());
    loadReport2.setBundleStats(stats);
    loadReports.put(ru1, loadReport1);
    loadReports.put(ru2, loadReport2);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ((SimpleLoadManagerImpl) loadManager).doLoadShedding();
    verify(loadManager, atLeastOnce()).doLoadShedding();
}

Example 15 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testPrimary.

@Test(enabled = true)
public void testPrimary() throws Exception {
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    // inject the load report and rankings
    Map<ResourceUnit, com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    loadReport.setSystemResourceUsage(new SystemResourceUsage());
    loadReports.put(ru1, loadReport);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), new HashSet<String>(), new ResourceQuota(), new HashSet<String>(), new ResourceQuota());
    Map<ResourceUnit, ResourceUnitRanking> rankings = new HashMap<>();
    rankings.put(ru1, ranking);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "resourceUnitRankings", rankings);
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
    // broker is not active so found should be null
    assertNotEquals(found, null, "did not find a broker when expected one to be found");
}