Examples with LoadReport com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport used on opensource projects

Example 6 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class SimpleLoadManagerImpl method isBrokerAvailableForRebalancing.

// todo: changeme: this can be optimized, we don't have to iterate through everytime
private boolean isBrokerAvailableForRebalancing(String bundleName, long maxLoadLevel) {
    NamespaceName namespaceName = new NamespaceName(getNamespaceNameFromBundleName(bundleName));
    Map<Long, Set<ResourceUnit>> availableBrokers = sortedRankings.get();
    // this does not have "http://" in front, hacky but no time to pretty up
    Multimap<Long, ResourceUnit> brokers = getFinalCandidates(namespaceName, availableBrokers);
    for (Object broker : brokers.values()) {
        ResourceUnit underloadedRU = (ResourceUnit) broker;
        LoadReport currentLoadReport = currentLoadReports.get(underloadedRU);
        if (isBelowLoadLevel(currentLoadReport.getSystemResourceUsage(), maxLoadLevel)) {
            return true;
        }
    }
    return false;
}

Example 7 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class SimpleLoadManagerImpl method updateRanking.

private void updateRanking() {
    try {
        synchronized (currentLoadReports) {
            currentLoadReports.clear();
            Set<String> activeBrokers = availableActiveBrokers.get();
            for (String broker : activeBrokers) {
                try {
                    String key = String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, broker);
                    LoadReport lr = loadReportCacheZk.get(key).orElseThrow(() -> new KeeperException.NoNodeException());
                    ResourceUnit ru = new SimpleResourceUnit(String.format("http://%s", lr.getName()), fromLoadReport(lr));
                    this.currentLoadReports.put(ru, lr);
                } catch (Exception e) {
                    log.warn("Error reading load report from Cache for broker - [{}], [{}]", broker, e);
                }
            }
            updateRealtimeResourceQuota();
            doLoadRanking();
        }
    } catch (Exception e) {
        log.warn("Error reading active brokers list from zookeeper while re-ranking load reports [{}]", e);
    }
}

Example 8 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class SimpleLoadManagerImpl method generateLoadReport.

@Override
public LoadReport generateLoadReport() throws Exception {
    long timeSinceLastGenMillis = System.currentTimeMillis() - lastLoadReport.getTimestamp();
    if (timeSinceLastGenMillis <= LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL) {
        return lastLoadReport;
    }
    try {
        LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
        loadReport.setName(String.format("%s:%s", pulsar.getAdvertisedAddress(), pulsar.getConfiguration().getWebServicePort()));
        SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage();
        loadReport.setOverLoaded(isAboveLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerOverloadedThresholdPercentage()));
        loadReport.setUnderLoaded(isBelowLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerUnderloadedThresholdPercentage()));
        loadReport.setSystemResourceUsage(systemResourceUsage);
        loadReport.setBundleStats(pulsar.getBrokerService().getBundleStats());
        loadReport.setTimestamp(System.currentTimeMillis());
        return loadReport;
    } catch (Exception e) {
        log.error("[{}] Failed to generate LoadReport for broker, reason [{}]", e.getMessage(), e);
        throw e;
    }
}

Example 9 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class ServerConnection method sendLookupResponse.

private void sendLookupResponse(long requestId) {
    try {
        LoadReport availableBroker = service.getDiscoveryProvider().nextBroker();
        ctx.writeAndFlush(Commands.newLookupResponse(availableBroker.getPulsarServiceUrl(), availableBroker.getPulsarServieUrlTls(), false, Redirect, requestId));
    } catch (PulsarServerException e) {
        LOG.warn("[{}] Failed to get next active broker {}", remoteAddress, e.getMessage(), e);
        ctx.writeAndFlush(Commands.newLookupResponse(ServerError.ServiceNotReady, e.getMessage(), requestId));
    }
}

Example 10 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class ZookeeperCacheLoaderTest method testZookeeperCacheLoader.

/**
     * Create znode for available broker in ZooKeeper and updates it again to verify ZooKeeper cache update
     *
     * @throws InterruptedException
     * @throws KeeperException
     * @throws IOException
     */
@Test
public void testZookeeperCacheLoader() throws InterruptedException, KeeperException, Exception {
    DiscoveryZooKeeperClientFactoryImpl.zk = mockZookKeeper;
    ZookeeperCacheLoader zkLoader = new ZookeeperCacheLoader(new DiscoveryZooKeeperClientFactoryImpl(), "");
    List<String> brokers = Lists.newArrayList("broker-1:15000", "broker-2:15000", "broker-3:15000");
    // 1. create znode for each broker
    brokers.stream().forEach(b -> {
        try {
            zkLoader.getLocalZkCache().getZooKeeper().create(LOADBALANCE_BROKERS_ROOT + "/" + b, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        } catch (KeeperException | InterruptedException e) {
            fail("failed while creating broker znodes");
        }
    });
    // wait for 100 msec: to get cache updated
    Thread.sleep(100);
    // 2. get available brokers from ZookeeperCacheLoader
    List<LoadReport> list = zkLoader.getAvailableBrokers();
    // 3. verify retrieved broker list
    Assert.assertTrue(brokers.containsAll(list));
    // 4.a add new broker
    zkLoader.getLocalZkCache().getZooKeeper().create(LOADBALANCE_BROKERS_ROOT + "/" + "broker-4:15000", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    brokers.add("broker-4:15000");
    // wait for 100 msec: to get cache updated
    Thread.sleep(100);
    // 4.b. get available brokers from ZookeeperCacheLoader
    list = zkLoader.getAvailableBrokers();
    // 4.c. verify retrieved broker list
    Assert.assertTrue(brokers.containsAll(list));
}