Examples with SimpleLoadManagerImpl org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl used on opensource projects

Example 1 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class LoadManager method create.

static LoadManager create(final PulsarService pulsar) {
    try {
        final ServiceConfiguration conf = pulsar.getConfiguration();
        final Class<?> loadManagerClass = Class.forName(conf.getLoadManagerClassName());
        // Assume there is a constructor with one argument of PulsarService.
        final Object loadManagerInstance = loadManagerClass.newInstance();
        if (loadManagerInstance instanceof LoadManager) {
            final LoadManager casted = (LoadManager) loadManagerInstance;
            casted.initialize(pulsar);
            return casted;
        } else if (loadManagerInstance instanceof ModularLoadManager) {
            final LoadManager casted = new ModularLoadManagerWrapper((ModularLoadManager) loadManagerInstance);
            casted.initialize(pulsar);
            return casted;
        }
    } catch (Exception e) {
        log.warn("Error when trying to create load manager: {}");
    }
    // If we failed to create a load manager, default to SimpleLoadManagerImpl.
    return new SimpleLoadManagerImpl(pulsar);
}

Example 2 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class LoadBalancerTest method testLoadBalanceDiscardingInactiveBrokersInSelection.

/**
 * This test puts few brokers in the ranking that load balance uses but does not put few brokers in the mock
 * zookeeper cache, if the broker is not in the zk cache, it's considered inactive.
 *
 * We should not see any of these inactive brokers assigned any namespace.
 */
@Test(enabled = false)
void testLoadBalanceDiscardingInactiveBrokersInSelection() throws Exception {
    long memoryMB = 2096;
    long cpuPercent = 12;
    long bInMbps = 100;
    long bOutMbps = 100;
    long threads = 3;
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    for (int i = 1; i <= 3; i++) {
        PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
        ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
        LoadRanker ranker = new ResourceAvailabilityRanker();
        long rank = ranker.getRank(rd);
        if (sortedRankingsInstance.containsKey(rank)) {
            // get the object set and put the rd in it
            sortedRankingsInstance.get(rank).add(ru1);
        } else {
            Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
            rus.add(ru1);
            sortedRankingsInstance.put(rank, rus);
        }
    }
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
    int totalNamespaces = 10;
    Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
    for (int i = 0; i < totalNamespaces; i++) {
        ResourceUnit found = loadManager.getLeastLoaded(TopicName.get("persistent://pulsar/use/primary-ns/topic" + i)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 0);
        }
    }
    String inactiveBroker = "prod1-broker3.messaging.use.example.com:8080";
    // check owner list contains only two entries, broker-3 should not be in
    assertTrue(namespaceOwner.size() == 2);
    assertTrue(!namespaceOwner.containsKey(inactiveBroker));
}

Example 3 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class LoadBalancerTest method testGetLeastLoadedBasic.

@Test(enabled = false)
public void testGetLeastLoadedBasic() throws Exception {
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[0]);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[1]);
    zkCacheField.set(pulsarServices[0], mockCache);
    zkCacheField.set(pulsarServices[1], mockCache);
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    // TODO move to its own test
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker1.messaging.use.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    assertEquals("http://prod1-broker1.messaging.use.example.com:8080", found.getResourceId());
    zkCacheField.set(pulsarServices[0], originalLZK1);
    zkCacheField.set(pulsarServices[1], originalLZK2);
}

Example 4 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class LoadBalancerTest method testStop.

/**
 * Ensure that the load manager's zookeeper data cache is shutdown after invoking stop().
 */
@Test
public void testStop() throws Exception {
    final SimpleLoadManagerImpl loadManager = (SimpleLoadManagerImpl) pulsarServices[0].getLoadManager().get();
    loadManager.stop();
    Field loadReportCacheField = SimpleLoadManagerImpl.class.getDeclaredField("loadReportCacheZk");
    loadReportCacheField.setAccessible(true);
    ZooKeeperDataCache<LoadReport> loadReportCache = (ZooKeeperDataCache<LoadReport>) loadReportCacheField.get(loadManager);
    Field IS_SHUTDOWN_UPDATER = ZooKeeperDataCache.class.getDeclaredField("IS_SHUTDOWN_UPDATER");
    IS_SHUTDOWN_UPDATER.setAccessible(true);
    final int TRUE = 1;
    assert (((AtomicIntegerFieldUpdater<ZooKeeperDataCache>) (IS_SHUTDOWN_UPDATER.get(loadReportCache))).get(loadReportCache) == TRUE);
}

Example 5 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testPrimarySecondary.

@Test(enabled = false)
public void testPrimarySecondary() throws Exception {
    createNamespacePolicies(pulsar1);
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    ZooKeeperChildrenCache zooKeeperChildrenCache = mock(ZooKeeperChildrenCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod2-broker7.messaging.use.example.com:8080", "prod2-broker8.messaging.use.example.com:8080", "prod2-broker9.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get()).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsar1);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsar2);
    log.info("lzk are {} 2: {}", originalLZK1.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT), originalLZK2.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    zkCacheField.set(pulsar1, mockCache);
    LocalZooKeeperCache newZk = (LocalZooKeeperCache) pulsar1.getLocalZkCache();
    log.info("lzk mocked are {}", newZk.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    ZooKeeperChildrenCache availableActiveBrokers = new ZooKeeperChildrenCache(pulsar1.getLocalZkCache(), SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT);
    log.info("lzk mocked active brokers are {}", availableActiveBrokers.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    assertEquals(found.getResourceId(), ru1.getResourceId());
    zkCacheField.set(pulsar1, originalLZK1);
}