Examples with SimpleLoadManagerImpl org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl used on opensource projects

Example 6 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testPrimary.

@Test(enabled = true)
public void testPrimary() throws Exception {
    createNamespacePolicies(pulsar1);
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    // inject the load report and rankings
    Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport.setSystemResourceUsage(new SystemResourceUsage());
    loadReports.put(ru1, loadReport);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), new HashSet<String>(), new ResourceQuota(), new HashSet<String>(), new ResourceQuota());
    Map<ResourceUnit, ResourceUnitRanking> rankings = new HashMap<>();
    rankings.put(ru1, ranking);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "resourceUnitRankings", rankings);
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    // broker is not active so found should be null
    assertNotEquals(found, null, "did not find a broker when expected one to be found");
}

Example 7 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testEvenBundleDistribution.

// Test that bundles belonging to the same namespace are evenly distributed.
@Test
public void testEvenBundleDistribution() throws Exception {
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(pulsar1.getNamespaceService().getNamespaceBundleFactory(), "pulsar", "use", "test", 16);
    final ResourceQuota quota = new ResourceQuota();
    final String quotaZPath = String.format("%s/%s/%s", ResourceQuotaCache.RESOURCE_QUOTA_ROOT, "namespace", bundles[0]);
    // Create high message rate quota for the first bundle to make it unlikely to be a coincidence of even
    // distribution.
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), quotaZPath, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(quota), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    int numAssignedToPrimary = 0;
    int numAssignedToSecondary = 0;
    pulsar1.getConfiguration().setLoadBalancerPlacementStrategy(SimpleLoadManagerImpl.LOADBALANCER_STRATEGY_LLS);
    final SimpleLoadManagerImpl loadManager = (SimpleLoadManagerImpl) pulsar1.getLoadManager().get();
    for (final NamespaceBundle bundle : bundles) {
        if (loadManager.getLeastLoaded(bundle).get().getResourceId().equals(primaryHost)) {
            ++numAssignedToPrimary;
        } else {
            ++numAssignedToSecondary;
        }
        // Check that number of assigned bundles are equivalent when an even number have been assigned.
        if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) {
            assert (numAssignedToPrimary == numAssignedToSecondary);
        }
    }
}

Example 8 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class LoadBalancerTest method testLoadBalanceDistributionAmongUnequallyLoaded.

@Test(enabled = false)
void testLoadBalanceDistributionAmongUnequallyLoaded() throws Exception {
    long memoryMB = 4096;
    long cpuPercent = 25;
    long bInMbps = 256;
    long bOutMbps = 256;
    long threads = 25;
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    int totalAvailabilityWeight = 0;
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    for (int i = 1; i <= 3; i++) {
        PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
        ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
        LoadRanker ranker = new ResourceAvailabilityRanker();
        long rank = ranker.getRank(rd);
        if (sortedRankingsInstance.containsKey(rank)) {
            // get the object set and put the rd in it
            sortedRankingsInstance.get(rank).add(ru1);
        } else {
            Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
            rus.add(ru1);
            sortedRankingsInstance.put(rank, rus);
        }
        totalAvailabilityWeight += rank;
    }
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
    int totalNamespaces = 1000;
    Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
    for (int i = 0; i < totalNamespaces; i++) {
        ResourceUnit found = loadManager.getLeastLoaded(TopicName.get("persistent://pulsar/use/primary-ns/topic-" + i)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 0);
        }
    }
    for (Map.Entry<Long, Set<ResourceUnit>> entry : sortedRankingsInstance.entrySet()) {
        int selectionProbability = (int) (((double) entry.getKey() / totalAvailabilityWeight) * 100);
        int idealExpectedOwned = selectionProbability * (int) ((double) totalNamespaces / 100);
        int expectedOwnedLowerBound = idealExpectedOwned - idealExpectedOwned / 10;
        int expectedOwnedUpperBound = idealExpectedOwned + idealExpectedOwned / 10;
        for (ResourceUnit ru : entry.getValue()) {
            assertTrue(namespaceOwner.containsKey(ru.getResourceId()));
            int ownedNamespaces = namespaceOwner.get(ru.getResourceId());
            assertTrue(ownedNamespaces > expectedOwnedLowerBound || ownedNamespaces < expectedOwnedUpperBound);
        }
    }
}

Example 9 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class LoadBalancerTest method testLoadbalanceDistributionAmongEquallyLoaded.

/*
     * Simple Test, creates three Resource Units (brokers) with equal load and expects that out of 1005 namespaces it
     * should be divided fairly equally with about 10% of variation
     *
     */
@Test(enabled = false)
public void testLoadbalanceDistributionAmongEquallyLoaded() throws Exception {
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    long memoryMB = 4096;
    long cpuPercent = 45;
    long bInMbps = 350;
    long bOutMbps = 180;
    long threads = 10;
    // TODO move to its own test
    PulsarResourceDescription rd = createResourceDescription(memoryMB, cpuPercent, bInMbps, bOutMbps, threads);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    for (String broker : activeBrokers) {
        ResourceUnit ru = new SimpleResourceUnit(broker, rd);
        rus.add(ru);
    }
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    LoadRanker ranker = new ResourceAvailabilityRanker();
    sortedRankingsInstance.put(ranker.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
}

Example 10 with SimpleLoadManagerImpl

use of org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testDoLoadShedding.

@Test(enabled = true)
public void testDoLoadShedding() throws Exception {
    LoadManager loadManager = spy(new SimpleLoadManagerImpl(pulsar1));
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://pulsar-broker1.com:8080", rd);
    ResourceUnit ru2 = new SimpleResourceUnit("http://pulsar-broker2.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    rus.add(ru2);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    // inject the load report and rankings
    SystemResourceUsage systemResource = new SystemResourceUsage();
    systemResource.setBandwidthIn(new ResourceUsage(90, 100));
    Map<String, NamespaceBundleStats> stats = Maps.newHashMap();
    NamespaceBundleStats nsb1 = new NamespaceBundleStats();
    nsb1.msgRateOut = 10000;
    NamespaceBundleStats nsb2 = new NamespaceBundleStats();
    nsb2.msgRateOut = 10000;
    stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1);
    stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2);
    Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport1 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport1.setSystemResourceUsage(systemResource);
    loadReport1.setBundleStats(stats);
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport2 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport2.setSystemResourceUsage(new SystemResourceUsage());
    loadReport2.setBundleStats(stats);
    loadReports.put(ru1, loadReport1);
    loadReports.put(ru2, loadReport2);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ((SimpleLoadManagerImpl) loadManager).doLoadShedding();
    verify(loadManager, atLeastOnce()).doLoadShedding();
}