Examples with SimpleResourceUnit com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit used on opensource projects

Example 6 with SimpleResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit in project pulsar by yahoo.

the class BrokerServiceLookupTest method testMultipleBrokerDifferentClusterLookup.

/**
     * Usecase: Redirection due to different cluster 
     * 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster: "use2" 
     * 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
     * redirects request to Broker2 whch serves "use2"
     * 3. Broker2 receives redirect request and own namespace bundle
     * 
     * @throws Exception
     */
@Test
public void testMultipleBrokerDifferentClusterLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**** start broker-2 ****/
    final String newCluster = "use2";
    final String property = "my-property2";
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    // Broker2 serves newCluster
    conf2.setClusterName(newCluster);
    String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort();
    admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
    admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster)));
    admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    URI brokerServiceUrl = new URI(broker2ServiceUrl);
    PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
    // enable authorization: so, broker can validate cluster and redirect if finds different cluster
    pulsar.getConfiguration().setAuthorizationEnabled(true);
    // restart broker with authorization enabled: it initialize AuthorizationManager
    stopBroker();
    startBroker();
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(resourceUnit).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), loadManager2);
    /**** started broker-2 ****/
    // load namespace-bundle by calling Broker2
    Consumer consumer = pulsarClient.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name", new ConsumerConfiguration());
    Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration());
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    // disable authorization 
    pulsar.getConfiguration().setAuthorizationEnabled(false);
    pulsarClient2.close();
    pulsar2.close();
    loadManager2 = null;
}

Example 7 with SimpleResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit in project pulsar by yahoo.

the class LoadBalancerTest method testLoadBalanceDiscardingInactiveBrokersInSelection.

/**
     * This test puts few brokers in the ranking that load balance uses but does not put few brokers in the mock
     * zookeeper cache, if the broker is not in the zk cache, it's considered inactive.
     *
     * We should not see any of these inactive brokers assigned any namespace.
     */
@Test(enabled = false)
void testLoadBalanceDiscardingInactiveBrokersInSelection() throws Exception {
    long memoryMB = 2096;
    long cpuPercent = 12;
    long bInMbps = 100;
    long bOutMbps = 100;
    long threads = 3;
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    for (int i = 1; i <= 3; i++) {
        PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
        ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
        LoadRanker ranker = new ResourceAvailabilityRanker();
        long rank = ranker.getRank(rd);
        if (sortedRankingsInstance.containsKey(rank)) {
            // get the object set and put the rd in it
            sortedRankingsInstance.get(rank).add(ru1);
        } else {
            Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
            rus.add(ru1);
            sortedRankingsInstance.put(rank, rus);
        }
    }
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
    int totalNamespaces = 10;
    Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
    for (int i = 0; i < totalNamespaces; i++) {
        ResourceUnit found = loadManager.getLeastLoaded(DestinationName.get("persistent://pulsar/use/primary-ns/topic" + i));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 0);
        }
    }
    String inactiveBroker = "prod1-broker3.messaging.use.example.com:8080";
    // check owner list contains only two entries, broker-3 should not be in
    assertTrue(namespaceOwner.size() == 2);
    assertTrue(!namespaceOwner.containsKey(inactiveBroker));
}