Examples with ServiceConfiguration org.apache.pulsar.broker.ServiceConfiguration used on opensource projects

Example 21 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class ModularLoadManagerStrategyTest method testLeastLongTermMessageRate.

// Test that least long term message rate works correctly.
@Test
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}

Example 22 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testMultipleBrokerLookup.

/**
 * UsecaseL Multiple Broker => Lookup Redirection test
 *
 * 1. Broker1 is a leader 2. Lookup request reaches to Broker2 which redirects to leader (Broker1) with
 * authoritative = false 3. Leader (Broker1) finds out least loaded broker as Broker2 and redirects request to
 * Broker2 with authoritative = true 4. Broker2 receives final request to own a bundle with authoritative = true and
 * client connects to Broker2
 *
 * @throws Exception
 */
@Test
public void testMultipleBrokerLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**
     ** start broker-2 ***
     */
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    // mock: return Broker2 as a Least-loaded broker when leader receies request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
    /**
     ** started broker-2 ***
     */
    URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
    // load namespace-bundle by calling Broker2
    Consumer<byte[]> consumer = pulsarClient2.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1").subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    pulsarClient2.close();
    pulsar2.close();
    loadManager1 = null;
    loadManager2 = null;
}

Example 23 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testMultipleBrokerDifferentClusterLookup.

/**
 * Usecase: Redirection due to different cluster 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster:
 * "use2" 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
 * redirects request to Broker2 whch serves "use2" 3. Broker2 receives redirect request and own namespace bundle
 *
 * @throws Exception
 */
@Test
public void testMultipleBrokerDifferentClusterLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**
     ** start broker-2 ***
     */
    final String newCluster = "use2";
    final String property = "my-property2";
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    // Broker2 serves newCluster
    conf2.setClusterName(newCluster);
    conf2.setZookeeperServers("localhost:2181");
    String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort();
    admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
    admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster)));
    admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    URI brokerServiceUrl = new URI(broker2ServiceUrl);
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
    // enable authorization: so, broker can validate cluster and redirect if finds different cluster
    pulsar.getConfiguration().setAuthorizationEnabled(true);
    // restart broker with authorization enabled: it initialize AuthorizationService
    stopBroker();
    startBroker();
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager2));
    /**
     ** started broker-2 ***
     */
    // load namespace-bundle by calling Broker2
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property2/use2/my-ns/my-topic1").subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = pulsarClient2.newProducer().topic("persistent://my-property2/use2/my-ns/my-topic1").create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    // disable authorization
    pulsar.getConfiguration().setAuthorizationEnabled(false);
    pulsarClient2.close();
    pulsar2.close();
    loadManager2 = null;
}

Example 24 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class HttpTopicLookupv2Test method setUp.

@SuppressWarnings("unchecked")
@BeforeMethod
public void setUp() throws Exception {
    pulsar = mock(PulsarService.class);
    ns = mock(NamespaceService.class);
    auth = mock(AuthorizationService.class);
    mockConfigCache = mock(ConfigurationCacheService.class);
    clustersListCache = mock(ZooKeeperChildrenCache.class);
    clustersCache = mock(ZooKeeperDataCache.class);
    policiesCache = mock(ZooKeeperDataCache.class);
    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    clusters = new TreeSet<String>();
    clusters.add("use");
    clusters.add("usc");
    clusters.add("usw");
    ClusterData useData = new ClusterData("http://broker.messaging.use.example.com:8080");
    ClusterData uscData = new ClusterData("http://broker.messaging.usc.example.com:8080");
    ClusterData uswData = new ClusterData("http://broker.messaging.usw.example.com:8080");
    doReturn(config).when(pulsar).getConfiguration();
    doReturn(mockConfigCache).when(pulsar).getConfigurationCache();
    doReturn(clustersListCache).when(mockConfigCache).clustersListCache();
    doReturn(clustersCache).when(mockConfigCache).clustersCache();
    doReturn(policiesCache).when(mockConfigCache).policiesCache();
    doReturn(Optional.of(useData)).when(clustersCache).get(AdminResource.path("clusters", "use"));
    doReturn(Optional.of(uscData)).when(clustersCache).get(AdminResource.path("clusters", "usc"));
    doReturn(Optional.of(uswData)).when(clustersCache).get(AdminResource.path("clusters", "usw"));
    doReturn(CompletableFuture.completedFuture(Optional.of(useData))).when(clustersCache).getAsync(AdminResource.path("clusters", "use"));
    doReturn(CompletableFuture.completedFuture(Optional.of(uscData))).when(clustersCache).getAsync(AdminResource.path("clusters", "usc"));
    doReturn(CompletableFuture.completedFuture(Optional.of(uswData))).when(clustersCache).getAsync(AdminResource.path("clusters", "usw"));
    doReturn(clusters).when(clustersListCache).get();
    doReturn(ns).when(pulsar).getNamespaceService();
    BrokerService brokerService = mock(BrokerService.class);
    doReturn(brokerService).when(pulsar).getBrokerService();
    doReturn(auth).when(brokerService).getAuthorizationService();
    doReturn(new Semaphore(1000)).when(brokerService).getLookupRequestSemaphore();
}

Example 25 with ServiceConfiguration

use of org.apache.pulsar.broker.ServiceConfiguration in project incubator-pulsar by apache.

the class AdvertisedAddressTest method setup.

@BeforeMethod
public void setup() throws Exception {
    bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, 5001);
    bkEnsemble.start();
    ServiceConfiguration config = new ServiceConfiguration();
    config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
    config.setWebServicePort(BROKER_WEBSERVICE_PORT);
    config.setClusterName("usc");
    config.setAdvertisedAddress("localhost");
    config.setBrokerServicePort(BROKER_SERVICE_PORT);
    config.setAdvertisedAddress(advertisedAddress);
    config.setManagedLedgerMaxEntriesPerLedger(5);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsar = new PulsarService(config);
    pulsar.start();
}