Examples with PulsarService com.yahoo.pulsar.broker.PulsarService used on opensource projects

Example 6 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method setup.

@BeforeMethod
void setup() throws Exception {
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
    bkEnsemble.start();
    // Start broker 1
    ServiceConfiguration config1 = spy(new ServiceConfiguration());
    config1.setClusterName("use");
    config1.setWebServicePort(PRIMARY_BROKER_WEBSERVICE_PORT);
    config1.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
    config1.setBrokerServicePort(PRIMARY_BROKER_PORT);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();
    url1 = new URL("http://127.0.0.1" + ":" + PRIMARY_BROKER_WEBSERVICE_PORT);
    admin1 = new PulsarAdmin(url1, (Authentication) null);
    brokerStatsClient1 = admin1.brokerStats();
    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setClusterName("use");
    config2.setWebServicePort(SECONDARY_BROKER_WEBSERVICE_PORT);
    config2.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
    config2.setBrokerServicePort(SECONDARY_BROKER_PORT);
    pulsar2 = new PulsarService(config2);
    pulsar2.start();
    url2 = new URL("http://127.0.0.1" + ":" + SECONDARY_BROKER_WEBSERVICE_PORT);
    admin2 = new PulsarAdmin(url2, (Authentication) null);
    brokerStatsClient2 = admin2.brokerStats();
    createNamespacePolicies(pulsar1);
    Thread.sleep(100);
}

Example 7 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class ServerCnxTest method setup.

@BeforeMethod
public void setup() throws Exception {
    svcConfig = spy(new ServiceConfiguration());
    pulsar = spy(new PulsarService(svcConfig));
    svcConfig.setKeepAliveIntervalSeconds(inSec(1, TimeUnit.SECONDS));
    svcConfig.setBacklogQuotaCheckEnabled(false);
    doReturn(svcConfig).when(pulsar).getConfiguration();
    doReturn("use").when(svcConfig).getClusterName();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = mock(ZooKeeper.class);
    doReturn(mockZk).when(pulsar).getZkClient();
    configCacheService = mock(ConfigurationCacheService.class);
    @SuppressWarnings("unchecked") ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    doReturn(Optional.empty()).when(zkDataCache).get(anyObject());
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    doReturn(configCacheService).when(pulsar).getConfigurationCache();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    namespaceService = mock(NamespaceService.class);
    doReturn(namespaceService).when(pulsar).getNamespaceService();
    doReturn(true).when(namespaceService).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(namespaceService).isServiceUnitActive(any(DestinationName.class));
    setupMLAsyncCallbackMocks();
    clientChannelHelper = new ClientChannelHelper();
}

Example 8 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class ReplicatorTestBase method setup.

void setup() throws Exception {
    log.info("--- Starting ReplicatorTestBase::setup ---");
    int globalZKPort = PortManager.nextFreePort();
    globalZkS = new ZookeeperServerTest(globalZKPort);
    globalZkS.start();
    // Start region 1
    int zkPort1 = PortManager.nextFreePort();
    bkEnsemble1 = new LocalBookkeeperEnsemble(3, zkPort1, PortManager.nextFreePort());
    bkEnsemble1.start();
    int webServicePort1 = PortManager.nextFreePort();
    // NOTE: we have to instantiate a new copy of System.getProperties() to make sure pulsar1 and pulsar2 have
    // completely
    // independent config objects instead of referring to the same properties object
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setClusterName("r1");
    config1.setWebServicePort(webServicePort1);
    config1.setZookeeperServers("127.0.0.1:" + zkPort1);
    config1.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config1.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
    config1.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config1.setBrokerServicePort(PortManager.nextFreePort());
    config1.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();
    ns1 = pulsar1.getBrokerService();
    url1 = new URL("http://127.0.0.1:" + webServicePort1);
    admin1 = new PulsarAdmin(url1, (Authentication) null);
    // Start region 2
    // Start zk & bks
    int zkPort2 = PortManager.nextFreePort();
    bkEnsemble2 = new LocalBookkeeperEnsemble(3, zkPort2, PortManager.nextFreePort());
    bkEnsemble2.start();
    int webServicePort2 = PortManager.nextFreePort();
    config2 = new ServiceConfiguration();
    config2.setClusterName("r2");
    config2.setWebServicePort(webServicePort2);
    config2.setZookeeperServers("127.0.0.1:" + zkPort2);
    config2.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config2.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
    config2.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config2.setBrokerServicePort(PortManager.nextFreePort());
    config2.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
    pulsar2 = new PulsarService(config2);
    pulsar2.start();
    ns2 = pulsar2.getBrokerService();
    url2 = new URL("http://127.0.0.1:" + webServicePort2);
    admin2 = new PulsarAdmin(url2, (Authentication) null);
    // Start region 3
    // Start zk & bks
    int zkPort3 = PortManager.nextFreePort();
    bkEnsemble3 = new LocalBookkeeperEnsemble(3, zkPort3, PortManager.nextFreePort());
    bkEnsemble3.start();
    int webServicePort3 = PortManager.nextFreePort();
    config3 = new ServiceConfiguration();
    config3.setClusterName("r3");
    config3.setWebServicePort(webServicePort3);
    config3.setZookeeperServers("127.0.0.1:" + zkPort3);
    config3.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config3.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
    config3.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config3.setBrokerServicePort(PortManager.nextFreePort());
    pulsar3 = new PulsarService(config3);
    pulsar3.start();
    ns3 = pulsar3.getBrokerService();
    url3 = new URL("http://127.0.0.1:" + webServicePort3);
    admin3 = new PulsarAdmin(url3, (Authentication) null);
    // Provision the global namespace
    admin1.clusters().createCluster("r1", new ClusterData(url1.toString(), null, pulsar1.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("r2", new ClusterData(url2.toString(), null, pulsar2.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("r3", new ClusterData(url3.toString(), null, pulsar3.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("global", new ClusterData("http://global:8080"));
    admin1.properties().createProperty("pulsar", new PropertyAdmin(Lists.newArrayList("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r2", "r3")));
    admin1.namespaces().createNamespace("pulsar/global/ns");
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
    admin1.namespaces().createNamespace("pulsar/global/ns1");
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns1", Lists.newArrayList("r1", "r2"));
    assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString());
    assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
    assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString());
    assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
    assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
    assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
    /*
         * assertEquals(admin2.clusters().getCluster("global").getServiceUrl(), "http://global:8080");
         * assertEquals(admin2.properties().getPropertyAdmin("pulsar").getAdminRoles(), Lists.newArrayList("appid1",
         * "appid2")); assertEquals(admin2.namespaces().getPolicies("pulsar/global/ns").replication_clusters,
         * Lists.newArrayList("r1", "r2", "r3"));
         *
         * admin1.namespaces().createNamespace("pulsar/global/ns2");
         * admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns2", Lists.newArrayList("r1", "r2",
         * "r3"));
         */
    Thread.sleep(100);
    log.info("--- ReplicatorTestBase::setup completed ---");
}

Example 9 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class BrokerServiceLookupTest method testMultipleBrokerLookup.

/**
     * UsecaseL Multiple Broker => Lookup Redirection test
     * 
     * 1. Broker1 is a leader
     * 2. Lookup request reaches to Broker2 which redirects to leader (Broker1) with authoritative = false
     * 3. Leader (Broker1) finds out least loaded broker as Broker2 and redirects request to Broker2 with authoritative = true
     * 4. Broker2 receives final request to own a bundle with authoritative = true and client connects to Broker2
     * 
     * @throws Exception
     */
@Test
public void testMultipleBrokerLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), loadManager2);
    // mock: return Broker2 as a Least-loaded broker when leader receies request [3] 
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(resourceUnit).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), loadManager1);
    /**** started broker-2 ****/
    URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
    // load namespace-bundle by calling Broker2
    Consumer consumer = pulsarClient2.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", new ConsumerConfiguration());
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", new ProducerConfiguration());
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    pulsarClient2.close();
    pulsar2.close();
    loadManager1 = null;
    loadManager2 = null;
}

Example 10 with PulsarService

use of com.yahoo.pulsar.broker.PulsarService in project pulsar by yahoo.

the class PulsarBrokerStarter method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new IllegalArgumentException("Need to specify a configuration file");
    }
    Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
        log.error("Uncaught exception in thread {}: {}", thread.getName(), exception.getMessage(), exception);
    });
    String configFile = args[0];
    ServiceConfiguration config = loadConfig(configFile);
    @SuppressWarnings("resource") final PulsarService service = new PulsarService(config);
    Runtime.getRuntime().addShutdownHook(service.getShutdownService());
    try {
        service.start();
        log.info("PulsarService started");
    } catch (PulsarServerException e) {
        log.error("Failed to start pulsar service.", e);
        Runtime.getRuntime().halt(1);
    }
    service.waitUntilClosed();
}