Examples with ServiceConfiguration com.yahoo.pulsar.broker.ServiceConfiguration used on opensource projects

Example 31 with ServiceConfiguration

use of com.yahoo.pulsar.broker.ServiceConfiguration in project pulsar by yahoo.

the class BrokerServiceLookupTest method testMultipleBrokerDifferentClusterLookup.

/**
     * Usecase: Redirection due to different cluster 
     * 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster: "use2" 
     * 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
     * redirects request to Broker2 whch serves "use2"
     * 3. Broker2 receives redirect request and own namespace bundle
     * 
     * @throws Exception
     */
@Test
public void testMultipleBrokerDifferentClusterLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**** start broker-2 ****/
    final String newCluster = "use2";
    final String property = "my-property2";
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    // Broker2 serves newCluster
    conf2.setClusterName(newCluster);
    String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort();
    admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
    admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster)));
    admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    URI brokerServiceUrl = new URI(broker2ServiceUrl);
    PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
    // enable authorization: so, broker can validate cluster and redirect if finds different cluster
    pulsar.getConfiguration().setAuthorizationEnabled(true);
    // restart broker with authorization enabled: it initialize AuthorizationManager
    stopBroker();
    startBroker();
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(resourceUnit).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), loadManager2);
    /**** started broker-2 ****/
    // load namespace-bundle by calling Broker2
    Consumer consumer = pulsarClient.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name", new ConsumerConfiguration());
    Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration());
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    // disable authorization 
    pulsar.getConfiguration().setAuthorizationEnabled(false);
    pulsarClient2.close();
    pulsar2.close();
    loadManager2 = null;
}

Example 32 with ServiceConfiguration

use of com.yahoo.pulsar.broker.ServiceConfiguration in project pulsar by yahoo.

the class BrokerServiceLookupTest method testPartitionTopicLookup.

/**
     * Create #PartitionedTopic and let it served by multiple brokers which requries 
     * a. tcp partitioned-metadata-lookup
     * b. multiple topic-lookup 
     * c. partitioned producer-consumer
     * 
     * @throws Exception
     */
@Test
public void testPartitionTopicLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int numPartitions = 8;
    DestinationName dn = DestinationName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    admin.persistentTopics().createPartitionedTopic(dn.toString(), numPartitions);
    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(pulsar.getConfiguration().getClusterName());
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager1).isCentralized();
    loadManagerField.set(pulsar.getNamespaceService(), loadManager1);
    // mock: redirect request to leader
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), loadManager2);
    /****  broker-2 started ****/
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(dn.toString(), producerConf);
    Consumer consumer = pulsarClient.subscribe(dn.toString(), "my-partitioned-subscriber", conf);
    for (int i = 0; i < 20; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 20; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        Assert.assertNotNull(msg, "Message should not be null");
        consumer.acknowledge(msg);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        Assert.assertTrue(messageSet.add(receivedMessage), "Message " + receivedMessage + " already received");
    }
    producer.close();
    consumer.unsubscribe();
    consumer.close();
    admin.persistentTopics().deletePartitionedTopic(dn.toString());
    pulsar2.close();
    loadManager2 = null;
    log.info("-- Exiting {} test --", methodName);
}

Example 33 with ServiceConfiguration

use of com.yahoo.pulsar.broker.ServiceConfiguration in project pulsar by yahoo.

the class WebServiceTest method setupEnv.

private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowUnversionedClients, boolean enableTls, boolean enableAuth, boolean allowInsecure) throws Exception {
    Set<String> providers = new HashSet<>();
    providers.add("com.yahoo.pulsar.broker.authentication.AuthenticationProviderTls");
    Set<String> roles = new HashSet<>();
    roles.add("client");
    ServiceConfiguration config = new ServiceConfiguration();
    config.setWebServicePort(BROKER_WEBSERVICE_PORT);
    config.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS);
    config.setClientLibraryVersionCheckEnabled(enableFilter);
    config.setAuthenticationEnabled(enableAuth);
    config.setAuthenticationProviders(providers);
    config.setAuthorizationEnabled(false);
    config.setClientLibraryVersionCheckAllowUnversioned(allowUnversionedClients);
    config.setSuperUserRoles(roles);
    config.setTlsEnabled(enableTls);
    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsAllowInsecureConnection(allowInsecure);
    config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
    config.setClusterName("local");
    // TLS certificate expects localhost
    config.setAdvertisedAddress("localhost");
    pulsar = spy(new PulsarService(config));
    doReturn(new MockedZooKeeperClientFactoryImpl()).when(pulsar).getZooKeeperClientFactory();
    pulsar.start();
    try {
        pulsar.getZkClient().delete("/minApiVersion", -1);
    } catch (Exception ex) {
    }
    pulsar.getZkClient().create("/minApiVersion", minApiVersion.getBytes(), null, CreateMode.PERSISTENT);
    String serviceUrl = BROKER_URL_BASE;
    ClientConfiguration clientConfig = new ClientConfiguration();
    if (enableTls && enableAuth) {
        serviceUrl = BROKER_URL_BASE_TLS;
        Map<String, String> authParams = new HashMap<>();
        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
        Authentication auth = new AuthenticationTls();
        auth.configure(authParams);
        clientConfig.setAuthentication(auth);
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
    }
    PulsarAdmin pulsarAdmin = new PulsarAdmin(new URL(serviceUrl), clientConfig);
    try {
        pulsarAdmin.clusters().createCluster(config.getClusterName(), new ClusterData(pulsar.getWebServiceAddress()));
    } catch (ConflictException ce) {
    // This is OK.
    } finally {
        pulsarAdmin.close();
    }
}

Example 34 with ServiceConfiguration

use of com.yahoo.pulsar.broker.ServiceConfiguration in project pulsar by yahoo.

the class SimpleLoadManagerImpl method start.

@Override
public void start() throws PulsarServerException {
    try {
        // Register the brokers in zk list
        ServiceConfiguration conf = pulsar.getConfiguration();
        if (pulsar.getZkClient().exists(LOADBALANCE_BROKERS_ROOT, false) == null) {
            try {
                ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), LOADBALANCE_BROKERS_ROOT, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            } catch (KeeperException.NodeExistsException e) {
            // ignore the exception, node might be present already
            }
        }
        String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" + conf.getWebServicePort();
        brokerZnodePath = LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress;
        LoadReport loadReport = null;
        try {
            loadReport = generateLoadReport();
            this.lastResourceUsageTimestamp = loadReport.getTimestamp();
        } catch (Exception e) {
            log.warn("Unable to get load report to write it on zookeeper [{}]", e);
        }
        String loadReportJson = "";
        if (loadReport != null) {
            loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(loadReport);
        }
        try {
            ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), brokerZnodePath, loadReportJson.getBytes(Charsets.UTF_8), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
        } catch (Exception e) {
            // Catching excption here to print the right error message
            log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
            throw e;
        }
        // first time, populate the broker ranking
        updateRanking();
        log.info("Created broker ephemeral node on {}", brokerZnodePath);
        // load default resource quota
        this.realtimeAvgResourceQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
        this.lastResourceQuotaUpdateTimestamp = System.currentTimeMillis();
        this.realtimeCpuLoadFactor = getDynamicConfigurationDouble(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_CPU_ZPATH, SETTING_NAME_LOAD_FACTOR_CPU, this.realtimeCpuLoadFactor);
        this.realtimeMemoryLoadFactor = getDynamicConfigurationDouble(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_MEM_ZPATH, SETTING_NAME_LOAD_FACTOR_MEM, this.realtimeMemoryLoadFactor);
    } catch (Exception e) {
        log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
        throw new PulsarServerException(e);
    }
}

Example 35 with ServiceConfiguration

use of com.yahoo.pulsar.broker.ServiceConfiguration in project pulsar by yahoo.

the class AuthenticationServiceTest method testAuthentication.

@Test
public void testAuthentication() throws Exception {
    ServiceConfiguration config = new ServiceConfiguration();
    Set<String> providersClassNames = Sets.newHashSet(MockAuthenticationProvider.class.getName());
    config.setAuthenticationProviders(providersClassNames);
    config.setAuthenticationEnabled(true);
    AuthenticationService service = new AuthenticationService(config);
    String result = service.authenticate(null, "auth");
    assertEquals(result, s_authentication_success);
}