Examples with ClusterData com.yahoo.pulsar.common.policies.data.ClusterData used on opensource projects

Example 21 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class BrokerServiceLookupTest method testMultipleBrokerDifferentClusterLookup.

/**
     * Usecase: Redirection due to different cluster 
     * 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster: "use2" 
     * 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
     * redirects request to Broker2 whch serves "use2"
     * 3. Broker2 receives redirect request and own namespace bundle
     * 
     * @throws Exception
     */
@Test
public void testMultipleBrokerDifferentClusterLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**** start broker-2 ****/
    final String newCluster = "use2";
    final String property = "my-property2";
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    // Broker2 serves newCluster
    conf2.setClusterName(newCluster);
    String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort();
    admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
    admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster)));
    admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    URI brokerServiceUrl = new URI(broker2ServiceUrl);
    PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
    // enable authorization: so, broker can validate cluster and redirect if finds different cluster
    pulsar.getConfiguration().setAuthorizationEnabled(true);
    // restart broker with authorization enabled: it initialize AuthorizationManager
    stopBroker();
    startBroker();
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(resourceUnit).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), loadManager2);
    /**** started broker-2 ****/
    // load namespace-bundle by calling Broker2
    Consumer consumer = pulsarClient.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name", new ConsumerConfiguration());
    Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration());
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    // disable authorization 
    pulsar.getConfiguration().setAuthorizationEnabled(false);
    pulsarClient2.close();
    pulsar2.close();
    loadManager2 = null;
}

Example 22 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class WebServiceTest method setupEnv.

private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowUnversionedClients, boolean enableTls, boolean enableAuth, boolean allowInsecure) throws Exception {
    Set<String> providers = new HashSet<>();
    providers.add("com.yahoo.pulsar.broker.authentication.AuthenticationProviderTls");
    Set<String> roles = new HashSet<>();
    roles.add("client");
    ServiceConfiguration config = new ServiceConfiguration();
    config.setWebServicePort(BROKER_WEBSERVICE_PORT);
    config.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS);
    config.setClientLibraryVersionCheckEnabled(enableFilter);
    config.setAuthenticationEnabled(enableAuth);
    config.setAuthenticationProviders(providers);
    config.setAuthorizationEnabled(false);
    config.setClientLibraryVersionCheckAllowUnversioned(allowUnversionedClients);
    config.setSuperUserRoles(roles);
    config.setTlsEnabled(enableTls);
    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsAllowInsecureConnection(allowInsecure);
    config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
    config.setClusterName("local");
    // TLS certificate expects localhost
    config.setAdvertisedAddress("localhost");
    pulsar = spy(new PulsarService(config));
    doReturn(new MockedZooKeeperClientFactoryImpl()).when(pulsar).getZooKeeperClientFactory();
    pulsar.start();
    try {
        pulsar.getZkClient().delete("/minApiVersion", -1);
    } catch (Exception ex) {
    }
    pulsar.getZkClient().create("/minApiVersion", minApiVersion.getBytes(), null, CreateMode.PERSISTENT);
    String serviceUrl = BROKER_URL_BASE;
    ClientConfiguration clientConfig = new ClientConfiguration();
    if (enableTls && enableAuth) {
        serviceUrl = BROKER_URL_BASE_TLS;
        Map<String, String> authParams = new HashMap<>();
        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
        Authentication auth = new AuthenticationTls();
        auth.configure(authParams);
        clientConfig.setAuthentication(auth);
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
    }
    PulsarAdmin pulsarAdmin = new PulsarAdmin(new URL(serviceUrl), clientConfig);
    try {
        pulsarAdmin.clusters().createCluster(config.getClusterName(), new ClusterData(pulsar.getWebServiceAddress()));
    } catch (ConflictException ce) {
    // This is OK.
    } finally {
        pulsarAdmin.close();
    }
}

Example 23 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class AuthenticatedProducerConsumerTest method testAuthemticationFilterNegative.

/**
     * Verifies: on 500 server error, broker invalidates session and client receives 500 correctly.
     * 
     * @throws Exception
     */
@Test
public void testAuthemticationFilterNegative() throws Exception {
    log.info("-- Starting {} test --", methodName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    internalSetup(authTls);
    final String cluster = "use";
    final ClusterData clusterData = new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS);
    // this will cause NPE and it should throw 500
    doReturn(null).when(pulsar).getGlobalZkCache();
    try {
        admin.clusters().createCluster(cluster, clusterData);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
    log.info("-- Exiting {} test --", methodName);
}

Example 24 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class PulsarService method start.

/**
     * Start the pulsar service instance.
     */
public void start() throws PulsarServerException {
    mutex.lock();
    try {
        if (state != State.Init) {
            throw new PulsarServerException("Cannot start the service once it was stopped");
        }
        // Now we are ready to start services
        localZooKeeperConnectionProvider = new LocalZooKeeperConnectionService(getZooKeeperClientFactory(), config.getZookeeperServers(), config.getZooKeeperSessionTimeoutMillis());
        localZooKeeperConnectionProvider.start(shutdownService);
        // Initialize and start service to access configuration repository.
        this.startZkCacheService();
        managedLedgerClientFactory = new ManagedLedgerClientFactory(config, getZkClient(), getBookKeeperClientFactory());
        this.brokerService = new BrokerService(this);
        // Start load management service (even if load balancing is disabled)
        this.loadManager = new SimpleLoadManagerImpl(this);
        this.startLoadManagementService();
        // needs load management service
        this.startNamespaceService();
        LOG.info("Starting Pulsar Broker service");
        brokerService.start();
        this.webService = new WebService(this);
        this.webService.addRestResources("/", "com.yahoo.pulsar.broker.web", false);
        this.webService.addRestResources("/admin", "com.yahoo.pulsar.broker.admin", true);
        this.webService.addRestResources("/lookup", "com.yahoo.pulsar.broker.lookup", true);
        if (config.isWebSocketServiceEnabled()) {
            // Use local broker address to avoid different IP address when using a VIP for service discovery
            this.webSocketService = new WebSocketService(new ClusterData(webServiceAddress, webServiceAddressTls), config);
            this.webSocketService.start();
            this.webService.addServlet(WebSocketProducerServlet.SERVLET_PATH, new ServletHolder(new WebSocketProducerServlet(webSocketService)), true);
            this.webService.addServlet(WebSocketConsumerServlet.SERVLET_PATH, new ServletHolder(new WebSocketConsumerServlet(webSocketService)), true);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Attempting to add static directory");
        }
        this.webService.addStaticResources("/static", "/static");
        // Register heartbeat and bootstrap namespaces.
        this.nsservice.registerBootstrapNamespaces();
        // Start the leader election service
        this.leaderElectionService = new LeaderElectionService(this, new LeaderListener() {

            @Override
            public synchronized void brokerIsTheLeaderNow() {
                if (getConfiguration().isLoadBalancerEnabled()) {
                    long loadSheddingInterval = TimeUnit.MINUTES.toMillis(getConfiguration().getLoadBalancerSheddingIntervalMinutes());
                    long resourceQuotaUpdateInterval = TimeUnit.MINUTES.toMillis(getConfiguration().getLoadBalancerResourceQuotaUpdateIntervalMinutes());
                    loadSheddingTask = loadManagerExecutor.scheduleAtFixedRate(new LoadSheddingTask(loadManager), loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS);
                    loadResourceQuotaTask = loadManagerExecutor.scheduleAtFixedRate(new LoadResourceQuotaUpdaterTask(loadManager), resourceQuotaUpdateInterval, resourceQuotaUpdateInterval, TimeUnit.MILLISECONDS);
                }
            }

            @Override
            public synchronized void brokerIsAFollowerNow() {
                if (loadSheddingTask != null) {
                    loadSheddingTask.cancel(false);
                    loadSheddingTask = null;
                }
                if (loadResourceQuotaTask != null) {
                    loadResourceQuotaTask.cancel(false);
                    loadResourceQuotaTask = null;
                }
            }
        });
        leaderElectionService.start();
        webService.start();
        this.metricsGenerator = new MetricsGenerator(this);
        state = State.Started;
        acquireSLANamespace();
        LOG.info("messaging service is ready, bootstrap service on port={}, broker url={}, cluster={}, configs={}", config.getWebServicePort(), brokerServiceUrl, config.getClusterName(), config);
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new PulsarServerException(e);
    } finally {
        mutex.unlock();
    }
}

Example 25 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class NamespacesTest method setup.

@Override
@BeforeMethod
public void setup() throws Exception {
    super.internalSetup();
    namespaces = spy(new Namespaces());
    namespaces.setServletContext(new MockServletContext());
    namespaces.setPulsar(pulsar);
    doReturn(mockZookKeeper).when(namespaces).globalZk();
    doReturn(mockZookKeeper).when(namespaces).localZk();
    doReturn(pulsar.getConfigurationCache().propertiesCache()).when(namespaces).propertiesCache();
    doReturn(pulsar.getConfigurationCache().policiesCache()).when(namespaces).policiesCache();
    doReturn(false).when(namespaces).isRequestHttps();
    doReturn("test").when(namespaces).clientAppId();
    doReturn(Sets.newTreeSet(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters();
    doNothing().when(namespaces).validateAdminAccessOnProperty("my-property");
    doNothing().when(namespaces).validateAdminAccessOnProperty("other-property");
    doNothing().when(namespaces).validateAdminAccessOnProperty("new-property");
    admin.clusters().createCluster("use", new ClusterData("http://broker-use.com:" + BROKER_WEBSERVICE_PORT));
    admin.clusters().createCluster("usw", new ClusterData("http://broker-usw.com:" + BROKER_WEBSERVICE_PORT));
    admin.clusters().createCluster("usc", new ClusterData("http://broker-usc.com:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty(this.testProperty, new PropertyAdmin(Lists.newArrayList("role1", "role2"), Sets.newHashSet("use", "usc", "usw")));
    createTestNamespaces(this.testProperty, this.testLocalNamespaces, new BundlesData());
    createGlobalTestNamespaces(this.testProperty, this.testGlobalNamespaces.get(0).getLocalName(), new BundlesData());
    nsSvc = pulsar.getNamespaceService();
}