Examples with ClusterData org.apache.pulsar.common.policies.data.ClusterData used on opensource projects

Example 6 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ClustersBase method getPeerCluster.

@GET
@Path("/{cluster}/peers")
@ApiOperation(value = "Get the peer-cluster data for the specified cluster.", response = Set.class)
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Cluster doesn't exist") })
public Set<String> getPeerCluster(@PathParam("cluster") String cluster) {
    validateSuperUserAccess();
    try {
        String clusterPath = path("clusters", cluster);
        byte[] content = globalZk().getData(clusterPath, null, null);
        ClusterData clusterData = jsonMapper().readValue(content, ClusterData.class);
        return clusterData.getPeerClusterNames();
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to get cluster {}: Does not exist", clientAppId(), cluster);
        throw new RestException(Status.NOT_FOUND, "Cluster does not exist");
    } catch (Exception e) {
        log.error("[{}] Failed to get cluster {}", clientAppId(), cluster, e);
        throw new RestException(e);
    }
}

Example 7 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ReplicatorTestBase method setup.

void setup() throws Exception {
    log.info("--- Starting ReplicatorTestBase::setup ---");
    int globalZKPort = PortManager.nextFreePort();
    globalZkS = new ZookeeperServerTest(globalZKPort);
    globalZkS.start();
    // Start region 1
    int zkPort1 = PortManager.nextFreePort();
    bkEnsemble1 = new LocalBookkeeperEnsemble(3, zkPort1, PortManager.nextFreePort());
    bkEnsemble1.start();
    int webServicePort1 = PortManager.nextFreePort();
    int webServicePortTls1 = PortManager.nextFreePort();
    // NOTE: we have to instantiate a new copy of System.getProperties() to make sure pulsar1 and pulsar2 have
    // completely
    // independent config objects instead of referring to the same properties object
    config1.setClusterName("r1");
    config1.setAdvertisedAddress("localhost");
    config1.setWebServicePort(webServicePort1);
    config1.setWebServicePortTls(webServicePortTls1);
    config1.setZookeeperServers("127.0.0.1:" + zkPort1);
    config1.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config1.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveTopic());
    config1.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config1.setBrokerServicePort(PortManager.nextFreePort());
    config1.setBrokerServicePortTls(PortManager.nextFreePort());
    config1.setTlsEnabled(true);
    config1.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config1.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config1.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
    config1.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
    config1.setDefaultNumberOfNamespaceBundles(1);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();
    ns1 = pulsar1.getBrokerService();
    url1 = new URL("http://localhost:" + webServicePort1);
    urlTls1 = new URL("https://localhost:" + webServicePortTls1);
    admin1 = new PulsarAdmin(url1, (Authentication) null);
    // Start region 2
    // Start zk & bks
    int zkPort2 = PortManager.nextFreePort();
    bkEnsemble2 = new LocalBookkeeperEnsemble(3, zkPort2, PortManager.nextFreePort());
    bkEnsemble2.start();
    int webServicePort2 = PortManager.nextFreePort();
    int webServicePortTls2 = PortManager.nextFreePort();
    config2.setClusterName("r2");
    config2.setAdvertisedAddress("localhost");
    config2.setWebServicePort(webServicePort2);
    config2.setWebServicePortTls(webServicePortTls2);
    config2.setZookeeperServers("127.0.0.1:" + zkPort2);
    config2.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config2.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveTopic());
    config2.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config2.setBrokerServicePort(PortManager.nextFreePort());
    config2.setBrokerServicePortTls(PortManager.nextFreePort());
    config2.setTlsEnabled(true);
    config2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config2.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
    config2.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
    config2.setDefaultNumberOfNamespaceBundles(1);
    pulsar2 = new PulsarService(config2);
    pulsar2.start();
    ns2 = pulsar2.getBrokerService();
    url2 = new URL("http://localhost:" + webServicePort2);
    urlTls2 = new URL("https://localhost:" + webServicePortTls2);
    admin2 = new PulsarAdmin(url2, (Authentication) null);
    // Start region 3
    // Start zk & bks
    int zkPort3 = PortManager.nextFreePort();
    bkEnsemble3 = new LocalBookkeeperEnsemble(3, zkPort3, PortManager.nextFreePort());
    bkEnsemble3.start();
    int webServicePort3 = PortManager.nextFreePort();
    int webServicePortTls3 = PortManager.nextFreePort();
    config3.setClusterName("r3");
    config3.setAdvertisedAddress("localhost");
    config3.setWebServicePort(webServicePort3);
    config3.setWebServicePortTls(webServicePortTls3);
    config3.setZookeeperServers("127.0.0.1:" + zkPort3);
    config3.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config3.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveTopic());
    config3.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config3.setBrokerServicePort(PortManager.nextFreePort());
    config3.setBrokerServicePortTls(PortManager.nextFreePort());
    config3.setTlsEnabled(true);
    config3.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config3.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config3.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
    config3.setDefaultNumberOfNamespaceBundles(1);
    pulsar3 = new PulsarService(config3);
    pulsar3.start();
    ns3 = pulsar3.getBrokerService();
    url3 = new URL("http://localhost:" + webServicePort3);
    urlTls3 = new URL("https://localhost:" + webServicePortTls3);
    admin3 = new PulsarAdmin(url3, (Authentication) null);
    // Provision the global namespace
    admin1.clusters().createCluster("r1", new ClusterData(url1.toString(), urlTls1.toString(), pulsar1.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("r2", new ClusterData(url2.toString(), urlTls2.toString(), pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("r3", new ClusterData(url3.toString(), urlTls3.toString(), pulsar3.getBrokerServiceUrl(), pulsar3.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("global", new ClusterData("http://global:8080", "https://global:8443"));
    admin1.properties().createProperty("pulsar", new PropertyAdmin(Lists.newArrayList("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r2", "r3")));
    admin1.namespaces().createNamespace("pulsar/global/ns");
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
    admin1.namespaces().createNamespace("pulsar/global/ns1");
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns1", Lists.newArrayList("r1", "r2"));
    assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString());
    assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
    assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString());
    assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
    assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
    assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
    Thread.sleep(100);
    log.info("--- ReplicatorTestBase::setup completed ---");
}

Example 8 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class TlsProducerConsumerBase method internalSetUpForNamespace.

protected void internalSetUpForNamespace() throws Exception {
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
    clientConf.setUseTls(true);
    clientConf.setTlsAllowInsecureConnection(false);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    clientConf.setAuthentication(AuthenticationTls.class.getName(), authParams);
    admin = spy(new PulsarAdmin(brokerUrlTls, clientConf));
    admin.clusters().createCluster(clusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
}

Example 9 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class AuthenticatedProducerConsumerTest method testAnonymousSyncProducerAndConsumer.

@Test(dataProvider = "batch")
public void testAnonymousSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    internalSetup(authTls);
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("anonymousUser"), Sets.newHashSet("use")));
    // make a PulsarAdmin instance as "anonymousUser" for http request
    admin.close();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setOperationTimeout(1, TimeUnit.SECONDS);
    admin = spy(new PulsarAdmin(brokerUrl, clientConf));
    admin.namespaces().createNamespace("my-property/use/my-ns");
    admin.persistentTopics().grantPermission("persistent://my-property/use/my-ns/my-topic", "anonymousUser", EnumSet.allOf(AuthAction.class));
    // setup the client
    pulsarClient.close();
    pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:" + BROKER_PORT).operationTimeout(1, TimeUnit.SECONDS).build();
    // unauthorized topic test
    Exception pulsarClientException = null;
    try {
        pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/other-topic").subscriptionName("my-subscriber-name").subscribe();
    } catch (Exception e) {
        pulsarClientException = e;
    }
    Assert.assertTrue(pulsarClientException instanceof PulsarClientException);
    testSyncProducerAndConsumer(batchMessageDelayMs);
    log.info("-- Exiting {} test --", methodName);
}

Example 10 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testMultipleBrokerDifferentClusterLookup.

/**
 * Usecase: Redirection due to different cluster 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster:
 * "use2" 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
 * redirects request to Broker2 whch serves "use2" 3. Broker2 receives redirect request and own namespace bundle
 *
 * @throws Exception
 */
@Test
public void testMultipleBrokerDifferentClusterLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    /**
     ** start broker-2 ***
     */
    final String newCluster = "use2";
    final String property = "my-property2";
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    // Broker2 serves newCluster
    conf2.setClusterName(newCluster);
    conf2.setZookeeperServers("localhost:2181");
    String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort();
    admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
    admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster)));
    admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    URI brokerServiceUrl = new URI(broker2ServiceUrl);
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
    // enable authorization: so, broker can validate cluster and redirect if finds different cluster
    pulsar.getConfiguration().setAuthorizationEnabled(true);
    // restart broker with authorization enabled: it initialize AuthorizationService
    stopBroker();
    startBroker();
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager2));
    /**
     ** started broker-2 ***
     */
    // load namespace-bundle by calling Broker2
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property2/use2/my-ns/my-topic1").subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = pulsarClient2.newProducer().topic("persistent://my-property2/use2/my-ns/my-topic1").create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();
    // disable authorization
    pulsar.getConfiguration().setAuthorizationEnabled(false);
    pulsarClient2.close();
    pulsar2.close();
    loadManager2 = null;
}