Examples with ClusterData org.apache.pulsar.common.policies.data.ClusterData used on opensource projects

Example 31 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class PulsarService method start.

/**
 * Start the pulsar service instance.
 */
public void start() throws PulsarServerException {
    mutex.lock();
    try {
        if (state != State.Init) {
            throw new PulsarServerException("Cannot start the service once it was stopped");
        }
        // Now we are ready to start services
        localZooKeeperConnectionProvider = new LocalZooKeeperConnectionService(getZooKeeperClientFactory(), config.getZookeeperServers(), config.getZooKeeperSessionTimeoutMillis());
        localZooKeeperConnectionProvider.start(shutdownService);
        // Initialize and start service to access configuration repository.
        this.startZkCacheService();
        this.bkClientFactory = getBookKeeperClientFactory();
        managedLedgerClientFactory = new ManagedLedgerClientFactory(config, getZkClient(), bkClientFactory);
        this.brokerService = new BrokerService(this);
        // Start load management service (even if load balancing is disabled)
        this.loadManager.set(LoadManager.create(this));
        this.startLoadManagementService();
        // needs load management service
        this.startNamespaceService();
        LOG.info("Starting Pulsar Broker service; version: '{}'", (brokerVersion != null ? brokerVersion : "unknown"));
        brokerService.start();
        this.webService = new WebService(this);
        Map<String, Object> attributeMap = Maps.newHashMap();
        attributeMap.put(WebService.ATTRIBUTE_PULSAR_NAME, this);
        Map<String, Object> vipAttributeMap = Maps.newHashMap();
        vipAttributeMap.put(VipStatus.ATTRIBUTE_STATUS_FILE_PATH, this.config.getStatusFilePath());
        this.webService.addRestResources("/", VipStatus.class.getPackage().getName(), false, vipAttributeMap);
        this.webService.addRestResources("/", "org.apache.pulsar.broker.web", false, attributeMap);
        this.webService.addRestResources("/admin", "org.apache.pulsar.broker.admin.v1", true, attributeMap);
        this.webService.addRestResources("/admin/v2", "org.apache.pulsar.broker.admin.v2", true, attributeMap);
        this.webService.addRestResources("/lookup", "org.apache.pulsar.broker.lookup", true, attributeMap);
        this.webService.addServlet("/metrics", new ServletHolder(new PrometheusMetricsServlet(this, config.exposeTopicLevelMetricsInPrometheus())), false, attributeMap);
        if (config.isWebSocketServiceEnabled()) {
            // Use local broker address to avoid different IP address when using a VIP for service discovery
            this.webSocketService = new WebSocketService(new ClusterData(webServiceAddress, webServiceAddressTls, brokerServiceUrl, brokerServiceUrlTls), config);
            this.webSocketService.start();
            final WebSocketServlet producerWebSocketServlet = new WebSocketProducerServlet(webSocketService);
            this.webService.addServlet(WebSocketProducerServlet.SERVLET_PATH, new ServletHolder(producerWebSocketServlet), true, attributeMap);
            this.webService.addServlet(WebSocketProducerServlet.SERVLET_PATH_V2, new ServletHolder(producerWebSocketServlet), true, attributeMap);
            final WebSocketServlet consumerWebSocketServlet = new WebSocketConsumerServlet(webSocketService);
            this.webService.addServlet(WebSocketConsumerServlet.SERVLET_PATH, new ServletHolder(consumerWebSocketServlet), true, attributeMap);
            this.webService.addServlet(WebSocketConsumerServlet.SERVLET_PATH_V2, new ServletHolder(consumerWebSocketServlet), true, attributeMap);
            final WebSocketServlet readerWebSocketServlet = new WebSocketReaderServlet(webSocketService);
            this.webService.addServlet(WebSocketReaderServlet.SERVLET_PATH, new ServletHolder(readerWebSocketServlet), true, attributeMap);
            this.webService.addServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new ServletHolder(readerWebSocketServlet), true, attributeMap);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Attempting to add static directory");
        }
        this.webService.addStaticResources("/static", "/static");
        // Register heartbeat and bootstrap namespaces.
        this.nsservice.registerBootstrapNamespaces();
        // Start the leader election service
        this.leaderElectionService = new LeaderElectionService(this, new LeaderListener() {

            @Override
            public synchronized void brokerIsTheLeaderNow() {
                if (getConfiguration().isLoadBalancerEnabled()) {
                    long loadSheddingInterval = TimeUnit.MINUTES.toMillis(getConfiguration().getLoadBalancerSheddingIntervalMinutes());
                    long resourceQuotaUpdateInterval = TimeUnit.MINUTES.toMillis(getConfiguration().getLoadBalancerResourceQuotaUpdateIntervalMinutes());
                    loadSheddingTask = loadManagerExecutor.scheduleAtFixedRate(new LoadSheddingTask(loadManager), loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS);
                    loadResourceQuotaTask = loadManagerExecutor.scheduleAtFixedRate(new LoadResourceQuotaUpdaterTask(loadManager), resourceQuotaUpdateInterval, resourceQuotaUpdateInterval, TimeUnit.MILLISECONDS);
                }
            }

            @Override
            public synchronized void brokerIsAFollowerNow() {
                if (loadSheddingTask != null) {
                    loadSheddingTask.cancel(false);
                    loadSheddingTask = null;
                }
                if (loadResourceQuotaTask != null) {
                    loadResourceQuotaTask.cancel(false);
                    loadResourceQuotaTask = null;
                }
            }
        });
        leaderElectionService.start();
        webService.start();
        this.metricsGenerator = new MetricsGenerator(this);
        schemaRegistryService = SchemaRegistryService.create(this);
        state = State.Started;
        acquireSLANamespace();
        LOG.info("messaging service is ready, bootstrap service on port={}, broker url={}, cluster={}, configs={}", config.getWebServicePort(), brokerServiceUrl, config.getClusterName(), ReflectionToStringBuilder.toString(config));
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new PulsarServerException(e);
    } finally {
        mutex.unlock();
    }
}

Example 32 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class PulsarClusterMetadataSetup method main.

public static void main(String[] args) throws Exception {
    Arguments arguments = new Arguments();
    JCommander jcommander = new JCommander();
    try {
        jcommander.addObject(arguments);
        jcommander.parse(args);
        if (arguments.help) {
            jcommander.usage();
            return;
        }
    } catch (Exception e) {
        jcommander.usage();
        throw e;
    }
    log.info("Setting up cluster {} with zk={} global-zk={}", arguments.cluster, arguments.zookeeper, arguments.globalZookeeper);
    // Format BookKeeper metadata
    ServerConfiguration bkConf = new ServerConfiguration();
    bkConf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class);
    bkConf.setZkServers(arguments.zookeeper);
    if (!BookKeeperAdmin.format(bkConf, false, /* interactive */
    false)) {
        throw new IOException("Failed to initialize BookKeeper metadata");
    }
    ZooKeeperClientFactory zkfactory = new ZookeeperClientFactoryImpl();
    ZooKeeper localZk = zkfactory.create(arguments.zookeeper, SessionType.ReadWrite, 30000).get();
    ZooKeeper globalZk = zkfactory.create(arguments.globalZookeeper, SessionType.ReadWrite, 30000).get();
    localZk.create("/managed-ledgers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    localZk.create("/namespace", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    try {
        ZkUtils.createFullPathOptimistic(globalZk, POLICIES_ROOT, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (NodeExistsException e) {
    // Ignore
    }
    try {
        ZkUtils.createFullPathOptimistic(globalZk, "/admin/clusters", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (NodeExistsException e) {
    // Ignore
    }
    ClusterData clusterData = new ClusterData(arguments.clusterWebServiceUrl, arguments.clusterWebServiceUrlTls, arguments.clusterBrokerServiceUrl, arguments.clusterBrokerServiceUrlTls);
    byte[] clusterDataJson = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(clusterData);
    globalZk.create("/admin/clusters/" + arguments.cluster, clusterDataJson, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Create marker for "global" cluster
    ClusterData globalClusterData = new ClusterData(null, null);
    byte[] globalClusterDataJson = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(globalClusterData);
    try {
        globalZk.create("/admin/clusters/global", globalClusterDataJson, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (NodeExistsException e) {
    // Ignore
    }
    log.info("Cluster metadata for '{}' setup correctly", arguments.cluster);
}

Example 33 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ProxyAuthorizationTest method test.

@Test
public void test() throws Exception {
    AuthorizationService auth = service.getAuthorizationService();
    assertEquals(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null), false);
    admin.clusters().createCluster(configClusterName, new ClusterData());
    admin.properties().createProperty("p1", new PropertyAdmin(Lists.newArrayList("role1"), Sets.newHashSet("c1")));
    waitForChange();
    admin.namespaces().createNamespace("p1/c1/ns1");
    waitForChange();
    assertEquals(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null), false);
    admin.namespaces().grantPermissionOnNamespace("p1/c1/ns1", "my-role", EnumSet.of(AuthAction.produce));
    waitForChange();
    assertEquals(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null), true);
    assertEquals(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null), true);
    admin.persistentTopics().grantPermission("persistent://p1/c1/ns1/ds2", "other-role", EnumSet.of(AuthAction.consume));
    waitForChange();
    assertEquals(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds2"), "other-role", null), true);
    assertEquals(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null), true);
    assertEquals(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds2"), "other-role", null), false);
    assertEquals(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds2"), "other-role", null, null), true);
    assertEquals(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds2"), "no-access-role", null, null), false);
    assertEquals(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "no-access-role", null), false);
    admin.namespaces().grantPermissionOnNamespace("p1/c1/ns1", "my-role", EnumSet.allOf(AuthAction.class));
    waitForChange();
    assertEquals(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null), true);
    assertEquals(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null), true);
    admin.namespaces().deleteNamespace("p1/c1/ns1");
    admin.properties().deleteProperty("p1");
    admin.clusters().deleteCluster("c1");
}

Example 34 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class BrokerClientIntegrationTest method testCleanProducer.

@Test
public void testCleanProducer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    admin.clusters().createCluster("global", new ClusterData());
    admin.namespaces().createNamespace("my-property/global/lookup");
    final int operationTimeOut = 500;
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString()).statsInterval(0, TimeUnit.SECONDS).operationTimeout(operationTimeOut, TimeUnit.MILLISECONDS).build();
    CountDownLatch latch = new CountDownLatch(1);
    pulsarClient.newProducer().topic("persistent://my-property/global/lookup/my-topic1").createAsync().handle((producer, e) -> {
        latch.countDown();
        return null;
    });
    latch.await(operationTimeOut + 1000, TimeUnit.MILLISECONDS);
    Field prodField = PulsarClientImpl.class.getDeclaredField("producers");
    prodField.setAccessible(true);
    @SuppressWarnings("unchecked") IdentityHashMap<ProducerBase<byte[]>, Boolean> producers = (IdentityHashMap<ProducerBase<byte[]>, Boolean>) prodField.get(pulsarClient);
    assertTrue(producers.isEmpty());
    pulsarClient.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 35 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class AuthenticatedProducerConsumerTest method testInternalServerExceptionOnLookup.

/**
 * verifies that topicLookup/PartitionMetadataLookup gives InternalServerError(500) instead 401(auth_failed) on
 * unknown-exception failure
 *
 * @throws Exception
 */
@Test
public void testInternalServerExceptionOnLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    internalSetup(authTls);
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    String namespace = "my-property/use/my-ns";
    admin.namespaces().createNamespace(namespace);
    String topic = "persistent://" + namespace + "1/topic1";
    // this will cause NPE and it should throw 500
    mockZookKeeper.shutdown();
    pulsar.getConfiguration().setSuperUserRoles(Sets.newHashSet());
    try {
        admin.persistentTopics().getPartitionedTopicMetadata(topic);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
    try {
        admin.lookups().lookupTopic(topic);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
}