Examples with SystemTopicBasedTopicPoliciesService org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService used on opensource projects

Example 1 with SystemTopicBasedTopicPoliciesService

use of org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService in project pulsar by apache.

the class TopicPoliciesTest method testTopicPolicyInitialValueWithNamespaceAlreadyLoaded.

@Test
public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception {
    TopicName topicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(myNamespace), "test-" + UUID.randomUUID());
    String topic = topicName.toString();
    SystemTopicBasedTopicPoliciesService policyService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService();
    // set up topic with maxSubscriptionsPerTopic = 10
    admin.topics().createNonPartitionedTopic(topic);
    admin.topicPolicies().setMaxSubscriptionsPerTopicAsync(topic, 10).get();
    // wait until topic loaded with right policy value.
    Awaitility.await().untilAsserted(() -> {
        AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get();
        assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10));
    });
    // unload the topic
    pulsar.getNamespaceService().unloadNamespaceBundle(pulsar.getNamespaceService().getBundle(topicName)).get();
    assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic));
    // load the nameserver, but topic is not init.
    log.info("lookup:{}", admin.lookups().lookupTopic(topic));
    assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBroker(topicName));
    assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic));
    // make sure namespace policy reader is fully started.
    Awaitility.await().untilAsserted(() -> {
        assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()));
    });
    // load the topic.
    AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get();
    assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10));
}

Example 2 with SystemTopicBasedTopicPoliciesService

use of org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService in project pulsar by yahoo.

the class PulsarService method start.

/**
 * Start the pulsar service instance.
 */
public void start() throws PulsarServerException {
    LOG.info("Starting Pulsar Broker service; version: '{}'", (brokerVersion != null ? brokerVersion : "unknown"));
    LOG.info("Git Revision {}", PulsarVersion.getGitSha());
    LOG.info("Git Branch {}", PulsarVersion.getGitBranch());
    LOG.info("Built by {} on {} at {}", PulsarVersion.getBuildUser(), PulsarVersion.getBuildHost(), PulsarVersion.getBuildTime());
    // start time mills
    long startTimestamp = System.currentTimeMillis();
    mutex.lock();
    try {
        if (state != State.Init) {
            throw new PulsarServerException("Cannot start the service once it was stopped");
        }
        if (!config.getWebServicePort().isPresent() && !config.getWebServicePortTls().isPresent()) {
            throw new IllegalArgumentException("webServicePort/webServicePortTls must be present");
        }
        if (config.isAuthorizationEnabled() && !config.isAuthenticationEnabled()) {
            throw new IllegalStateException("Invalid broker configuration. Authentication must be enabled with " + "authenticationEnabled=true when authorization is enabled with authorizationEnabled=true.");
        }
        if (config.getDefaultRetentionSizeInMB() > 0 && config.getBacklogQuotaDefaultLimitBytes() > 0 && config.getBacklogQuotaDefaultLimitBytes() >= (config.getDefaultRetentionSizeInMB() * 1024L * 1024L)) {
            throw new IllegalArgumentException(String.format("The retention size must > the backlog quota limit " + "size, but the configured backlog quota limit bytes is %d, the retention size is %d", config.getBacklogQuotaDefaultLimitBytes(), config.getDefaultRetentionSizeInMB() * 1024L * 1024L));
        }
        if (config.getDefaultRetentionTimeInMinutes() > 0 && config.getBacklogQuotaDefaultLimitSecond() > 0 && config.getBacklogQuotaDefaultLimitSecond() >= config.getDefaultRetentionTimeInMinutes() * 60) {
            throw new IllegalArgumentException(String.format("The retention time must > the backlog quota limit " + "time, but the configured backlog quota limit time duration is %d, " + "the retention time duration is %d", config.getBacklogQuotaDefaultLimitSecond(), config.getDefaultRetentionTimeInMinutes() * 60));
        }
        if (!config.getLoadBalancerOverrideBrokerNicSpeedGbps().isPresent() && config.isLoadBalancerEnabled() && LinuxInfoUtils.isLinux() && !LinuxInfoUtils.checkHasNicSpeeds()) {
            throw new IllegalStateException("Unable to read VM NIC speed. You must set " + "[loadBalancerOverrideBrokerNicSpeedGbps] to override it when load balancer is enabled.");
        }
        localMetadataStore = createLocalMetadataStore();
        localMetadataStore.registerSessionListener(this::handleMetadataSessionEvent);
        coordinationService = new CoordinationServiceImpl(localMetadataStore);
        if (config.isConfigurationStoreSeparated()) {
            configurationMetadataStore = createConfigurationMetadataStore();
            shouldShutdownConfigurationMetadataStore = true;
        } else {
            configurationMetadataStore = localMetadataStore;
            shouldShutdownConfigurationMetadataStore = false;
        }
        pulsarResources = new PulsarResources(localMetadataStore, configurationMetadataStore, config.getMetadataStoreOperationTimeoutSeconds());
        pulsarResources.getClusterResources().getStore().registerListener(this::handleDeleteCluster);
        orderedExecutor = OrderedExecutor.newBuilder().numThreads(config.getNumOrderedExecutorThreads()).name("pulsar-ordered").build();
        // Initialize the message protocol handlers
        protocolHandlers = ProtocolHandlers.load(config);
        protocolHandlers.initialize(config);
        // Now we are ready to start services
        this.bkClientFactory = newBookKeeperClientFactory();
        managedLedgerClientFactory = ManagedLedgerStorage.create(config, localMetadataStore, bkClientFactory, ioEventLoopGroup);
        this.brokerService = newBrokerService(this);
        // Start load management service (even if load balancing is disabled)
        this.loadManager.set(LoadManager.create(this));
        // needs load management service and before start broker service,
        this.startNamespaceService();
        schemaStorage = createAndStartSchemaStorage();
        schemaRegistryService = SchemaRegistryService.create(schemaStorage, config.getSchemaRegistryCompatibilityCheckers());
        this.defaultOffloader = createManagedLedgerOffloader(OffloadPoliciesImpl.create(this.getConfiguration().getProperties()));
        this.brokerInterceptor = BrokerInterceptors.load(config);
        brokerService.setInterceptor(getBrokerInterceptor());
        this.brokerInterceptor.initialize(this);
        brokerService.start();
        // Load additional servlets
        this.brokerAdditionalServlets = AdditionalServlets.load(config);
        this.webService = new WebService(this);
        createMetricsServlet();
        this.addWebServerHandlers(webService, metricsServlet, this.config);
        this.webService.start();
        // Refresh addresses and update configuration, since the port might have been dynamically assigned
        if (config.getBrokerServicePort().equals(Optional.of(0))) {
            config.setBrokerServicePort(brokerService.getListenPort());
        }
        if (config.getBrokerServicePortTls().equals(Optional.of(0))) {
            config.setBrokerServicePortTls(brokerService.getListenPortTls());
        }
        this.webServiceAddress = webAddress(config);
        this.webServiceAddressTls = webAddressTls(config);
        this.brokerServiceUrl = brokerUrl(config);
        this.brokerServiceUrlTls = brokerUrlTls(config);
        if (null != this.webSocketService) {
            ClusterDataImpl clusterData = ClusterDataImpl.builder().serviceUrl(webServiceAddress).serviceUrlTls(webServiceAddressTls).brokerServiceUrl(brokerServiceUrl).brokerServiceUrlTls(brokerServiceUrlTls).build();
            this.webSocketService.setLocalCluster(clusterData);
        }
        // Initialize namespace service, after service url assigned. Should init zk and refresh self owner info.
        this.nsService.initialize();
        // Start topic level policies service
        if (config.isTopicLevelPoliciesEnabled() && config.isSystemTopicEnabled()) {
            this.topicPoliciesService = new SystemTopicBasedTopicPoliciesService(this);
        }
        this.topicPoliciesService.start();
        // Start the leader election service
        startLeaderElectionService();
        // Register heartbeat and bootstrap namespaces.
        this.nsService.registerBootstrapNamespaces();
        // Register pulsar system namespaces and start transaction meta store service
        if (config.isTransactionCoordinatorEnabled()) {
            this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient());
            this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer"));
            transactionBufferClient = TransactionBufferClientImpl.create(this, transactionTimer, config.getTransactionBufferClientMaxConcurrentRequests(), config.getTransactionBufferClientOperationTimeoutInMills());
            transactionMetadataStoreService = new TransactionMetadataStoreService(TransactionMetadataStoreProvider.newProvider(config.getTransactionMetadataStoreProviderClassName()), this, transactionBufferClient, transactionTimer);
            transactionBufferProvider = TransactionBufferProvider.newProvider(config.getTransactionBufferProviderClassName());
            transactionPendingAckStoreProvider = TransactionPendingAckStoreProvider.newProvider(config.getTransactionPendingAckStoreProviderClassName());
        }
        this.metricsGenerator = new MetricsGenerator(this);
        // By starting the Load manager service, the broker will also become visible
        // to the rest of the broker by creating the registration z-node. This needs
        // to be done only when the broker is fully operative.
        this.startLoadManagementService();
        // Initialize the message protocol handlers.
        // start the protocol handlers only after the broker is ready,
        // so that the protocol handlers can access broker service properly.
        this.protocolHandlers.start(brokerService);
        Map<String, Map<InetSocketAddress, ChannelInitializer<SocketChannel>>> protocolHandlerChannelInitializers = this.protocolHandlers.newChannelInitializers();
        this.brokerService.startProtocolHandlers(protocolHandlerChannelInitializers);
        acquireSLANamespace();
        // start function worker service if necessary
        this.startWorkerService(brokerService.getAuthenticationService(), brokerService.getAuthorizationService());
        // start packages management service if necessary
        if (config.isEnablePackagesManagement()) {
            this.startPackagesManagementService();
        }
        // Start the task to publish resource usage, if necessary
        this.resourceUsageTransportManager = DISABLE_RESOURCE_USAGE_TRANSPORT_MANAGER;
        if (isNotBlank(config.getResourceUsageTransportClassName())) {
            Class<?> clazz = Class.forName(config.getResourceUsageTransportClassName());
            Constructor<?> ctor = clazz.getConstructor(PulsarService.class);
            Object object = ctor.newInstance(new Object[] { this });
            this.resourceUsageTransportManager = (ResourceUsageTopicTransportManager) object;
        }
        this.resourceGroupServiceManager = new ResourceGroupService(this);
        long currentTimestamp = System.currentTimeMillis();
        final long bootstrapTimeSeconds = TimeUnit.MILLISECONDS.toSeconds(currentTimestamp - startTimestamp);
        final String bootstrapMessage = "bootstrap service " + (config.getWebServicePort().isPresent() ? "port = " + config.getWebServicePort().get() : "") + (config.getWebServicePortTls().isPresent() ? ", tls-port = " + config.getWebServicePortTls() : "") + (StringUtils.isNotEmpty(brokerServiceUrl) ? ", broker url= " + brokerServiceUrl : "") + (StringUtils.isNotEmpty(brokerServiceUrlTls) ? ", broker tls url= " + brokerServiceUrlTls : "");
        LOG.info("messaging service is ready, bootstrap_seconds={}", bootstrapTimeSeconds);
        LOG.info("messaging service is ready, {}, cluster={}, configs={}", bootstrapMessage, config.getClusterName(), config);
        state = State.Started;
    } catch (Exception e) {
        LOG.error("Failed to start Pulsar service: {}", e.getMessage(), e);
        throw new PulsarServerException(e);
    } finally {
        mutex.unlock();
    }
}

Example 3 with SystemTopicBasedTopicPoliciesService

use of org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService in project pulsar by yahoo.

the class TopicPoliciesTest method testTopicPolicyInitialValueWithNamespaceAlreadyLoaded.

@Test
public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception {
    TopicName topicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(myNamespace), "test-" + UUID.randomUUID());
    String topic = topicName.toString();
    SystemTopicBasedTopicPoliciesService policyService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService();
    // set up topic with maxSubscriptionsPerTopic = 10
    admin.topics().createNonPartitionedTopic(topic);
    admin.topicPolicies().setMaxSubscriptionsPerTopicAsync(topic, 10).get();
    // wait until topic loaded with right policy value.
    Awaitility.await().untilAsserted(() -> {
        AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get();
        assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10));
    });
    // unload the topic
    pulsar.getNamespaceService().unloadNamespaceBundle(pulsar.getNamespaceService().getBundle(topicName)).get();
    assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic));
    // load the nameserver, but topic is not init.
    log.info("lookup:{}", admin.lookups().lookupTopic(topic));
    assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBroker(topicName));
    assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic));
    // make sure namespace policy reader is fully started.
    Awaitility.await().untilAsserted(() -> {
        assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()));
    });
    // load the topic.
    AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get();
    assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10));
}

Example 4 with SystemTopicBasedTopicPoliciesService

use of org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService in project pulsar by yahoo.

the class TopicPoliciesTest method testGlobalTopicPolicies.

@Test
public void testGlobalTopicPolicies() throws Exception {
    final String topic = testTopic + UUID.randomUUID();
    pulsarClient.newProducer().topic(topic).create().close();
    Awaitility.await().untilAsserted(() -> Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNull());
    admin.topicPolicies(true).setRetention(topic, new RetentionPolicies(1, 2));
    SystemTopicBasedTopicPoliciesService topicPoliciesService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService();
    // check global topic policies can be added correctly.
    Awaitility.await().untilAsserted(() -> assertNotNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic), true)));
    TopicPolicies topicPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic), true);
    assertNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic)));
    assertEquals(topicPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 1);
    assertEquals(topicPolicies.getRetentionPolicies().getRetentionSizeInMB(), 2);
    // check global topic policies can be updated correctly.
    admin.topicPolicies(true).setRetention(topic, new RetentionPolicies(3, 4));
    Awaitility.await().untilAsserted(() -> {
        TopicPolicies tempPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic), true);
        assertNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic)));
        assertEquals(tempPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 3);
        assertEquals(tempPolicies.getRetentionPolicies().getRetentionSizeInMB(), 4);
    });
    // Local topic policies and global topic policies can exist together.
    admin.topicPolicies().setRetention(topic, new RetentionPolicies(10, 20));
    Awaitility.await().untilAsserted(() -> assertNotNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic))));
    TopicPolicies tempPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic), true);
    assertEquals(tempPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 3);
    assertEquals(tempPolicies.getRetentionPolicies().getRetentionSizeInMB(), 4);
    tempPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic));
    assertEquals(tempPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 10);
    assertEquals(tempPolicies.getRetentionPolicies().getRetentionSizeInMB(), 20);
    // check remove global topic policies can be removed correctly.
    admin.topicPolicies(true).removeRetention(topic);
    Awaitility.await().untilAsserted(() -> assertNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic), true).getRetentionPolicies()));
}

Example 5 with SystemTopicBasedTopicPoliciesService

use of org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService in project incubator-pulsar by apache.

the class TopicPoliciesTest method testTopicPolicyInitialValueWithNamespaceAlreadyLoaded.

@Test
public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception {
    TopicName topicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(myNamespace), "test-" + UUID.randomUUID());
    String topic = topicName.toString();
    SystemTopicBasedTopicPoliciesService policyService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService();
    // set up topic with maxSubscriptionsPerTopic = 10
    admin.topics().createNonPartitionedTopic(topic);
    admin.topicPolicies().setMaxSubscriptionsPerTopicAsync(topic, 10).get();
    // wait until topic loaded with right policy value.
    Awaitility.await().untilAsserted(() -> {
        AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get();
        assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10));
    });
    // unload the topic
    pulsar.getNamespaceService().unloadNamespaceBundle(pulsar.getNamespaceService().getBundle(topicName)).get();
    assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic));
    // load the nameserver, but topic is not init.
    log.info("lookup:{}", admin.lookups().lookupTopic(topic));
    assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBroker(topicName));
    assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic));
    // make sure namespace policy reader is fully started.
    Awaitility.await().untilAsserted(() -> {
        assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()));
    });
    // load the topic.
    AbstractTopic topic1 = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get();
    assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10));
}