Examples with NamespaceResources org.apache.pulsar.broker.resources.NamespaceResources used on opensource projects

Example 1 with NamespaceResources

use of org.apache.pulsar.broker.resources.NamespaceResources in project starlight-for-kafka by datastax.

the class KafkaServiceConfigurationTest method testAllowedNamespaces.

@Test
public void testAllowedNamespaces() throws Exception {
    final KafkaServiceConfiguration conf = new KafkaServiceConfiguration();
    assertEquals(conf.getKopAllowedNamespaces(), Collections.singletonList("${tenant}/default"));
    assertEquals(KafkaRequestHandler.expandAllowedNamespaces(conf.getKopAllowedNamespaces(), "my-tenant", null).get(), Collections.singletonList("my-tenant/default"));
    conf.setKafkaNamespace("my-ns");
    assertEquals(KafkaRequestHandler.expandAllowedNamespaces(conf.getKopAllowedNamespaces(), "my-tenant", null).get(), Collections.singletonList("my-tenant/my-ns"));
    conf.setKopAllowedNamespaces(Collections.singleton("my-tenant-2/my-ns-2"));
    assertEquals(KafkaRequestHandler.expandAllowedNamespaces(conf.getKopAllowedNamespaces(), "my-tenant", null).get(), Collections.singletonList("my-tenant-2/my-ns-2"));
    conf.setKopAllowedNamespaces(null);
    assertEquals(KafkaRequestHandler.expandAllowedNamespaces(conf.getKopAllowedNamespaces(), "my-tenant", null).get(), Collections.singletonList("my-tenant/my-ns"));
    conf.setKopAllowedNamespaces(Sets.newHashSet("my-tenant/my-ns-0", "my-tenant/my-ns-1"));
    assertEquals(KafkaRequestHandler.expandAllowedNamespaces(conf.getKopAllowedNamespaces(), "my-tenant", null).get(), Arrays.asList("my-tenant/my-ns-0", "my-tenant/my-ns-1"));
    conf.setKopAllowedNamespaces(Collections.singleton("${tenant}/*"));
    PulsarService pulsarService = mock(PulsarService.class);
    PulsarResources pulsarResources = mock(PulsarResources.class);
    NamespaceResources namespaceResources = mock(NamespaceResources.class);
    when(pulsarService.getPulsarResources()).thenReturn(pulsarResources);
    when(pulsarResources.getNamespaceResources()).thenReturn(namespaceResources);
    when(namespaceResources.getChildrenAsync(any(String.class))).thenReturn(CompletableFuture.completedFuture(Arrays.asList("one", "two")));
    assertEquals(KafkaRequestHandler.expandAllowedNamespaces(conf.getKopAllowedNamespaces(), "logged", pulsarService).get(), Arrays.asList("logged/one", "logged/two"));
}

Example 2 with NamespaceResources

use of org.apache.pulsar.broker.resources.NamespaceResources in project pulsar by apache.

the class PulsarClusterMetadataSetup method createNamespaceIfAbsent.

static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, String cluster) throws IOException {
    NamespaceResources namespaceResources = resources.getNamespaceResources();
    if (!namespaceResources.namespaceExists(namespaceName)) {
        Policies policies = new Policies();
        policies.bundles = getBundles(16);
        policies.replication_clusters = Collections.singleton(cluster);
        namespaceResources.createPolicies(namespaceName, policies);
    } else {
        namespaceResources.setPolicies(namespaceName, policies -> {
            policies.replication_clusters.add(cluster);
            return policies;
        });
    }
}

Example 3 with NamespaceResources

use of org.apache.pulsar.broker.resources.NamespaceResources in project pulsar by apache.

the class PersistentTopicTest method testCheckInactiveSubscriptions.

@Test
public void testCheckInactiveSubscriptions() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = new ConcurrentOpenHashMap<>(16, 1);
    // This subscription is connected by consumer.
    PersistentSubscription nonDeletableSubscription1 = spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "nonDeletableSubscription1", cursorMock, false);
    subscriptions.put(nonDeletableSubscription1.getName(), nonDeletableSubscription1);
    // This subscription is not connected by consumer.
    PersistentSubscription deletableSubscription1 = spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "deletableSubscription1", cursorMock, false);
    subscriptions.put(deletableSubscription1.getName(), deletableSubscription1);
    // This subscription is replicated.
    PersistentSubscription nonDeletableSubscription2 = spyWithClassAndConstructorArgs(PersistentSubscription.class, topic, "nonDeletableSubscription2", cursorMock, true);
    subscriptions.put(nonDeletableSubscription2.getName(), nonDeletableSubscription2);
    Field field = topic.getClass().getDeclaredField("subscriptions");
    field.setAccessible(true);
    field.set(topic, subscriptions);
    Method addConsumerToSubscription = AbstractTopic.class.getDeclaredMethod("addConsumerToSubscription", Subscription.class, Consumer.class);
    addConsumerToSubscription.setAccessible(true);
    Consumer consumer = new Consumer(nonDeletableSubscription1, SubType.Shared, topic.getName(), 1, 0, "consumer1", true, serverCnx, "app1", Collections.emptyMap(), false, InitialPosition.Latest, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH);
    addConsumerToSubscription.invoke(topic, nonDeletableSubscription1, consumer);
    NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources();
    NamespaceName ns = TopicName.get(successTopicName).getNamespaceObject();
    doReturn(Optional.of(new Policies())).when(nsr).getPolicies(ns);
    ServiceConfiguration svcConfig = spy(ServiceConfiguration.class);
    doReturn(5).when(svcConfig).getSubscriptionExpirationTimeMinutes();
    doReturn(svcConfig).when(pulsar).getConfiguration();
    doReturn(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(6)).when(cursorMock).getLastActive();
    topic.checkInactiveSubscriptions();
    verify(nonDeletableSubscription1, times(0)).delete();
    verify(deletableSubscription1, times(1)).delete();
    verify(nonDeletableSubscription2, times(0)).delete();
}

Example 4 with NamespaceResources

use of org.apache.pulsar.broker.resources.NamespaceResources in project pulsar by yahoo.

the class BrokerService method getManagedLedgerConfig.

public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(TopicName topicName) {
    NamespaceName namespace = topicName.getNamespaceObject();
    ServiceConfiguration serviceConfig = pulsar.getConfiguration();
    NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources();
    LocalPoliciesResources lpr = pulsar.getPulsarResources().getLocalPolicies();
    return nsr.getPoliciesAsync(namespace).thenCombine(lpr.getLocalPoliciesAsync(namespace), (policies, localPolicies) -> {
        PersistencePolicies persistencePolicies = null;
        RetentionPolicies retentionPolicies = null;
        OffloadPoliciesImpl topicLevelOffloadPolicies = null;
        if (pulsar.getConfig().isTopicLevelPoliciesEnabled()) {
            try {
                TopicPolicies topicPolicies = pulsar.getTopicPoliciesService().getTopicPolicies(topicName);
                if (topicPolicies != null) {
                    persistencePolicies = topicPolicies.getPersistence();
                    retentionPolicies = topicPolicies.getRetentionPolicies();
                    topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies();
                }
            } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
                log.debug("Topic {} policies have not been initialized yet.", topicName);
            }
        }
        if (persistencePolicies == null) {
            persistencePolicies = policies.map(p -> p.persistence).orElseGet(() -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), serviceConfig.getManagedLedgerDefaultWriteQuorum(), serviceConfig.getManagedLedgerDefaultAckQuorum(), serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit()));
        }
        if (retentionPolicies == null) {
            retentionPolicies = policies.map(p -> p.retention_policies).orElseGet(() -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), serviceConfig.getDefaultRetentionSizeInMB()));
        }
        ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
        managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble());
        managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
        managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
        if (serviceConfig.isStrictBookieAffinityEnabled()) {
            managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName(IsolatedBookieEnsemblePlacementPolicy.class);
            if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) {
                Map<String, Object> properties = Maps.newHashMap();
                properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary());
                properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary());
                managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties);
            } else if (isSystemTopic(topicName)) {
                Map<String, Object> properties = Maps.newHashMap();
                properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*");
                properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, "*");
                managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties);
            } else {
                Map<String, Object> properties = Maps.newHashMap();
                properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "");
                properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, "");
                managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties);
            }
        } else {
            if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) {
                managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName(IsolatedBookieEnsemblePlacementPolicy.class);
                Map<String, Object> properties = Maps.newHashMap();
                properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary());
                properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary());
                managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties);
            }
        }
        managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
        managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType());
        managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword());
        managedLedgerConfig.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
        managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled(serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled());
        managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore(serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore());
        managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
        managedLedgerConfig.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        managedLedgerConfig.setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes());
        managedLedgerConfig.setMetadataOperationsTimeoutSeconds(serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds());
        managedLedgerConfig.setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds());
        managedLedgerConfig.setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds());
        managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize());
        managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled(serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled());
        managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum());
        managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum());
        managedLedgerConfig.setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger());
        managedLedgerConfig.setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds());
        managedLedgerConfig.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES);
        managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB());
        managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData());
        managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery());
        managedLedgerConfig.setInactiveLedgerRollOverTime(serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS);
        managedLedgerConfig.setCacheEvictionByMarkDeletedPosition(serviceConfig.isCacheEvictionByMarkDeletedPosition());
        OffloadPoliciesImpl nsLevelOffloadPolicies = (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null);
        OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration(topicLevelOffloadPolicies, OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), getPulsar().getConfig().getProperties());
        if (NamespaceService.isSystemServiceNamespace(namespace.toString())) {
            managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE);
        } else {
            if (topicLevelOffloadPolicies != null) {
                try {
                    LedgerOffloader topicLevelLedgerOffLoader = pulsar().createManagedLedgerOffloader(offloadPolicies);
                    managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader);
                } catch (PulsarServerException e) {
                    throw new RuntimeException(e);
                }
            } else {
                // If the topic level policy is null, use the namespace level
                managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies));
            }
        }
        managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled(serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled());
        managedLedgerConfig.setNewEntriesCheckDelayInMillis(serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis());
        return managedLedgerConfig;
    });
}

Example 5 with NamespaceResources

use of org.apache.pulsar.broker.resources.NamespaceResources in project pulsar by yahoo.

the class PulsarClusterMetadataSetup method createNamespaceIfAbsent.

static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, String cluster) throws IOException {
    NamespaceResources namespaceResources = resources.getNamespaceResources();
    if (!namespaceResources.namespaceExists(namespaceName)) {
        Policies policies = new Policies();
        policies.bundles = getBundles(16);
        policies.replication_clusters = Collections.singleton(cluster);
        namespaceResources.createPolicies(namespaceName, policies);
    } else {
        namespaceResources.setPolicies(namespaceName, policies -> {
            policies.replication_clusters.add(cluster);
            return policies;
        });
    }
}