Search in sources :

Example 1 with TopicPolicies

use of org.apache.pulsar.common.policies.data.TopicPolicies in project pulsar by apache.

the class NamespaceEventsSystemTopicServiceTest method testSendAndReceiveNamespaceEvents.

@Test
public void testSendAndReceiveNamespaceEvents() throws Exception {
    TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory.createTopicPoliciesSystemTopicClient(NamespaceName.get(NAMESPACE1));
    TopicPolicies policies = TopicPolicies.builder().maxProducerPerTopic(10).build();
    PulsarEvent event = PulsarEvent.builder().eventType(EventType.TOPIC_POLICY).actionType(ActionType.INSERT).topicPoliciesEvent(TopicPoliciesEvent.builder().domain("persistent").tenant("system-topic").namespace(NamespaceName.get(NAMESPACE1).getLocalName()).topic("my-topic").policies(policies).build()).build();
    systemTopicClientForNamespace1.newWriter().write(event);
    SystemTopicClient.Reader reader = systemTopicClientForNamespace1.newReader();
    Message<PulsarEvent> received = reader.readNext();
    log.info("Receive pulsar event from system topic : {}", received.getValue());
    // test event send and receive
    Assert.assertEquals(received.getValue(), event);
    Assert.assertEquals(systemTopicClientForNamespace1.getWriters().size(), 1);
    Assert.assertEquals(systemTopicClientForNamespace1.getReaders().size(), 1);
    // test new reader read
    SystemTopicClient.Reader reader1 = systemTopicClientForNamespace1.newReader();
    Message<PulsarEvent> received1 = reader1.readNext();
    log.info("Receive pulsar event from system topic : {}", received1.getValue());
    Assert.assertEquals(received1.getValue(), event);
    // test writers and readers
    Assert.assertEquals(systemTopicClientForNamespace1.getReaders().size(), 2);
    SystemTopicClient.Writer writer = systemTopicClientForNamespace1.newWriter();
    Assert.assertEquals(systemTopicClientForNamespace1.getWriters().size(), 2);
    writer.close();
    reader.close();
    Assert.assertEquals(systemTopicClientForNamespace1.getWriters().size(), 1);
    Assert.assertEquals(systemTopicClientForNamespace1.getReaders().size(), 1);
    systemTopicClientForNamespace1.close();
    Assert.assertEquals(systemTopicClientForNamespace1.getWriters().size(), 0);
    Assert.assertEquals(systemTopicClientForNamespace1.getReaders().size(), 0);
}
Also used : TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) PulsarEvent(org.apache.pulsar.common.events.PulsarEvent) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)

Example 2 with TopicPolicies

use of org.apache.pulsar.common.policies.data.TopicPolicies in project pulsar by apache.

the class TransactionTest method testSubscriptionRecreateTopic.

@Test
public void testSubscriptionRecreateTopic() throws PulsarAdminException, NoSuchFieldException, IllegalAccessException, PulsarClientException {
    String topic = "persistent://pulsar/system/testReCreateTopic";
    String subName = "sub_testReCreateTopic";
    int retentionSizeInMbSetTo = 5;
    int retentionSizeInMbSetTopic = 6;
    int retentionSizeInMinutesSetTo = 5;
    int retentionSizeInMinutesSetTopic = 6;
    admin.topics().createNonPartitionedTopic(topic);
    PulsarService pulsarService = super.getPulsarServiceList().get(0);
    pulsarService.getBrokerService().getTopics().clear();
    ManagedLedgerFactory managedLedgerFactory = pulsarService.getBrokerService().getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(managedLedgerFactory);
    ledgers.remove(TopicName.get(topic).getPersistenceNamingEncoding());
    try {
        admin.topics().createNonPartitionedTopic(topic);
        Assert.fail();
    } catch (PulsarAdminException.ConflictException e) {
        log.info("Cann`t create topic again");
    }
    admin.topics().setRetention(topic, new RetentionPolicies(retentionSizeInMinutesSetTopic, retentionSizeInMbSetTopic));
    pulsarClient.newConsumer().topic(topic).subscriptionName(subName).subscribe();
    pulsarService.getBrokerService().getTopicIfExists(topic).thenAccept(option -> {
        if (!option.isPresent()) {
            log.error("Failed o get Topic named: {}", topic);
            Assert.fail();
        }
        PersistentTopic originPersistentTopic = (PersistentTopic) option.get();
        String pendingAckTopicName = MLPendingAckStore.getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName);
        try {
            admin.topics().setRetention(pendingAckTopicName, new RetentionPolicies(retentionSizeInMinutesSetTo, retentionSizeInMbSetTo));
        } catch (PulsarAdminException e) {
            log.error("Failed to get./setRetention of topic with Exception:" + e);
            Assert.fail();
        }
        PersistentSubscription subscription = originPersistentTopic.getSubscription(subName);
        subscription.getPendingAckManageLedger().thenAccept(managedLedger -> {
            long retentionSize = managedLedger.getConfig().getRetentionSizeInMB();
            if (!originPersistentTopic.getTopicPolicies().isPresent()) {
                log.error("Failed to getTopicPolicies of :" + originPersistentTopic);
                Assert.fail();
            }
            TopicPolicies topicPolicies = originPersistentTopic.getTopicPolicies().get();
            Assert.assertEquals(retentionSizeInMbSetTopic, retentionSize);
            MLPendingAckStoreProvider mlPendingAckStoreProvider = new MLPendingAckStoreProvider();
            CompletableFuture<PendingAckStore> future = mlPendingAckStoreProvider.newPendingAckStore(subscription);
            future.thenAccept(pendingAckStore -> {
                ((MLPendingAckStore) pendingAckStore).getManagedLedger().thenAccept(managedLedger1 -> {
                    Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), retentionSizeInMbSetTo);
                });
            });
        });
    });
}
Also used : RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) MLPendingAckStoreProvider(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) Field(java.lang.reflect.Field) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) CompletableFuture(java.util.concurrent.CompletableFuture) PulsarService(org.apache.pulsar.broker.PulsarService) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) PendingAckStore(org.apache.pulsar.broker.transaction.pendingack.PendingAckStore) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Test(org.testng.annotations.Test)

Example 3 with TopicPolicies

use of org.apache.pulsar.common.policies.data.TopicPolicies in project pulsar by apache.

the class PersistentTopicsBase method internalSetDeduplicationSnapshotInterval.

protected CompletableFuture<Void> internalSetDeduplicationSnapshotInterval(Integer interval, boolean isGlobal) {
    if (interval != null && interval < 0) {
        throw new RestException(Status.PRECONDITION_FAILED, "interval must be 0 or more");
    }
    return getTopicPoliciesAsyncWithRetry(topicName, isGlobal).thenCompose(op -> {
        TopicPolicies policies = op.orElseGet(TopicPolicies::new);
        policies.setDeduplicationSnapshotIntervalSeconds(interval);
        policies.setIsGlobal(isGlobal);
        return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, policies);
    });
}
Also used : InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) RestException(org.apache.pulsar.broker.web.RestException)

Example 4 with TopicPolicies

use of org.apache.pulsar.common.policies.data.TopicPolicies in project pulsar by apache.

the class PersistentTopicsBase method internalRemoveSubscriptionDispatchRate.

protected CompletableFuture<Void> internalRemoveSubscriptionDispatchRate(boolean isGlobal) {
    return getTopicPoliciesAsyncWithRetry(topicName, isGlobal).thenCompose(op -> {
        if (!op.isPresent()) {
            return CompletableFuture.completedFuture(null);
        }
        TopicPolicies topicPolicies = op.get();
        topicPolicies.setSubscriptionDispatchRate(null);
        topicPolicies.setIsGlobal(isGlobal);
        return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, op.get());
    });
}
Also used : InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies)

Example 5 with TopicPolicies

use of org.apache.pulsar.common.policies.data.TopicPolicies in project pulsar by apache.

the class PersistentTopicsBase method internalSetBacklogQuota.

protected CompletableFuture<Void> internalSetBacklogQuota(BacklogQuota.BacklogQuotaType backlogQuotaType, BacklogQuotaImpl backlogQuota, boolean isGlobal) {
    BacklogQuota.BacklogQuotaType finalBacklogQuotaType = backlogQuotaType == null ? BacklogQuota.BacklogQuotaType.destination_storage : backlogQuotaType;
    return validateTopicPolicyOperationAsync(topicName, PolicyName.BACKLOG, PolicyOperation.WRITE).thenAccept(__ -> validatePoliciesReadOnlyAccess()).thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal)).thenCompose(op -> {
        TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
        return getRetentionPoliciesAsync(topicName, topicPolicies).thenCompose(retentionPolicies -> {
            if (!checkBacklogQuota(backlogQuota, retentionPolicies)) {
                log.warn("[{}] Failed to update backlog configuration for topic {}: conflicts with" + " retention quota", clientAppId(), topicName);
                return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED, "Backlog Quota exceeds configured retention quota for topic. " + "Please increase retention quota and retry"));
            }
            if (backlogQuota != null) {
                topicPolicies.getBackLogQuotaMap().put(finalBacklogQuotaType.name(), backlogQuota);
            } else {
                topicPolicies.getBackLogQuotaMap().remove(finalBacklogQuotaType.name());
            }
            Map<String, BacklogQuotaImpl> backLogQuotaMap = topicPolicies.getBackLogQuotaMap();
            topicPolicies.setIsGlobal(isGlobal);
            return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies).thenRun(() -> {
                try {
                    log.info("[{}] Successfully updated backlog quota map: namespace={}, " + "topic={}, map={}", clientAppId(), namespaceName, topicName.getLocalName(), jsonMapper().writeValueAsString(backLogQuotaMap));
                } catch (JsonProcessingException ignore) {
                }
            });
        });
    });
}
Also used : Version(com.github.zafarkhaja.semver.Version) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) ManagedLedgerOfflineBacklog(org.apache.bookkeeper.mledger.impl.ManagedLedgerOfflineBacklog) Topic(org.apache.pulsar.broker.service.Topic) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) LedgerOffloader(org.apache.bookkeeper.mledger.LedgerOffloader) DelayedDeliveryPolicies(org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) TopicOperation(org.apache.pulsar.common.policies.data.TopicOperation) PolicyOperation(org.apache.pulsar.common.policies.data.PolicyOperation) Map(java.util.Map) RestException(org.apache.pulsar.broker.web.RestException) NamespaceOperation(org.apache.pulsar.common.policies.data.NamespaceOperation) DispatchRateImpl(org.apache.pulsar.common.policies.data.impl.DispatchRateImpl) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) PersistencePolicies(org.apache.pulsar.common.policies.data.PersistencePolicies) AsyncResponse(javax.ws.rs.container.AsyncResponse) EventsTopicNames.checkTopicIsTransactionCoordinatorAssign(org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsTransactionCoordinatorAssign) Set(java.util.Set) PublishRate(org.apache.pulsar.common.policies.data.PublishRate) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) WebApplicationException(javax.ws.rs.WebApplicationException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) Subscription(org.apache.pulsar.broker.service.Subscription) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) ArrayList(java.util.ArrayList) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) DateFormatter(org.apache.pulsar.common.util.DateFormatter) BiConsumer(java.util.function.BiConsumer) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) ResponseBuilder(javax.ws.rs.core.Response.ResponseBuilder) DispatchRate(org.apache.pulsar.common.policies.data.DispatchRate) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) ManagedLedgerInfo(org.apache.bookkeeper.mledger.ManagedLedgerInfo) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) ExecutionException(java.util.concurrent.ExecutionException) PulsarService.isTransactionInternalName(org.apache.pulsar.broker.PulsarService.isTransactionInternalName) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) SubscriptionInvalidCursorPosition(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionInvalidCursorPosition) AuthorizationService(org.apache.pulsar.broker.authorization.AuthorizationService) DEFAULT_OPERATION_TIMEOUT_SEC(org.apache.pulsar.broker.resources.PulsarResources.DEFAULT_OPERATION_TIMEOUT_SEC) AdminResource(org.apache.pulsar.broker.admin.AdminResource) LoggerFactory(org.slf4j.LoggerFactory) KeyValue(org.apache.pulsar.common.api.proto.KeyValue) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ManagedLedgerFactoryImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) PreconditionFailedException(org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) AuthAction(org.apache.pulsar.common.policies.data.AuthAction) AuthPolicies(org.apache.pulsar.common.policies.data.AuthPolicies) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) BrokerEntryMetadata(org.apache.pulsar.common.api.proto.BrokerEntryMetadata) InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) PartitionedTopicInternalStats(org.apache.pulsar.common.policies.data.PartitionedTopicInternalStats) SchemaCompatibilityStrategy(org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy) OffloadPoliciesImpl(org.apache.pulsar.common.policies.data.OffloadPoliciesImpl) BitSetRecyclable(org.apache.pulsar.common.util.collections.BitSetRecyclable) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CompletionException(java.util.concurrent.CompletionException) StreamingOutput(javax.ws.rs.core.StreamingOutput) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) PulsarByteBufAllocator(org.apache.pulsar.common.allocator.PulsarByteBufAllocator) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) BacklogQuotaImpl(org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl) Base64(java.util.Base64) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Response(javax.ws.rs.core.Response) EncryptionKeys(org.apache.pulsar.common.api.proto.EncryptionKeys) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) PolicyName(org.apache.pulsar.common.policies.data.PolicyName) PartitionedTopicStatsImpl(org.apache.pulsar.common.policies.data.stats.PartitionedTopicStatsImpl) Optional(java.util.Optional) IntStream(java.util.stream.IntStream) CompressionCodec(org.apache.pulsar.common.compression.CompressionCodec) TopicName(org.apache.pulsar.common.naming.TopicName) Entry(org.apache.bookkeeper.mledger.Entry) ManagedLedgerInfoCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ManagedLedgerInfoCallback) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) ByteBuf(io.netty.buffer.ByteBuf) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) CompressionCodecProvider(org.apache.pulsar.common.compression.CompressionCodecProvider) TopicDomain(org.apache.pulsar.common.naming.TopicDomain) Status(javax.ws.rs.core.Response.Status) PartitionedManagedLedgerInfo(org.apache.pulsar.common.naming.PartitionedManagedLedgerInfo) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) PersistentReplicator(org.apache.pulsar.broker.service.persistent.PersistentReplicator) Logger(org.slf4j.Logger) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) TransactionCoordinatorID(org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID) Collections(java.util.Collections) InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) BacklogQuotaImpl(org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl) RestException(org.apache.pulsar.broker.web.RestException) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException)

Aggregations

TopicPolicies (org.apache.pulsar.common.policies.data.TopicPolicies)41 InactiveTopicPolicies (org.apache.pulsar.common.policies.data.InactiveTopicPolicies)27 RestException (org.apache.pulsar.broker.web.RestException)12 TopicName (org.apache.pulsar.common.naming.TopicName)9 Test (org.testng.annotations.Test)9 CompletableFuture (java.util.concurrent.CompletableFuture)7 List (java.util.List)6 MockedPulsarServiceBaseTest (org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)6 RetentionPolicies (org.apache.pulsar.common.policies.data.RetentionPolicies)6 Map (java.util.Map)5 PulsarServerException (org.apache.pulsar.broker.PulsarServerException)5 Lists (com.google.common.collect.Lists)4 Sets (com.google.common.collect.Sets)4 TimeUnit (java.util.concurrent.TimeUnit)4 MessageId (org.apache.pulsar.client.api.MessageId)4 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)4 MessageImpl (org.apache.pulsar.client.impl.MessageImpl)4 PulsarEvent (org.apache.pulsar.common.events.PulsarEvent)4 NamespaceName (org.apache.pulsar.common.naming.NamespaceName)4 Logger (org.slf4j.Logger)4