Search in sources :

Example 1 with PersistencePolicies

use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.

the class PulsarAdminToolTest method namespaces.

@Test
void namespaces() throws Exception {
    PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
    Namespaces mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    Lookup mockLookup = mock(Lookup.class);
    when(admin.lookups()).thenReturn(mockLookup);
    CmdNamespaces namespaces = new CmdNamespaces(admin);
    namespaces.run(split("list myprop"));
    verify(mockNamespaces).getNamespaces("myprop");
    namespaces.run(split("list-cluster myprop/clust"));
    verify(mockNamespaces).getNamespaces("myprop", "clust");
    namespaces.run(split("destinations myprop/clust/ns1"));
    verify(mockNamespaces).getDestinations("myprop/clust/ns1");
    namespaces.run(split("policies myprop/clust/ns1"));
    verify(mockNamespaces).getPolicies("myprop/clust/ns1");
    namespaces.run(split("create myprop/clust/ns1"));
    verify(mockNamespaces).createNamespace("myprop/clust/ns1");
    namespaces.run(split("delete myprop/clust/ns1"));
    verify(mockNamespaces).deleteNamespace("myprop/clust/ns1");
    namespaces.run(split("permissions myprop/clust/ns1"));
    verify(mockNamespaces).getPermissions("myprop/clust/ns1");
    namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume"));
    verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", EnumSet.of(AuthAction.produce, AuthAction.consume));
    namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1"));
    verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1");
    namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc"));
    verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", Lists.newArrayList("use", "usw", "usc"));
    namespaces.run(split("get-clusters myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1");
    namespaces.run(split("unload myprop/clust/ns1"));
    verify(mockNamespaces).unload("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff"));
    verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff");
    namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff"));
    verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff");
    namespaces.run(split("get-backlog-quotas myprop/clust/ns1"));
    verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1");
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10, RetentionPolicy.producer_request_hold));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10l * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
    namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0"));
    verify(mockNamespaces).setPersistence("myprop/clust/ns1", new PersistencePolicies(2, 1, 1, 100.0d));
    namespaces.run(split("get-persistence myprop/clust/ns1"));
    verify(mockNamespaces).getPersistence("myprop/clust/ns1");
    namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300"));
    verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);
    namespaces.run(split("get-message-ttl myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1");
    namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M"));
    verify(mockNamespaces).setRetention("myprop/clust/ns1", new RetentionPolicies(60, 1));
    namespaces.run(split("get-retention myprop/clust/ns1"));
    verify(mockNamespaces).getRetention("myprop/clust/ns1");
    namespaces.run(split("clear-backlog myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
}
Also used : RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) Namespaces(com.yahoo.pulsar.client.admin.Namespaces) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) PulsarAdmin(com.yahoo.pulsar.client.admin.PulsarAdmin) Lookup(com.yahoo.pulsar.client.admin.Lookup) BacklogQuota(com.yahoo.pulsar.common.policies.data.BacklogQuota) Test(org.testng.annotations.Test)

Example 2 with PersistencePolicies

use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.

the class BrokerService method getManagedLedgerConfig.

public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(DestinationName topicName) {
    CompletableFuture<ManagedLedgerConfig> future = new CompletableFuture<>();
    // Execute in background thread, since getting the policies might block if the z-node wasn't already cached
    pulsar.getOrderedExecutor().submitOrdered(topicName, safeRun(() -> {
        NamespaceName namespace = topicName.getNamespaceObject();
        ServiceConfiguration serviceConfig = pulsar.getConfiguration();
        // Get persistence policy for this destination
        Policies policies;
        try {
            policies = pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", namespace.getProperty(), namespace.getCluster(), namespace.getLocalName())).orElse(null);
        } catch (Throwable t) {
            // Ignoring since if we don't have policies, we fallback on the default
            log.warn("Got exception when reading persistence policy for {}: {}", topicName, t.getMessage(), t);
            future.completeExceptionally(t);
            return;
        }
        PersistencePolicies persistencePolicies = policies != null ? policies.persistence : null;
        RetentionPolicies retentionPolicies = policies != null ? policies.retention_policies : null;
        if (persistencePolicies == null) {
            // Apply default values
            persistencePolicies = new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), serviceConfig.getManagedLedgerDefaultWriteQuorum(), serviceConfig.getManagedLedgerDefaultAckQuorum(), serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit());
        }
        if (retentionPolicies == null) {
            retentionPolicies = new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), serviceConfig.getDefaultRetentionSizeInMB());
        }
        ManagedLedgerConfig config = new ManagedLedgerConfig();
        config.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble());
        config.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
        config.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
        config.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
        config.setDigestType(DigestType.CRC32);
        config.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
        config.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
        config.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        config.setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        config.setMaxSizePerLedgerMb(2048);
        config.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize());
        config.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum());
        config.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum());
        config.setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger());
        config.setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds());
        config.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES);
        config.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB());
        future.complete(config);
    }, (exception) -> future.completeExceptionally(exception)));
    return future;
}
Also used : RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) LoggerFactory(org.slf4j.LoggerFactory) NamespaceBundleStats(com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats) Stat(org.apache.zookeeper.data.Stat) Policies(com.yahoo.pulsar.common.policies.data.Policies) EpollMode(io.netty.channel.epoll.EpollMode) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PersistentOfflineTopicStats(com.yahoo.pulsar.common.policies.data.PersistentOfflineTopicStats) ClusterData(com.yahoo.pulsar.common.policies.data.ClusterData) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Map(java.util.Map) ZooKeeperCacheListener(com.yahoo.pulsar.zookeeper.ZooKeeperCacheListener) PulsarService(com.yahoo.pulsar.broker.PulsarService) FieldParser(com.yahoo.pulsar.common.util.FieldParser) PulsarClientImpl(com.yahoo.pulsar.client.impl.PulsarClientImpl) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) URI(java.net.URI) PulsarClient(com.yahoo.pulsar.client.api.PulsarClient) SystemUtils(org.apache.commons.lang.SystemUtils) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) Set(java.util.Set) NioServerSocketChannel(io.netty.channel.socket.nio.NioServerSocketChannel) PooledByteBufAllocator(io.netty.buffer.PooledByteBufAllocator) InetSocketAddress(java.net.InetSocketAddress) Executors(java.util.concurrent.Executors) NioEventLoopGroup(io.netty.channel.nio.NioEventLoopGroup) ObjectMapperFactory(com.yahoo.pulsar.common.util.ObjectMapperFactory) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Metrics(com.yahoo.pulsar.broker.stats.Metrics) List(java.util.List) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) ClientConfiguration(com.yahoo.pulsar.client.api.ClientConfiguration) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) Optional(java.util.Optional) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ChannelOption(io.netty.channel.ChannelOption) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) EpollChannelOption(io.netty.channel.epoll.EpollChannelOption) Lists(com.google.common.collect.Lists) FieldContext(com.yahoo.pulsar.common.configuration.FieldContext) ByteBuf(io.netty.buffer.ByteBuf) EpollServerSocketChannel(io.netty.channel.epoll.EpollServerSocketChannel) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) ConcurrentOpenHashSet(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) EpollEventLoopGroup(io.netty.channel.epoll.EpollEventLoopGroup) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) PersistentReplicator(com.yahoo.pulsar.broker.service.persistent.PersistentReplicator) ClusterReplicationMetrics(com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics) Logger(org.slf4j.Logger) ZooKeeperDataCache(com.yahoo.pulsar.zookeeper.ZooKeeperDataCache) EventLoopGroup(io.netty.channel.EventLoopGroup) KeeperException(org.apache.zookeeper.KeeperException) Semaphore(java.util.concurrent.Semaphore) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) IOException(java.io.IOException) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) Field(java.lang.reflect.Field) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarWebResource(com.yahoo.pulsar.broker.web.PulsarWebResource) Closeable(java.io.Closeable) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationManager(com.yahoo.pulsar.broker.authorization.AuthorizationManager) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) AuthenticationService(com.yahoo.pulsar.broker.authentication.AuthenticationService) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) CompletableFuture(java.util.concurrent.CompletableFuture) Policies(com.yahoo.pulsar.common.policies.data.Policies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig)

Example 3 with PersistencePolicies

use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.

the class Namespaces method setPersistence.

@POST
@Path("/{property}/{cluster}/{namespace}/persistence")
@ApiOperation(value = "Set the persistence configuration for all the destinations on a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") })
public void setPersistence(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, PersistencePolicies persistence) {
    validatePoliciesReadOnlyAccess();
    try {
        Stat nodeStat = new Stat();
        final String path = path("policies", property, cluster, namespace);
        byte[] content = globalZk().getData(path, null, nodeStat);
        Policies policies = jsonMapper().readValue(content, Policies.class);
        policies.persistence = persistence;
        globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
        policiesCache().invalidate(path("policies", property, cluster, namespace));
        log.info("[{}] Successfully updated persistence configuration: namespace={}/{}/{}, map={}", clientAppId(), property, cluster, namespace, jsonMapper().writeValueAsString(policies.persistence));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update persistence configuration for namespace {}/{}/{}: does not exist", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (KeeperException.BadVersionException e) {
        log.warn("[{}] Failed to update persistence configuration for namespace {}/{}/{}: concurrent modification", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.CONFLICT, "Concurrent modification");
    } catch (Exception e) {
        log.error("[{}] Failed to update persistence configuration for namespace {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
}
Also used : Stat(org.apache.zookeeper.data.Stat) Policies(com.yahoo.pulsar.common.policies.data.Policies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) NoNodeException(org.apache.zookeeper.KeeperException.NoNodeException) RestException(com.yahoo.pulsar.broker.web.RestException) KeeperException(org.apache.zookeeper.KeeperException) RestException(com.yahoo.pulsar.broker.web.RestException) WebApplicationException(javax.ws.rs.WebApplicationException) SubscriptionBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) KeeperException(org.apache.zookeeper.KeeperException) PulsarAdminException(com.yahoo.pulsar.client.admin.PulsarAdminException) NoNodeException(org.apache.zookeeper.KeeperException.NoNodeException) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 4 with PersistencePolicies

use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.

the class AdminApiTest method namespaces.

@Test(invocationCount = 1)
public void namespaces() throws PulsarAdminException, PulsarServerException, Exception {
    admin.clusters().createCluster("usw", new ClusterData());
    PropertyAdmin propertyAdmin = new PropertyAdmin(Lists.newArrayList("role1", "role2"), Sets.newHashSet("use", "usw"));
    admin.properties().updateProperty("prop-xyz", propertyAdmin);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1").bundles, Policies.defaultBundle());
    admin.namespaces().createNamespace("prop-xyz/use/ns2");
    admin.namespaces().createNamespace("prop-xyz/use/ns3", 4);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.numBundles, 4);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.boundaries.size(), 5);
    admin.namespaces().deleteNamespace("prop-xyz/use/ns3");
    try {
        admin.namespaces().createNamespace("non-existing/usw/ns1");
        fail("Should not have passed");
    } catch (NotFoundException e) {
    // Ok
    }
    assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
    assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
    try {
        admin.namespaces().createNamespace("prop-xyz/usc/ns1");
        fail("Should not have passed");
    } catch (NotAuthorizedException e) {
    // Ok, got the non authorized exception since usc cluster is not in the allowed clusters list.
    }
    admin.namespaces().grantPermissionOnNamespace("prop-xyz/use/ns1", "my-role", EnumSet.allOf(AuthAction.class));
    Policies policies = new Policies();
    policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class));
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
    assertEquals(admin.namespaces().getPermissions("prop-xyz/use/ns1"), policies.auth_policies.namespace_auth);
    assertEquals(admin.namespaces().getDestinations("prop-xyz/use/ns1"), Lists.newArrayList());
    admin.namespaces().revokePermissionsOnNamespace("prop-xyz/use/ns1", "my-role");
    policies.auth_policies.namespace_auth.remove("my-role");
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
    assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(1, 1, 1, 0.0));
    admin.namespaces().setPersistence("prop-xyz/use/ns1", new PersistencePolicies(3, 2, 1, 10.0));
    assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
    // Force topic creation and namespace being loaded
    Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1/my-topic");
    producer.close();
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns1/my-topic");
    admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1", "0x00000000_0xffffffff");
    NamespaceName ns = new NamespaceName("prop-xyz/use/ns1");
    // Now, w/ bundle policies, we will use default bundle
    NamespaceBundle defaultBundle = bundleFactory.getFullBundle(ns);
    int i = 0;
    for (; i < 10; i++) {
        Optional<NamespaceEphemeralData> data1 = pulsar.getNamespaceService().getOwnershipCache().getOwnerAsync(defaultBundle).get();
        if (!data1.isPresent()) {
            // Already unloaded
            break;
        }
        LOG.info("Waiting for unload namespace {} to complete. Current service unit isDisabled: {}", defaultBundle, data1.get().isDisabled());
        Thread.sleep(1000);
    }
    assertTrue(i < 10);
    admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
    assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns2"));
    try {
        admin.namespaces().unload("prop-xyz/use/ns1");
        fail("should have raised exception");
    } catch (Exception e) {
    // OK excepted
    }
    // Force topic creation and namespace being loaded
    producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns2/my-topic");
    producer.close();
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns2/my-topic");
// both unload and delete should succeed for ns2 on other broker with a redirect
// otheradmin.namespaces().unload("prop-xyz/use/ns2");
}
Also used : NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) Policies(com.yahoo.pulsar.common.policies.data.Policies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) NotFoundException(com.yahoo.pulsar.client.admin.PulsarAdminException.NotFoundException) NotAuthorizedException(com.yahoo.pulsar.client.admin.PulsarAdminException.NotAuthorizedException) NotAuthorizedException(com.yahoo.pulsar.client.admin.PulsarAdminException.NotAuthorizedException) PreconditionFailedException(com.yahoo.pulsar.client.admin.PulsarAdminException.PreconditionFailedException) NotFoundException(com.yahoo.pulsar.client.admin.PulsarAdminException.NotFoundException) PulsarAdminException(com.yahoo.pulsar.client.admin.PulsarAdminException) ConflictException(com.yahoo.pulsar.client.admin.PulsarAdminException.ConflictException) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException) AuthAction(com.yahoo.pulsar.common.policies.data.AuthAction) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) ClusterData(com.yahoo.pulsar.common.policies.data.ClusterData) PropertyAdmin(com.yahoo.pulsar.common.policies.data.PropertyAdmin) Producer(com.yahoo.pulsar.client.api.Producer) NamespaceEphemeralData(com.yahoo.pulsar.broker.namespace.NamespaceEphemeralData) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(com.yahoo.pulsar.broker.auth.MockedPulsarServiceBaseTest)

Example 5 with PersistencePolicies

use of com.yahoo.pulsar.common.policies.data.PersistencePolicies in project pulsar by yahoo.

the class Namespaces method getPersistence.

@GET
@Path("/{property}/{cluster}/{namespace}/persistence")
@ApiOperation(value = "Get the persistence configuration for a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") })
public PersistencePolicies getPersistence(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace) {
    validateAdminAccessOnProperty(property);
    Policies policies = getNamespacePolicies(property, cluster, namespace);
    if (policies.persistence == null) {
        return new PersistencePolicies(config().getManagedLedgerDefaultEnsembleSize(), config().getManagedLedgerDefaultWriteQuorum(), config().getManagedLedgerDefaultAckQuorum(), 0.0d);
    } else {
        return policies.persistence;
    }
}
Also used : Policies(com.yahoo.pulsar.common.policies.data.Policies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Aggregations

PersistencePolicies (com.yahoo.pulsar.common.policies.data.PersistencePolicies)5 RetentionPolicies (com.yahoo.pulsar.common.policies.data.RetentionPolicies)5 Policies (com.yahoo.pulsar.common.policies.data.Policies)4 PulsarServerException (com.yahoo.pulsar.broker.PulsarServerException)2 PulsarAdminException (com.yahoo.pulsar.client.admin.PulsarAdminException)2 NamespaceBundle (com.yahoo.pulsar.common.naming.NamespaceBundle)2 NamespaceName (com.yahoo.pulsar.common.naming.NamespaceName)2 ClusterData (com.yahoo.pulsar.common.policies.data.ClusterData)2 KeeperException (org.apache.zookeeper.KeeperException)2 Stat (org.apache.zookeeper.data.Stat)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)1 Lists (com.google.common.collect.Lists)1 PulsarService (com.yahoo.pulsar.broker.PulsarService)1 ServiceConfiguration (com.yahoo.pulsar.broker.ServiceConfiguration)1 AdminResource (com.yahoo.pulsar.broker.admin.AdminResource)1 MockedPulsarServiceBaseTest (com.yahoo.pulsar.broker.auth.MockedPulsarServiceBaseTest)1 AuthenticationService (com.yahoo.pulsar.broker.authentication.AuthenticationService)1 AuthorizationManager (com.yahoo.pulsar.broker.authorization.AuthorizationManager)1 NamespaceEphemeralData (com.yahoo.pulsar.broker.namespace.NamespaceEphemeralData)1