Examples with PersistencePolicies org.apache.pulsar.common.policies.data.PersistencePolicies used on opensource projects

Example 1 with PersistencePolicies

use of org.apache.pulsar.common.policies.data.PersistencePolicies in project incubator-pulsar by apache.

the class PulsarAdminToolTest method namespaces.

@Test
void namespaces() throws Exception {
    PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
    Namespaces mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    Lookup mockLookup = mock(Lookup.class);
    when(admin.lookups()).thenReturn(mockLookup);
    CmdNamespaces namespaces = new CmdNamespaces(admin);
    namespaces.run(split("list myprop"));
    verify(mockNamespaces).getNamespaces("myprop");
    namespaces.run(split("list-cluster myprop/clust"));
    verify(mockNamespaces).getNamespaces("myprop", "clust");
    namespaces.run(split("topics myprop/clust/ns1"));
    verify(mockNamespaces).getTopics("myprop/clust/ns1");
    namespaces.run(split("policies myprop/clust/ns1"));
    verify(mockNamespaces).getPolicies("myprop/clust/ns1");
    namespaces.run(split("create myprop/clust/ns1"));
    verify(mockNamespaces).createNamespace("myprop/clust/ns1");
    namespaces.run(split("delete myprop/clust/ns1"));
    verify(mockNamespaces).deleteNamespace("myprop/clust/ns1");
    namespaces.run(split("permissions myprop/clust/ns1"));
    verify(mockNamespaces).getPermissions("myprop/clust/ns1");
    namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume"));
    verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", EnumSet.of(AuthAction.produce, AuthAction.consume));
    namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1"));
    verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1");
    namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc"));
    verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", Lists.newArrayList("use", "usw", "usc"));
    namespaces.run(split("get-clusters myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1");
    namespaces.run(split("unload myprop/clust/ns1"));
    verify(mockNamespaces).unload("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff"));
    verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff");
    namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff"));
    verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff", false);
    namespaces.run(split("get-backlog-quotas myprop/clust/ns1"));
    verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1");
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10, RetentionPolicy.producer_request_hold));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10l * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
    namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0"));
    verify(mockNamespaces).setPersistence("myprop/clust/ns1", new PersistencePolicies(2, 1, 1, 100.0d));
    namespaces.run(split("get-persistence myprop/clust/ns1"));
    verify(mockNamespaces).getPersistence("myprop/clust/ns1");
    namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300"));
    verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);
    namespaces.run(split("set-deduplication myprop/clust/ns1 --enable"));
    verify(mockNamespaces).setDeduplicationStatus("myprop/clust/ns1", true);
    namespaces.run(split("get-message-ttl myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1");
    namespaces.run(split("set-anti-affinity-group myprop/clust/ns1 -g group"));
    verify(mockNamespaces).setNamespaceAntiAffinityGroup("myprop/clust/ns1", "group");
    namespaces.run(split("get-anti-affinity-group myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceAntiAffinityGroup("myprop/clust/ns1");
    namespaces.run(split("get-anti-affinity-namespaces -p dummy -c cluster -g group"));
    verify(mockNamespaces).getAntiAffinityNamespaces("dummy", "cluster", "group");
    namespaces.run(split("delete-anti-affinity-group myprop/clust/ns1 "));
    verify(mockNamespaces).deleteNamespaceAntiAffinityGroup("myprop/clust/ns1");
    namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M"));
    verify(mockNamespaces).setRetention("myprop/clust/ns1", new RetentionPolicies(60, 1));
    namespaces.run(split("get-retention myprop/clust/ns1"));
    verify(mockNamespaces).getRetention("myprop/clust/ns1");
    namespaces.run(split("clear-backlog myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("get-max-producers-per-topic myprop/clust/ns1"));
    verify(mockNamespaces).getMaxProducersPerTopic("myprop/clust/ns1");
    namespaces.run(split("set-max-producers-per-topic myprop/clust/ns1 -p 1"));
    verify(mockNamespaces).setMaxProducersPerTopic("myprop/clust/ns1", 1);
    namespaces.run(split("get-max-consumers-per-topic myprop/clust/ns1"));
    verify(mockNamespaces).getMaxConsumersPerTopic("myprop/clust/ns1");
    namespaces.run(split("set-max-consumers-per-topic myprop/clust/ns1 -c 2"));
    verify(mockNamespaces).setMaxConsumersPerTopic("myprop/clust/ns1", 2);
    namespaces.run(split("get-max-consumers-per-subscription myprop/clust/ns1"));
    verify(mockNamespaces).getMaxConsumersPerSubscription("myprop/clust/ns1");
    namespaces.run(split("set-max-consumers-per-subscription myprop/clust/ns1 -c 3"));
    verify(mockNamespaces).setMaxConsumersPerSubscription("myprop/clust/ns1", 3);
}

Example 2 with PersistencePolicies

use of org.apache.pulsar.common.policies.data.PersistencePolicies in project incubator-pulsar by apache.

the class AdminApiTest method namespaces.

@Test(invocationCount = 1)
public void namespaces() throws PulsarAdminException, PulsarServerException, Exception {
    admin.clusters().createCluster("usw", new ClusterData());
    PropertyAdmin propertyAdmin = new PropertyAdmin(Lists.newArrayList("role1", "role2"), Sets.newHashSet("use", "usw"));
    admin.properties().updateProperty("prop-xyz", propertyAdmin);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1").bundles, Policies.defaultBundle());
    admin.namespaces().createNamespace("prop-xyz/use/ns2");
    admin.namespaces().createNamespace("prop-xyz/use/ns3", 4);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.numBundles, 4);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.boundaries.size(), 5);
    admin.namespaces().deleteNamespace("prop-xyz/use/ns3");
    try {
        admin.namespaces().createNamespace("non-existing/usw/ns1");
        fail("Should not have passed");
    } catch (NotFoundException e) {
    // Ok
    }
    assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
    assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
    try {
        admin.namespaces().createNamespace("prop-xyz/usc/ns1");
        fail("Should not have passed");
    } catch (NotAuthorizedException e) {
    // Ok, got the non authorized exception since usc cluster is not in the allowed clusters list.
    }
    admin.namespaces().grantPermissionOnNamespace("prop-xyz/use/ns1", "my-role", EnumSet.allOf(AuthAction.class));
    Policies policies = new Policies();
    policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class));
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
    assertEquals(admin.namespaces().getPermissions("prop-xyz/use/ns1"), policies.auth_policies.namespace_auth);
    assertEquals(admin.namespaces().getTopics("prop-xyz/use/ns1"), Lists.newArrayList());
    admin.namespaces().revokePermissionsOnNamespace("prop-xyz/use/ns1", "my-role");
    policies.auth_policies.namespace_auth.remove("my-role");
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
    assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(1, 1, 1, 0.0));
    admin.namespaces().setPersistence("prop-xyz/use/ns1", new PersistencePolicies(3, 2, 1, 10.0));
    assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
    // Force topic creation and namespace being loaded
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1/my-topic").create();
    producer.close();
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns1/my-topic");
    admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1", "0x00000000_0xffffffff");
    NamespaceName ns = NamespaceName.get("prop-xyz/use/ns1");
    // Now, w/ bundle policies, we will use default bundle
    NamespaceBundle defaultBundle = bundleFactory.getFullBundle(ns);
    int i = 0;
    for (; i < 10; i++) {
        Optional<NamespaceEphemeralData> data1 = pulsar.getNamespaceService().getOwnershipCache().getOwnerAsync(defaultBundle).get();
        if (!data1.isPresent()) {
            // Already unloaded
            break;
        }
        LOG.info("Waiting for unload namespace {} to complete. Current service unit isDisabled: {}", defaultBundle, data1.get().isDisabled());
        Thread.sleep(1000);
    }
    assertTrue(i < 10);
    admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
    assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns2"));
    try {
        admin.namespaces().unload("prop-xyz/use/ns1");
        fail("should have raised exception");
    } catch (Exception e) {
    // OK excepted
    }
    // Force topic creation and namespace being loaded
    producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns2/my-topic").create();
    producer.close();
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns2/my-topic");
// both unload and delete should succeed for ns2 on other broker with a redirect
// otheradmin.namespaces().unload("prop-xyz/use/ns2");
}

Example 3 with PersistencePolicies

use of org.apache.pulsar.common.policies.data.PersistencePolicies in project incubator-pulsar by apache.

the class AdminApiTest2 method testUpdatePersistencePolicyUpdateManagedCursor.

/**
 * validates update of persistent-policies reflects on managed-ledger and managed-cursor
 *
 * @throws Exception
 */
@Test
public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
    final String namespace = "prop-xyz/use/ns2";
    final String topicName = "persistent://" + namespace + "/topic1";
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 3, 50.0));
    assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 50.0));
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topic.getManagedLedger();
    ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.getCursors().iterator().next();
    final double newThrottleRate = 100;
    final int newEnsembleSize = 5;
    admin.namespaces().setPersistence(namespace, new PersistencePolicies(newEnsembleSize, 3, 3, newThrottleRate));
    retryStrategically((test) -> managedLedger.getConfig().getEnsembleSize() == newEnsembleSize && cursor.getThrottleMarkDelete() != newThrottleRate, 5, 200);
    // (1) verify cursor.markDelete has been updated
    assertEquals(cursor.getThrottleMarkDelete(), newThrottleRate);
    // (2) verify new ledger creation takes new config
    producer.close();
    consumer.close();
}

Example 4 with PersistencePolicies

use of org.apache.pulsar.common.policies.data.PersistencePolicies in project incubator-pulsar by apache.

the class PersistencePoliciesTest method testPersistencePolicies.

@Test
public void testPersistencePolicies() {
    PersistencePolicies policy0 = new PersistencePolicies();
    PersistencePolicies policy1 = new PersistencePolicies();
    assertTrue(policy0.equals(policy1));
    assertFalse(policy0.equals(new OldPolicies()));
}

Example 5 with PersistencePolicies

use of org.apache.pulsar.common.policies.data.PersistencePolicies in project incubator-pulsar by apache.

the class NamespacesBase method internalSetPersistence.

protected void internalSetPersistence(PersistencePolicies persistence) {
    validatePoliciesReadOnlyAccess();
    validatePersistencePolicies(persistence);
    try {
        Stat nodeStat = new Stat();
        final String path = path(POLICIES, namespaceName.toString());
        byte[] content = globalZk().getData(path, null, nodeStat);
        Policies policies = jsonMapper().readValue(content, Policies.class);
        policies.persistence = persistence;
        globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
        policiesCache().invalidate(path(POLICIES, namespaceName.toString()));
        log.info("[{}] Successfully updated persistence configuration: namespace={}, map={}", clientAppId(), namespaceName, jsonMapper().writeValueAsString(policies.persistence));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update persistence configuration for namespace {}: does not exist", clientAppId(), namespaceName);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (KeeperException.BadVersionException e) {
        log.warn("[{}] Failed to update persistence configuration for namespace {}: concurrent modification", clientAppId(), namespaceName);
        throw new RestException(Status.CONFLICT, "Concurrent modification");
    } catch (Exception e) {
        log.error("[{}] Failed to update persistence configuration for namespace {}", clientAppId(), namespaceName, e);
        throw new RestException(e);
    }
}