Examples with RetentionPolicies org.apache.pulsar.common.policies.data.RetentionPolicies used on opensource projects

Example 1 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class PulsarAdminToolTest method namespaces.

@Test
void namespaces() throws Exception {
    PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
    Namespaces mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    Lookup mockLookup = mock(Lookup.class);
    when(admin.lookups()).thenReturn(mockLookup);
    CmdNamespaces namespaces = new CmdNamespaces(admin);
    namespaces.run(split("list myprop"));
    verify(mockNamespaces).getNamespaces("myprop");
    namespaces.run(split("list-cluster myprop/clust"));
    verify(mockNamespaces).getNamespaces("myprop", "clust");
    namespaces.run(split("topics myprop/clust/ns1"));
    verify(mockNamespaces).getTopics("myprop/clust/ns1");
    namespaces.run(split("policies myprop/clust/ns1"));
    verify(mockNamespaces).getPolicies("myprop/clust/ns1");
    namespaces.run(split("create myprop/clust/ns1"));
    verify(mockNamespaces).createNamespace("myprop/clust/ns1");
    namespaces.run(split("delete myprop/clust/ns1"));
    verify(mockNamespaces).deleteNamespace("myprop/clust/ns1");
    namespaces.run(split("permissions myprop/clust/ns1"));
    verify(mockNamespaces).getPermissions("myprop/clust/ns1");
    namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume"));
    verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", EnumSet.of(AuthAction.produce, AuthAction.consume));
    namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1"));
    verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1");
    namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc"));
    verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", Lists.newArrayList("use", "usw", "usc"));
    namespaces.run(split("get-clusters myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1");
    namespaces.run(split("unload myprop/clust/ns1"));
    verify(mockNamespaces).unload("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff"));
    verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff");
    namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff"));
    verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff", false);
    namespaces.run(split("get-backlog-quotas myprop/clust/ns1"));
    verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1");
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10, RetentionPolicy.producer_request_hold));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10l * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
    namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0"));
    verify(mockNamespaces).setPersistence("myprop/clust/ns1", new PersistencePolicies(2, 1, 1, 100.0d));
    namespaces.run(split("get-persistence myprop/clust/ns1"));
    verify(mockNamespaces).getPersistence("myprop/clust/ns1");
    namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300"));
    verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);
    namespaces.run(split("set-deduplication myprop/clust/ns1 --enable"));
    verify(mockNamespaces).setDeduplicationStatus("myprop/clust/ns1", true);
    namespaces.run(split("get-message-ttl myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1");
    namespaces.run(split("set-anti-affinity-group myprop/clust/ns1 -g group"));
    verify(mockNamespaces).setNamespaceAntiAffinityGroup("myprop/clust/ns1", "group");
    namespaces.run(split("get-anti-affinity-group myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceAntiAffinityGroup("myprop/clust/ns1");
    namespaces.run(split("get-anti-affinity-namespaces -p dummy -c cluster -g group"));
    verify(mockNamespaces).getAntiAffinityNamespaces("dummy", "cluster", "group");
    namespaces.run(split("delete-anti-affinity-group myprop/clust/ns1 "));
    verify(mockNamespaces).deleteNamespaceAntiAffinityGroup("myprop/clust/ns1");
    namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M"));
    verify(mockNamespaces).setRetention("myprop/clust/ns1", new RetentionPolicies(60, 1));
    namespaces.run(split("get-retention myprop/clust/ns1"));
    verify(mockNamespaces).getRetention("myprop/clust/ns1");
    namespaces.run(split("clear-backlog myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("get-max-producers-per-topic myprop/clust/ns1"));
    verify(mockNamespaces).getMaxProducersPerTopic("myprop/clust/ns1");
    namespaces.run(split("set-max-producers-per-topic myprop/clust/ns1 -p 1"));
    verify(mockNamespaces).setMaxProducersPerTopic("myprop/clust/ns1", 1);
    namespaces.run(split("get-max-consumers-per-topic myprop/clust/ns1"));
    verify(mockNamespaces).getMaxConsumersPerTopic("myprop/clust/ns1");
    namespaces.run(split("set-max-consumers-per-topic myprop/clust/ns1 -c 2"));
    verify(mockNamespaces).setMaxConsumersPerTopic("myprop/clust/ns1", 2);
    namespaces.run(split("get-max-consumers-per-subscription myprop/clust/ns1"));
    verify(mockNamespaces).getMaxConsumersPerSubscription("myprop/clust/ns1");
    namespaces.run(split("set-max-consumers-per-subscription myprop/clust/ns1 -c 3"));
    verify(mockNamespaces).setMaxConsumersPerSubscription("myprop/clust/ns1", 3);
}

Example 2 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class FunctionMetadataSetup method setupFunctionMetadata.

/**
 * Setup function metadata.
 *
 * @param workerConfig worker config
 * @return dlog uri for store function jars
 * @throws InterruptedException interrupted at setting up metadata
 * @throws PulsarAdminException when encountering exception on pulsar admin operation
 * @throws IOException when create dlog namespace for storing function jars.
 */
public static URI setupFunctionMetadata(WorkerConfig workerConfig) throws InterruptedException, PulsarAdminException, IOException {
    // initializing pulsar functions namespace
    PulsarAdmin admin = Utils.getPulsarAdminClient(workerConfig.getPulsarWebServiceUrl());
    InternalConfigurationData internalConf;
    // make sure pulsar broker is up
    log.info("Checking if pulsar service at {} is up...", workerConfig.getPulsarWebServiceUrl());
    int maxRetries = workerConfig.getInitialBrokerReconnectMaxRetries();
    int retries = 0;
    while (true) {
        try {
            admin.clusters().getClusters();
            break;
        } catch (PulsarAdminException e) {
            log.warn("Failed to retrieve clusters from pulsar service", e);
            log.warn("Retry to connect to Pulsar service at {}", workerConfig.getPulsarWebServiceUrl());
            if (retries >= maxRetries) {
                log.error("Failed to connect to Pulsar service at {} after {} attempts", workerConfig.getPulsarFunctionsNamespace(), maxRetries);
                throw e;
            }
            retries++;
            Thread.sleep(1000);
        }
    }
    // getting namespace policy
    log.info("Initializing Pulsar Functions namespace...");
    try {
        try {
            admin.namespaces().getPolicies(workerConfig.getPulsarFunctionsNamespace());
        } catch (PulsarAdminException e) {
            if (e.getStatusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
                // if not found than create
                try {
                    admin.namespaces().createNamespace(workerConfig.getPulsarFunctionsNamespace());
                } catch (PulsarAdminException e1) {
                    // prevent race condition with other workers starting up
                    if (e1.getStatusCode() != Response.Status.CONFLICT.getStatusCode()) {
                        log.error("Failed to create namespace {} for pulsar functions", workerConfig.getPulsarFunctionsNamespace(), e1);
                        throw e1;
                    }
                }
                try {
                    admin.namespaces().setRetention(workerConfig.getPulsarFunctionsNamespace(), new RetentionPolicies(Integer.MAX_VALUE, Integer.MAX_VALUE));
                } catch (PulsarAdminException e1) {
                    log.error("Failed to set retention policy for pulsar functions namespace", e);
                    throw new RuntimeException(e1);
                }
            } else {
                log.error("Failed to get retention policy for pulsar function namespace {}", workerConfig.getPulsarFunctionsNamespace(), e);
                throw e;
            }
        }
        try {
            internalConf = admin.brokers().getInternalConfigurationData();
        } catch (PulsarAdminException e) {
            log.error("Failed to retrieve broker internal configuration", e);
            throw e;
        }
    } finally {
        admin.close();
    }
    // TODO: move this as part of pulsar cluster initialization later
    try {
        return Utils.initializeDlogNamespace(internalConf.getZookeeperServers(), internalConf.getLedgersRootPath());
    } catch (IOException ioe) {
        log.error("Failed to initialize dlog namespace at zookeeper {} for storing function packages", internalConf.getZookeeperServers(), ioe);
        throw ioe;
    }
}

Example 3 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class NamespacesBase method internalSetRetention.

protected void internalSetRetention(RetentionPolicies retention) {
    validatePoliciesReadOnlyAccess();
    try {
        Stat nodeStat = new Stat();
        final String path = path(POLICIES, namespaceName.toString());
        byte[] content = globalZk().getData(path, null, nodeStat);
        Policies policies = jsonMapper().readValue(content, Policies.class);
        if (!checkQuotas(policies, retention)) {
            log.warn("[{}] Failed to update retention configuration for namespace {}: conflicts with backlog quota", clientAppId(), namespaceName);
            throw new RestException(Status.PRECONDITION_FAILED, "Retention Quota must exceed configured backlog quota for namespace.");
        }
        policies.retention_policies = retention;
        globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
        policiesCache().invalidate(path(POLICIES, namespaceName.toString()));
        log.info("[{}] Successfully updated retention configuration: namespace={}, map={}", clientAppId(), namespaceName, jsonMapper().writeValueAsString(policies.retention_policies));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update retention configuration for namespace {}: does not exist", clientAppId(), namespaceName);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (KeeperException.BadVersionException e) {
        log.warn("[{}] Failed to update retention configuration for namespace {}: concurrent modification", clientAppId(), namespaceName);
        throw new RestException(Status.CONFLICT, "Concurrent modification");
    } catch (RestException pfe) {
        throw pfe;
    } catch (Exception e) {
        log.error("[{}] Failed to update retention configuration for namespace {}", clientAppId(), namespaceName, e);
        throw new RestException(e);
    }
}

Example 4 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class AdminApiTest method persistentTopicsCursorReset.

@Test(dataProvider = "topicName")
public void persistentTopicsCursorReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // create consumer and subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    // Allow at least 1ms for messages to have different timestamps
    Thread.sleep(1);
    long messageTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 5, 5);
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    // messages should still be available due to retention
    admin.persistentTopics().resetCursor(topicName, "my-sub", messageTimestamp);
    int receivedAfterReset = 0;
    for (int i = 4; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(message.getData(), expected.getBytes());
    }
    assertEquals(receivedAfterReset, 6);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}

Example 5 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class AdminApiTest method persistentTopicsCursorResetAfterReset.

@Test(dataProvider = "topicName")
public void persistentTopicsCursorResetAfterReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // create consumer and subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    // Allow at least 1ms for messages to have different timestamps
    Thread.sleep(1);
    long firstTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 3, 5);
    Thread.sleep(1);
    long secondTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 2, 8);
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    messages.forEach(message -> {
        LOG.info("Peeked message: {}", new String(message.getData()));
    });
    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    admin.persistentTopics().resetCursor(topicName, "my-sub", firstTimestamp);
    int receivedAfterReset = 0;
    // Should received messages from 4-9
    for (int i = 4; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 6);
    // Reset at 2nd timestamp
    receivedAfterReset = 0;
    admin.persistentTopics().resetCursor(topicName, "my-sub", secondTimestamp);
    // Should received messages from 7-9
    for (int i = 7; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 3);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}