Examples with RetentionPolicies org.apache.pulsar.common.policies.data.RetentionPolicies used on opensource projects

Example 6 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class AdminApiTest2 method testResetCursorOnPosition.

/**
 * Verifies reset-cursor at specific position using admin-api.
 *
 * <pre>
 * 1. Publish 50 messages
 * 2. Consume 20 messages
 * 3. reset cursor position on 10th message
 * 4. consume 40 messages from reset position
 * </pre>
 *
 * @param namespaceName
 * @throws Exception
 */
@Test(dataProvider = "namespaceNames", timeOut = 10000)
public void testResetCursorOnPosition(String namespaceName) throws Exception {
    final String topicName = "persistent://prop-xyz/use/" + namespaceName + "/resetPosition";
    final int totalProducedMessages = 50;
    // set retention
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    // create consumer and subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Shared).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, totalProducedMessages, 0);
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    Message<byte[]> message = null;
    MessageIdImpl resetMessageId = null;
    int resetPositionId = 10;
    for (int i = 0; i < 20; i++) {
        message = consumer.receive(1, TimeUnit.SECONDS);
        consumer.acknowledge(message);
        if (i == resetPositionId) {
            resetMessageId = (MessageIdImpl) message.getMessageId();
        }
    }
    // close consumer which will clean up intenral-receive-queue
    consumer.close();
    // messages should still be available due to retention
    MessageIdImpl messageId = new MessageIdImpl(resetMessageId.getLedgerId(), resetMessageId.getEntryId(), -1);
    // reset position at resetMessageId
    admin.persistentTopics().resetCursor(topicName, "my-sub", messageId);
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Shared).subscribe();
    MessageIdImpl msgId2 = (MessageIdImpl) consumer.receive(1, TimeUnit.SECONDS).getMessageId();
    assertEquals(resetMessageId, msgId2);
    // start with 1 because we have already received 1 msg
    int receivedAfterReset = 1;
    for (int i = 0; i < totalProducedMessages; i++) {
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
        if (message == null) {
            break;
        }
        consumer.acknowledge(message);
        ++receivedAfterReset;
    }
    assertEquals(receivedAfterReset, totalProducedMessages - resetPositionId);
    // invalid topic name
    try {
        admin.persistentTopics().resetCursor(topicName + "invalid", "my-sub", messageId);
        fail("It should have failed due to invalid topic name");
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    // invalid cursor name
    try {
        admin.persistentTopics().resetCursor(topicName, "invalid-sub", messageId);
        fail("It should have failed due to invalid subscription name");
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    // invalid position
    try {
        messageId = new MessageIdImpl(0, 0, -1);
        admin.persistentTopics().resetCursor(topicName, "my-sub", messageId);
        fail("It should have failed due to invalid subscription name");
    } catch (PulsarAdminException.PreconditionFailedException e) {
    // Ok
    }
    consumer.close();
}

Example 7 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class RetentionPolicesTest method testRetentionPolices.

@Test
public void testRetentionPolices() {
    RetentionPolicies retentionPolicy0 = new RetentionPolicies();
    RetentionPolicies retentionPolicy1 = new RetentionPolicies(1, 100);
    Assert.assertFalse(retentionPolicy0.equals(null));
    Assert.assertTrue(retentionPolicy0.equals(retentionPolicy0));
    Assert.assertFalse(retentionPolicy0.hashCode() == retentionPolicy1.hashCode());
    Assert.assertFalse(retentionPolicy0.toString().equals(retentionPolicy1.toString()));
}

Example 8 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class NamespacesBase method internalSetBacklogQuota.

protected void internalSetBacklogQuota(BacklogQuotaType backlogQuotaType, BacklogQuota backlogQuota) {
    validateAdminAccessOnProperty(namespaceName.getProperty());
    validatePoliciesReadOnlyAccess();
    if (backlogQuotaType == null) {
        backlogQuotaType = BacklogQuotaType.destination_storage;
    }
    try {
        Stat nodeStat = new Stat();
        final String path = path(POLICIES, namespaceName.toString());
        byte[] content = globalZk().getData(path, null, nodeStat);
        Policies policies = jsonMapper().readValue(content, Policies.class);
        RetentionPolicies r = policies.retention_policies;
        if (r != null) {
            Policies p = new Policies();
            p.backlog_quota_map.put(backlogQuotaType, backlogQuota);
            if (!checkQuotas(p, r)) {
                log.warn("[{}] Failed to update backlog configuration for namespace {}: conflicts with retention quota", clientAppId(), namespaceName);
                throw new RestException(Status.PRECONDITION_FAILED, "Backlog Quota exceeds configured retention quota for namespace. Please increase retention quota and retry");
            }
        }
        policies.backlog_quota_map.put(backlogQuotaType, backlogQuota);
        globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
        policiesCache().invalidate(path(POLICIES, namespaceName.toString()));
        log.info("[{}] Successfully updated backlog quota map: namespace={}, map={}", clientAppId(), namespaceName, jsonMapper().writeValueAsString(policies.backlog_quota_map));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update backlog quota map for namespace {}: does not exist", clientAppId(), namespaceName);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (KeeperException.BadVersionException e) {
        log.warn("[{}] Failed to update backlog quota map for namespace {}: concurrent modification", clientAppId(), namespaceName);
        throw new RestException(Status.CONFLICT, "Concurrent modification");
    } catch (RestException pfe) {
        throw pfe;
    } catch (Exception e) {
        log.error("[{}] Failed to update backlog quota map for namespace {}", clientAppId(), namespaceName, e);
        throw new RestException(e);
    }
}

Example 9 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class NamespacesBase method internalGetRetention.

protected RetentionPolicies internalGetRetention() {
    validateAdminAccessOnProperty(namespaceName.getProperty());
    Policies policies = getNamespacePolicies(namespaceName);
    if (policies.retention_policies == null) {
        return new RetentionPolicies(config().getDefaultRetentionTimeInMinutes(), config().getDefaultRetentionSizeInMB());
    } else {
        return policies.retention_policies;
    }
}

Example 10 with RetentionPolicies

use of org.apache.pulsar.common.policies.data.RetentionPolicies in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testInfiniteRetentionPolicy.

/**
 * A topic that has retention policy set to -1, should not be GCed until it has been inactive for at least the
 * retention time and the data should never be deleted
 */
@Test
public void testInfiniteRetentionPolicy() throws Exception {
    // Retain data forever
    admin.namespaces().setRetention("prop/use/ns-abc", new RetentionPolicies(-1, -1));
    // 1. Simple successful GC
    String topicName = "persistent://prop/use/ns-abc/topic-10";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    runGC();
    // Should not have been deleted, since we have retention
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    // Remove retention
    admin.namespaces().setRetention("prop/use/ns-abc", new RetentionPolicies(0, 10));
    Thread.sleep(300);
    // 2. Topic is not GCed with live connection
    String subName = "sub1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();
    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    // 4. Topic can be GCed after unsubscribe
    admin.persistentTopics().deleteSubscription(topicName, subName);
    runGC();
    assertNull(pulsar.getBrokerService().getTopicReference(topicName));
}