Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 76 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testReplicatePeekAndSkip.

@Test(timeOut = 30000)
public void testReplicatePeekAndSkip() throws Exception {
    SortedSet<String> testDests = new TreeSet<String>();
    final TopicName dest = TopicName.get("persistent://pulsar/global/ns/peekAndSeekTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(topic.getReplicators().keys().get(0));
    replicator.skipMessages(2);
    CompletableFuture<Entry> result = replicator.peekNthMessage(1);
    Entry entry = result.get(50, TimeUnit.MILLISECONDS);
    assertNull(entry);
    consumer1.close();
}

Example 77 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testResumptionAfterBacklogRelaxed.

/**
 * Issue #199
 *
 * It verifies that: if the remote cluster reaches backlog quota limit, replicator temporarily stops and once the
 * backlog drains it should resume replication.
 *
 * @throws Exception
 */
@Test(timeOut = 60000, enabled = true, priority = -1)
public void testResumptionAfterBacklogRelaxed() throws Exception {
    List<RetentionPolicy> policies = Lists.newArrayList();
    policies.add(RetentionPolicy.producer_exception);
    policies.add(RetentionPolicy.producer_request_hold);
    for (RetentionPolicy policy : policies) {
        // Use 1Mb quota by default
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1 * 1024 * 1024, policy));
        Thread.sleep(200);
        TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns1/%s-%d", policy, System.currentTimeMillis()));
        // Producer on r1
        MessageProducer producer1 = new MessageProducer(url1, dest);
        // Consumer on r2
        MessageConsumer consumer2 = new MessageConsumer(url2, dest);
        // Replicator for r1 -> r2
        PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
        Replicator replicator = topic.getPersistentReplicator("r2");
        // Produce 1 message in r1. This message will be replicated immediately into r2 and it will become part of
        // local backlog
        producer1.produce(1);
        Thread.sleep(500);
        // Restrict backlog quota limit to 1 byte to stop replication
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1, policy));
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        assertEquals(replicator.getStats().replicationBacklog, 0);
        // Next message will not be replicated, because r2 has reached the quota
        producer1.produce(1);
        Thread.sleep(500);
        assertEquals(replicator.getStats().replicationBacklog, 1);
        // Consumer will now drain 1 message and the replication backlog will be cleared
        consumer2.receive(1);
        // Wait until the 2nd message got delivered to consumer
        consumer2.receive(1);
        int retry = 10;
        for (int i = 0; i < retry && replicator.getStats().replicationBacklog > 0; i++) {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
        assertEquals(replicator.getStats().replicationBacklog, 0);
        producer1.close();
        consumer2.close();
    }
}

Example 78 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testReplicatorClearBacklog.

@Test(timeOut = 30000)
public void testReplicatorClearBacklog() throws Exception {
    // This test is to verify that reset cursor fails on global topic
    SortedSet<String> testDests = new TreeSet<String>();
    final TopicName dest = TopicName.get("persistent://pulsar/global/ns/clearBacklogTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = (PersistentReplicator) spy(topic.getReplicators().get(topic.getReplicators().keys().get(0)));
    replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
    replicator.clearBacklog().get();
    Thread.sleep(100);
    // for code-coverage
    replicator.updateRates();
    // for code-coverage
    replicator.expireMessages(1);
    ReplicatorStats status = replicator.getStats();
    assertTrue(status.replicationBacklog == 0);
    consumer1.close();
}

Example 79 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ResendRequestTest method testFailoverInactiveConsumer.

@Test(timeOut = testTimeout)
public void testFailoverInactiveConsumer() throws Exception {
    String key = "testFailoverInactiveConsumer";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1;
    Message<byte[]> message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
    } while (message1 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 5. Consumer 2 asks for a redelivery but the request is ignored
    log.info("Consumer 2 asks for resend");
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    assertEquals(message1, null);
    assertEquals(message2, null);
}

Example 80 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ResendRequestTest method testFailoverSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedNormalTopic() throws Exception {
    String key = "testFailoverSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("consumer-2").subscribe();
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1;
    Message<byte[]> message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    } while (message1 != null || message2 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 5. Consumer 1 asks for resend
    consumer1.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    // 6. Consumer 1 acknowledges a few messages
    receivedConsumer1 = receivedConsumer2 = 0;
    for (int i = 0; i < totalMessages / 2; i++) {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            log.info("Consumer 1 Acknowledged: " + new String(message1.getData()));
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    }
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages / 2);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 7. Consumer 1 close
    consumer1.close();
    // 8. Checking if all messages are received by Consumer 2
    message2 = consumer2.receive();
    int acknowledgedMessages = 0;
    int unAcknowledgedMessages = 0;
    boolean flag = true;
    do {
        if (flag) {
            consumer2.acknowledge(message2);
            acknowledgedMessages += 1;
            log.info("Consumer 2 Acknowledged: " + new String(message2.getData()));
        } else {
            unAcknowledgedMessages += 1;
        }
        flag = !flag;
        log.info("Consumer 2 Received: " + new String(message2.getData()));
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + (unAcknowledgedMessages + acknowledgedMessages));
    log.info("Consumer 2 acknowledges = " + acknowledgedMessages);
    assertEquals(unAcknowledgedMessages + acknowledgedMessages, totalMessages - receivedConsumer1);
    // 9 .Consumer 2 asks for a resend
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    message2 = consumer2.receive();
    receivedConsumer2 = 0;
    do {
        receivedConsumer2 += 1;
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    assertEquals(unAcknowledgedMessages, receivedConsumer2);
}