Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 21 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testCloseReplicatorStartProducer.

/**
 * It verifies that PersistentReplicator considers CursorAlreadyClosedException as non-retriable-read exception and
 * it should closed the producer as cursor is already closed because replicator is already deleted.
 *
 * @throws Exception
 */
@Test(timeOut = 15000)
public void testCloseReplicatorStartProducer() throws Exception {
    TopicName dest = TopicName.get("persistent://pulsar/global/ns1/closeCursor");
    // Producer on r1
    MessageProducer producer1 = new MessageProducer(url1, dest);
    // Consumer on r1
    MessageConsumer consumer1 = new MessageConsumer(url1, dest);
    // Consumer on r2
    MessageConsumer consumer2 = new MessageConsumer(url2, dest);
    // Replicator for r1 -> r2
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = (PersistentReplicator) topic.getPersistentReplicator("r2");
    // close the cursor
    Field cursorField = PersistentReplicator.class.getDeclaredField("cursor");
    cursorField.setAccessible(true);
    ManagedCursor cursor = (ManagedCursor) cursorField.get(replicator);
    cursor.close();
    // try to read entries
    CountDownLatch latch = new CountDownLatch(1);
    producer1.produce(10);
    cursor.asyncReadEntriesOrWait(10, new ReadEntriesCallback() {

        @Override
        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            latch.countDown();
            fail("it should have been failed");
        }

        @Override
        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
            assertTrue(exception instanceof CursorAlreadyClosedException);
        }
    }, null);
    // replicator-readException: cursorAlreadyClosed
    replicator.readEntriesFailed(new CursorAlreadyClosedException("Cursor already closed exception"), null);
    // wait replicator producer to be closed
    Thread.sleep(1000);
    // Replicator producer must be closed
    Field producerField = AbstractReplicator.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    @SuppressWarnings("unchecked") ProducerImpl<byte[]> replicatorProducer = (ProducerImpl<byte[]>) producerField.get(replicator);
    assertEquals(replicatorProducer, null);
    producer1.close();
    consumer1.close();
    consumer2.close();
}

Example 22 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testConcurrentReplicator.

@SuppressWarnings("unchecked")
@Test(timeOut = 30000)
public void testConcurrentReplicator() throws Exception {
    log.info("--- Starting ReplicatorTest::testConcurrentReplicator ---");
    final String namespace = "pulsar/global/concurrent";
    admin1.namespaces().createNamespace(namespace);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("r1", "r2"));
    final TopicName topicName = TopicName.get(String.format("persistent://" + namespace + "/topic-%d", 0));
    PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Producer<byte[]> producer = client1.newProducer().topic(topicName.toString()).create();
    producer.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName.toString()).get();
    PulsarClientImpl pulsarClient = spy((PulsarClientImpl) pulsar1.getBrokerService().getReplicationClient("r3"));
    final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class);
    startRepl.setAccessible(true);
    Field replClientField = BrokerService.class.getDeclaredField("replicationClients");
    replClientField.setAccessible(true);
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients = (ConcurrentOpenHashMap<String, PulsarClient>) replClientField.get(pulsar1.getBrokerService());
    replicationClients.put("r3", pulsarClient);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("r1", "r2", "r3"));
    ExecutorService executor = Executors.newFixedThreadPool(5);
    for (int i = 0; i < 5; i++) {
        executor.submit(() -> {
            try {
                startRepl.invoke(topic, "r3");
            } catch (Exception e) {
                fail("setting replicator failed", e);
            }
        });
    }
    Thread.sleep(3000);
    Mockito.verify(pulsarClient, Mockito.times(1)).createProducerAsync(Mockito.any(ProducerConfigurationData.class), Mockito.any(Schema.class));
    client1.shutdown();
}

Example 23 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testDeleteReplicatorFailure.

/**
 * It verifies that: if it fails while removing replicator-cluster-cursor: it should not restart the replicator and
 * it should have cleaned up from the list
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testDeleteReplicatorFailure() throws Exception {
    log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
    final String topicName = "persistent://pulsar/global/ns/repltopicbatch";
    final TopicName dest = TopicName.get(topicName);
    MessageProducer producer1 = new MessageProducer(url1, dest);
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName);
    final String replicatorClusterName = topic.getReplicators().keys().get(0);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topic.getManagedLedger();
    CountDownLatch latch = new CountDownLatch(1);
    // delete cursor already : so next time if topic.removeReplicator will get exception but then it should
    // remove-replicator from the list even with failure
    ledger.asyncDeleteCursor("pulsar.repl." + replicatorClusterName, new DeleteCursorCallback() {

        @Override
        public void deleteCursorComplete(Object ctx) {
            latch.countDown();
        }

        @Override
        public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
        }
    }, null);
    latch.await();
    Method removeReplicator = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeReplicator.setAccessible(true);
    // invoke removeReplicator : it fails as cursor is not present: but still it should remove the replicator from
    // list without restarting it
    @SuppressWarnings("unchecked") CompletableFuture<Void> result = (CompletableFuture<Void>) removeReplicator.invoke(topic, replicatorClusterName);
    result.thenApply((v) -> {
        assertNull(topic.getPersistentReplicator(replicatorClusterName));
        return null;
    });
    producer1.close();
}

Example 24 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ReplicatorTest method testReplicatorProducerClosing.

@SuppressWarnings("unchecked")
@Test(priority = 5, timeOut = 30000)
public void testReplicatorProducerClosing() throws Exception {
    log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
    final String topicName = "persistent://pulsar/global/ns/repltopicbatch";
    final TopicName dest = TopicName.get(topicName);
    MessageProducer producer1 = new MessageProducer(url1, dest);
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName);
    final String replicatorClusterName = topic.getReplicators().keys().get(0);
    Replicator replicator = topic.getPersistentReplicator(replicatorClusterName);
    pulsar2.close();
    pulsar3.close();
    replicator.disconnect(false);
    Thread.sleep(100);
    Field field = AbstractReplicator.class.getDeclaredField("producer");
    field.setAccessible(true);
    ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) field.get(replicator);
    assertNull(producer);
    producer1.close();
}

Example 25 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ResendRequestTest method testSharedSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedNormalTopic() throws Exception {
    String key = "testSharedSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(totalMessages / 2).subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
        message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Send a resend request from Consumer 1
    log.info("Consumer 1 sent a resend request");
    consumer1.redeliverUnacknowledgedMessages();
    // 6. Consumer 1's unAcked messages should be sent to Consumer 1 or 2
    int receivedMessagesAfterRedelivery = 0;
    receivedConsumer1 = 0;
    message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            ++receivedMessagesAfterRedelivery;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            ++receivedMessagesAfterRedelivery;
        }
        message1 = consumer1.receive(200, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(200, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Additional received = " + receivedMessagesAfterRedelivery);
    assertTrue(receivedMessagesAfterRedelivery > 0);
    assertEquals(receivedConsumer1 + receivedConsumer2, totalMessages);
}