Search in sources :

Example 11 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testPublishMessage.

@Test
public void testPublishMessage() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    /*
         * MessageMetadata.Builder messageMetadata = MessageMetadata.newBuilder();
         * messageMetadata.setPublishTime(System.currentTimeMillis()); messageMetadata.setProducerName("producer-name");
         * messageMetadata.setSequenceId(1);
         */
    ByteBuf payload = Unpooled.wrappedBuffer("content".getBytes());
    final CountDownLatch latch = new CountDownLatch(1);
    topic.publishMessage(payload, (exception, ledgerId, entryId) -> {
        latch.countDown();
    });
    assertTrue(latch.await(1, TimeUnit.SECONDS));
}
Also used : PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ByteBuf(io.netty.buffer.ByteBuf) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 12 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testDispatcherMultiConsumerReadFailed.

@Test
public void testDispatcherMultiConsumerReadFailed() throws Exception {
    PersistentTopic topic = spy(new PersistentTopic(successTopicName, ledgerMock, brokerService));
    ManagedCursor cursor = mock(ManagedCursor.class);
    when(cursor.getName()).thenReturn("cursor");
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor);
    dispatcher.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
    verify(topic, atLeast(1)).getBrokerService();
}
Also used : ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistentDispatcherMultipleConsumers(com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 13 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testCompression.

@Test(dataProvider = "codec")
public void testCompression(CompressionType compressionType) throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic0" + compressionType;
    // 1. producer connect
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (int i = 0; i < 10; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        assertEquals(msg.getData(), ("my-message-" + i).getBytes());
    }
    // 3. producer disconnect
    producer.close();
    consumer.close();
}
Also used : Producer(com.yahoo.pulsar.client.api.Producer) Consumer(com.yahoo.pulsar.client.api.Consumer) Message(com.yahoo.pulsar.client.api.Message) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Example 14 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testActiveSubscriptionWithCache.

/**
     * Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
     * unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
     * closed
     *
     * @throws Exception
     */
@Test
public void testActiveSubscriptionWithCache() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int recvQueueSize = 4;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    // (1) Create subscription
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    // (2) Produce Messages
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // (3) Get Entry cache
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
    /************* Validation on non-empty active-cursor **************/
    // (4) Get ActiveCursor : which is list of active subscription
    Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
    ManagedCursor curosr = activeCursors.iterator().next();
    // (4.1) Validate: active Cursor must be non-empty
    assertNotNull(curosr);
    // (4.2) Validate: validate cursor name
    assertEquals(subName, curosr.getName());
    // (4.3) Validate: entryCache should have cached messages
    assertTrue(entryCache.getSize() != 0);
    /************* Validation on empty active-cursor **************/
    // (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
    consumer.close();
    Thread.sleep(1000);
    // (5.1) Validate: active-consumer must be empty
    assertFalse(ledger.getActiveCursors().iterator().hasNext());
    // (5.2) Validate: Entry-cache must be cleared
    assertTrue(entryCache.getSize() == 0);
}
Also used : ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) Field(java.lang.reflect.Field) Message(com.yahoo.pulsar.client.api.Message) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ConsumerConfiguration(com.yahoo.pulsar.client.api.ConsumerConfiguration) EntryCacheImpl(org.apache.bookkeeper.mledger.impl.EntryCacheImpl) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 15 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testConcurrentConsumerThreads.

// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
    // test concurrent consumer threads on same consumerId
    final String topicName = "persistent://prop/use/ns-abc/topic3";
    final String subName = "sub3";
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
                for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
                    Message msg = consumer.receive();
                    consumer.acknowledge(msg);
                }
                return null;
            }
        });
    }
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    barrier.await();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    // 1. cumulatively all threads drain the backlog
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 2. flow control works the same as single consumer single thread
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
}
Also used : Message(com.yahoo.pulsar.client.api.Message) PersistentSubscription(com.yahoo.pulsar.broker.service.persistent.PersistentSubscription) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) PulsarAdminException(com.yahoo.pulsar.client.admin.PulsarAdminException) CyclicBarrier(java.util.concurrent.CyclicBarrier) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ConsumerConfiguration(com.yahoo.pulsar.client.api.ConsumerConfiguration) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) Test(org.testng.annotations.Test)

Aggregations

PersistentTopic (com.yahoo.pulsar.broker.service.persistent.PersistentTopic)92 Test (org.testng.annotations.Test)67 Producer (com.yahoo.pulsar.client.api.Producer)35 Consumer (com.yahoo.pulsar.client.api.Consumer)33 Message (com.yahoo.pulsar.client.api.Message)31 PersistentSubscription (com.yahoo.pulsar.broker.service.persistent.PersistentSubscription)29 DestinationName (com.yahoo.pulsar.common.naming.DestinationName)24 ConsumerConfiguration (com.yahoo.pulsar.client.api.ConsumerConfiguration)23 PulsarClientException (com.yahoo.pulsar.client.api.PulsarClientException)21 CompletableFuture (java.util.concurrent.CompletableFuture)17 KeeperException (org.apache.zookeeper.KeeperException)15 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)14 PersistentReplicator (com.yahoo.pulsar.broker.service.persistent.PersistentReplicator)13 IOException (java.io.IOException)13 SubscriptionBusyException (com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException)12 PersistentDispatcherSingleActiveConsumer (com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer)12 RestException (com.yahoo.pulsar.broker.web.RestException)12 PreconditionFailedException (com.yahoo.pulsar.client.admin.PulsarAdminException.PreconditionFailedException)12 ProducerConfiguration (com.yahoo.pulsar.client.api.ProducerConfiguration)12 CountDownLatch (java.util.concurrent.CountDownLatch)12