Search in sources :

Example 16 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageReplay.

/**
     * Verify: 
     * 1. Broker should not replay already acknowledged messages 
     * 2. Dispatcher should not stuck while dispatching new messages due to previous-replay 
     * of invalid/already-acked messages
     * 
     * @throws Exception
     */
@Test
public void testMessageReplay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(1);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
    replayMap.setAccessible(true);
    TreeSet<PositionImpl> messagesToReplay = Sets.newTreeSet();
    assertNotNull(subRef);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()));
        }
    }
    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);
    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);
    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(new PositionImpl(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId()));
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());
    consumer.close();
    producer.close();
}
Also used : Message(com.yahoo.pulsar.client.api.Message) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) MessageIdImpl(com.yahoo.pulsar.client.impl.MessageIdImpl) PersistentSubscription(com.yahoo.pulsar.broker.service.persistent.PersistentSubscription) Field(java.lang.reflect.Field) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) PersistentDispatcherMultipleConsumers(com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ConsumerConfiguration(com.yahoo.pulsar.client.api.ConsumerConfiguration) Test(org.testng.annotations.Test)

Example 17 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentDispatcherFailoverConsumerTest method testAddRemoveConsumer.

@Test
public void testAddRemoveConsumer() throws Exception {
    log.info("--- Starting PersistentDispatcherFailoverConsumerTest::testAddConsumer ---");
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, cursorMock);
    int partitionIndex = 0;
    PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock, SubType.Failover, partitionIndex, topic);
    // 1. Verify no consumers connected
    assertFalse(pdfc.isConsumerConnected());
    // 2. Add consumer
    Consumer consumer1 = new Consumer(sub, SubType.Exclusive, 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnx, "myrole-1");
    pdfc.addConsumer(consumer1);
    List<Consumer> consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(1, consumers.size());
    // 3. Add again, duplicate allowed
    pdfc.addConsumer(consumer1);
    consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(2, consumers.size());
    // 4. Verify active consumer
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    // 5. Add another consumer which does not change active consumer
    Consumer consumer2 = new Consumer(sub, SubType.Exclusive, 2, /* consumer id */
    0, "Cons2", /* consumer name */
    50000, serverCnx, "myrole-1");
    pdfc.addConsumer(consumer2);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    assertEquals(3, consumers.size());
    // 6. Add a consumer which changes active consumer
    Consumer consumer0 = new Consumer(sub, SubType.Exclusive, 0, /* consumer id */
    0, "Cons0", /* consumer name */
    50000, serverCnx, "myrole-1");
    pdfc.addConsumer(consumer0);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer0.consumerName());
    assertEquals(4, consumers.size());
    // 7. Remove last consumer
    pdfc.removeConsumer(consumer2);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer0.consumerName());
    assertEquals(3, consumers.size());
    // 8. Verify if we can unsubscribe when more than one consumer is connected
    assertFalse(pdfc.canUnsubscribe(consumer0));
    // 9. Remove active consumer
    pdfc.removeConsumer(consumer0);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    assertEquals(2, consumers.size());
    // 10. Attempt to remove already removed consumer
    String cause = "";
    try {
        pdfc.removeConsumer(consumer0);
    } catch (Exception e) {
        cause = e.getMessage();
    }
    assertEquals(cause, "Consumer was not connected");
    // 11. Remove active consumer
    pdfc.removeConsumer(consumer1);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    assertEquals(1, consumers.size());
    // 11. With only one consumer, unsubscribe is allowed
    assertTrue(pdfc.canUnsubscribe(consumer1));
}
Also used : Consumer(com.yahoo.pulsar.broker.service.Consumer) PersistentDispatcherSingleActiveConsumer(com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) PersistentSubscription(com.yahoo.pulsar.broker.service.persistent.PersistentSubscription) PersistentDispatcherSingleActiveConsumer(com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Test(org.testng.annotations.Test)

Example 18 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerConsumer.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerConsumer(CompressionType compressionType) throws Exception {
    int numMsgs = 500;
    int numMsgsInBatch = numMsgs / 20;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer";
    final String subscriptionName = "pc-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        if (i % 2 == 0) {
            consumer.acknowledgeCumulative(msg);
        } else {
            lastunackedMsg = msg;
        }
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) Message(com.yahoo.pulsar.client.api.Message) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Example 19 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BrokerService method addTopicToStatsMaps.

private void addTopicToStatsMaps(DestinationName topicName, PersistentTopic topic) {
    try {
        NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getBundle(topicName);
        if (namespaceBundle != null) {
            synchronized (multiLayerTopicsMap) {
                String serviceUnit = namespaceBundle.toString();
                //
                multiLayerTopicsMap.computeIfAbsent(topicName.getNamespace(), //
                k -> new ConcurrentOpenHashMap<>()).computeIfAbsent(serviceUnit, //
                k -> new ConcurrentOpenHashMap<>()).put(topicName.toString(), topic);
            }
        }
        invalidateOfflineTopicStatCache(topicName);
    } catch (Exception e) {
        log.warn("Got exception when retrieving bundle name during create persistent topic", e);
    }
}
Also used : NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) LoggerFactory(org.slf4j.LoggerFactory) NamespaceBundleStats(com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats) Stat(org.apache.zookeeper.data.Stat) Policies(com.yahoo.pulsar.common.policies.data.Policies) EpollMode(io.netty.channel.epoll.EpollMode) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PersistentOfflineTopicStats(com.yahoo.pulsar.common.policies.data.PersistentOfflineTopicStats) ClusterData(com.yahoo.pulsar.common.policies.data.ClusterData) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Map(java.util.Map) ZooKeeperCacheListener(com.yahoo.pulsar.zookeeper.ZooKeeperCacheListener) PulsarService(com.yahoo.pulsar.broker.PulsarService) FieldParser(com.yahoo.pulsar.common.util.FieldParser) PulsarClientImpl(com.yahoo.pulsar.client.impl.PulsarClientImpl) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) URI(java.net.URI) PulsarClient(com.yahoo.pulsar.client.api.PulsarClient) SystemUtils(org.apache.commons.lang.SystemUtils) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) Set(java.util.Set) NioServerSocketChannel(io.netty.channel.socket.nio.NioServerSocketChannel) PooledByteBufAllocator(io.netty.buffer.PooledByteBufAllocator) InetSocketAddress(java.net.InetSocketAddress) Executors(java.util.concurrent.Executors) NioEventLoopGroup(io.netty.channel.nio.NioEventLoopGroup) ObjectMapperFactory(com.yahoo.pulsar.common.util.ObjectMapperFactory) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Metrics(com.yahoo.pulsar.broker.stats.Metrics) List(java.util.List) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) ClientConfiguration(com.yahoo.pulsar.client.api.ClientConfiguration) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) Optional(java.util.Optional) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ChannelOption(io.netty.channel.ChannelOption) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) EpollChannelOption(io.netty.channel.epoll.EpollChannelOption) Lists(com.google.common.collect.Lists) FieldContext(com.yahoo.pulsar.common.configuration.FieldContext) ByteBuf(io.netty.buffer.ByteBuf) EpollServerSocketChannel(io.netty.channel.epoll.EpollServerSocketChannel) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) ConcurrentOpenHashSet(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) EpollEventLoopGroup(io.netty.channel.epoll.EpollEventLoopGroup) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) PersistentReplicator(com.yahoo.pulsar.broker.service.persistent.PersistentReplicator) ClusterReplicationMetrics(com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics) Logger(org.slf4j.Logger) ZooKeeperDataCache(com.yahoo.pulsar.zookeeper.ZooKeeperDataCache) EventLoopGroup(io.netty.channel.EventLoopGroup) KeeperException(org.apache.zookeeper.KeeperException) Semaphore(java.util.concurrent.Semaphore) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) IOException(java.io.IOException) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) Field(java.lang.reflect.Field) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarWebResource(com.yahoo.pulsar.broker.web.PulsarWebResource) Closeable(java.io.Closeable) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationManager(com.yahoo.pulsar.broker.authorization.AuthorizationManager) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) AuthenticationService(com.yahoo.pulsar.broker.authentication.AuthenticationService) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) KeeperException(org.apache.zookeeper.KeeperException) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) IOException(java.io.IOException) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException)

Example 20 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class Namespaces method clearBacklog.

private void clearBacklog(NamespaceName nsName, String bundleRange, String subscription) {
    try {
        List<PersistentTopic> topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange);
        List<CompletableFuture<Void>> futures = Lists.newArrayList();
        if (subscription != null) {
            if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) {
                subscription = PersistentReplicator.getRemoteCluster(subscription);
            }
            for (PersistentTopic topic : topicList) {
                futures.add(topic.clearBacklog(subscription));
            }
        } else {
            for (PersistentTopic topic : topicList) {
                futures.add(topic.clearBacklog());
            }
        }
        FutureUtil.waitForAll(futures).get();
    } catch (Exception e) {
        log.error("[{}] Failed to clear backlog for namespace {}/{}, subscription: {}", clientAppId(), nsName.toString(), bundleRange, subscription, e);
        throw new RestException(e);
    }
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) RestException(com.yahoo.pulsar.broker.web.RestException) RestException(com.yahoo.pulsar.broker.web.RestException) WebApplicationException(javax.ws.rs.WebApplicationException) SubscriptionBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) KeeperException(org.apache.zookeeper.KeeperException) PulsarAdminException(com.yahoo.pulsar.client.admin.PulsarAdminException) NoNodeException(org.apache.zookeeper.KeeperException.NoNodeException) PulsarServerException(com.yahoo.pulsar.broker.PulsarServerException)

Aggregations

PersistentTopic (com.yahoo.pulsar.broker.service.persistent.PersistentTopic)92 Test (org.testng.annotations.Test)67 Producer (com.yahoo.pulsar.client.api.Producer)35 Consumer (com.yahoo.pulsar.client.api.Consumer)33 Message (com.yahoo.pulsar.client.api.Message)31 PersistentSubscription (com.yahoo.pulsar.broker.service.persistent.PersistentSubscription)29 DestinationName (com.yahoo.pulsar.common.naming.DestinationName)24 ConsumerConfiguration (com.yahoo.pulsar.client.api.ConsumerConfiguration)23 PulsarClientException (com.yahoo.pulsar.client.api.PulsarClientException)21 CompletableFuture (java.util.concurrent.CompletableFuture)17 KeeperException (org.apache.zookeeper.KeeperException)15 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)14 PersistentReplicator (com.yahoo.pulsar.broker.service.persistent.PersistentReplicator)13 IOException (java.io.IOException)13 SubscriptionBusyException (com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException)12 PersistentDispatcherSingleActiveConsumer (com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer)12 RestException (com.yahoo.pulsar.broker.web.RestException)12 PreconditionFailedException (com.yahoo.pulsar.client.admin.PulsarAdminException.PreconditionFailedException)12 ProducerConfiguration (com.yahoo.pulsar.client.api.ProducerConfiguration)12 CountDownLatch (java.util.concurrent.CountDownLatch)12