Examples with Consumer org.apache.pulsar.broker.service.Consumer used on opensource projects

Example 6 with Consumer

use of org.apache.pulsar.broker.service.Consumer in project incubator-pulsar by apache.

the class PersistentTopicConcurrentTest method testConcurrentTopicDeleteAndSubsUnsubscribe.

// @Test
public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), InitialPosition.Latest);
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                Thread.sleep(4, 730);
                log.info("@@@@@@@@ DELETER TH");
                log.info("deleter outcome is " + topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                log.info("&&&&&&&&& UNSUBSCRIBER TH");
                // Thread.sleep(2,0);
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                log.info("unsubscribe result : " + ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 7 with Consumer

use of org.apache.pulsar.broker.service.Consumer in project incubator-pulsar by apache.

the class PersistentTopic method subscribe.

@Override
public CompletableFuture<Consumer> subscribe(final ServerCnx cnx, String subscriptionName, long consumerId, SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageId startMessageId, Map<String, String> metadata, boolean readCompacted, InitialPosition initialPosition) {
    final CompletableFuture<Consumer> future = new CompletableFuture<>();
    if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) {
        future.completeExceptionally(new NotAllowedException("readCompacted only allowed on failover or exclusive subscriptions"));
        return future;
    }
    if (isBlank(subscriptionName)) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Empty subscription name", topic);
        }
        future.completeExceptionally(new NamingException("Empty subscription name"));
        return future;
    }
    if (hasBatchMessagePublished && !cnx.isBatchMessageCompatibleVersion()) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Consumer doesn't support batch-message {}", topic, subscriptionName);
        }
        future.completeExceptionally(new UnsupportedVersionException("Consumer doesn't support batch-message"));
        return future;
    }
    if (subscriptionName.startsWith(replicatorPrefix) || subscriptionName.equals(DEDUPLICATION_CURSOR_NAME)) {
        log.warn("[{}] Failed to create subscription for {}", topic, subscriptionName);
        future.completeExceptionally(new NamingException("Subscription with reserved subscription name attempted"));
        return future;
    }
    lock.readLock().lock();
    try {
        if (isFenced) {
            log.warn("[{}] Attempting to subscribe to a fenced topic", topic);
            future.completeExceptionally(new TopicFencedException("Topic is temporarily unavailable"));
            return future;
        }
        USAGE_COUNT_UPDATER.incrementAndGet(this);
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Added consumer -- count: {}", topic, subscriptionName, consumerName, USAGE_COUNT_UPDATER.get(this));
        }
    } finally {
        lock.readLock().unlock();
    }
    CompletableFuture<? extends Subscription> subscriptionFuture = // 
    isDurable ? // 
    getDurableSubscription(subscriptionName, initialPosition) : getNonDurableSubscription(subscriptionName, startMessageId);
    int maxUnackedMessages = isDurable ? brokerService.pulsar().getConfiguration().getMaxUnackedMessagesPerConsumer() : 0;
    subscriptionFuture.thenAccept(subscription -> {
        try {
            Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, maxUnackedMessages, cnx, cnx.getRole(), metadata, readCompacted, initialPosition);
            subscription.addConsumer(consumer);
            if (!cnx.isActive()) {
                consumer.close();
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, consumer.consumerName(), USAGE_COUNT_UPDATER.get(PersistentTopic.this));
                }
                future.completeExceptionally(new BrokerServiceException("Connection was closed while the opening the cursor "));
            } else {
                log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId);
                future.complete(consumer);
            }
        } catch (BrokerServiceException e) {
            if (e instanceof ConsumerBusyException) {
                log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName);
            } else if (e instanceof SubscriptionBusyException) {
                log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
            }
            USAGE_COUNT_UPDATER.decrementAndGet(PersistentTopic.this);
            future.completeExceptionally(e);
        }
    }).exceptionally(ex -> {
        log.warn("[{}] Failed to create subscription for {}: ", topic, subscriptionName, ex.getMessage());
        USAGE_COUNT_UPDATER.decrementAndGet(PersistentTopic.this);
        future.completeExceptionally(new PersistenceException(ex));
        return null;
    });
    return future;
}

Example 8 with Consumer

use of org.apache.pulsar.broker.service.Consumer in project incubator-pulsar by apache.

the class PersistentTopic method updateRates.

public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace) {
    TopicStats topicStats = threadLocalTopicStats.get();
    topicStats.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    topicStatsStream.startObject(topic);
    producers.forEach(producer -> {
        producer.updateRates();
        PublisherStats publisherStats = producer.getStats();
        topicStats.aggMsgRateIn += publisherStats.msgRateIn;
        topicStats.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
    });
    // Creating publishers object for backward compatibility
    topicStatsStream.startList("publishers");
    topicStatsStream.endList();
    // Start replicator stats
    topicStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStats.remotePublishersStats.size();
    replicators.forEach((cluster, replicator) -> {
        // Update replicator cursor state
        ((PersistentReplicator) replicator).updateCursorState();
        // Update replicator stats
        ReplicatorStats rStat = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = topicStats.remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            rStat.msgRateIn = pubStats.msgRateIn;
            rStat.msgThroughputIn = pubStats.msgThroughputIn;
            rStat.inboundConnection = pubStats.address;
            rStat.inboundConnectedSince = pubStats.connectedSince;
        }
        topicStats.aggMsgRateOut += rStat.msgRateOut;
        topicStats.aggMsgThroughputOut += rStat.msgThroughputOut;
        // Populate replicator specific stats here
        topicStatsStream.startObject(cluster);
        topicStatsStream.writePair("connected", rStat.connected);
        topicStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
        topicStatsStream.writePair("msgRateIn", rStat.msgRateIn);
        topicStatsStream.writePair("msgRateOut", rStat.msgRateOut);
        topicStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
        topicStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
        topicStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
        topicStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
        topicStatsStream.writePair("inboundConnection", rStat.inboundConnection);
        topicStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
        topicStatsStream.writePair("outboundConnection", rStat.outboundConnection);
        topicStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
        topicStatsStream.endObject();
        nsStats.msgReplBacklog += rStat.replicationBacklog;
        if (replStats.isMetricsEnabled()) {
            String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
            ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
            boolean update = false;
            if (replicationMetrics == null) {
                replicationMetrics = ReplicationMetrics.get();
                update = true;
            }
            replicationMetrics.connected += rStat.connected ? 1 : 0;
            replicationMetrics.msgRateOut += rStat.msgRateOut;
            replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
            replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
            if (update) {
                replStats.put(namespaceClusterKey, replicationMetrics);
            }
        }
    });
    // Close replication
    topicStatsStream.endObject();
    // Start subscription stats
    topicStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        // Start subscription name & consumers
        try {
            topicStatsStream.startObject(subscriptionName);
            Object[] consumers = subscription.getConsumers().array();
            nsStats.consumerCount += consumers.length;
            bundleStats.consumerCount += consumers.length;
            topicStatsStream.startList("consumers");
            for (Object consumerObj : consumers) {
                Consumer consumer = (Consumer) consumerObj;
                consumer.updateRates();
                ConsumerStats consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                // Populate consumer specific stats here
                topicStatsStream.startObject();
                topicStatsStream.writePair("address", consumerStats.address);
                topicStatsStream.writePair("consumerName", consumerStats.consumerName);
                topicStatsStream.writePair("availablePermits", consumerStats.availablePermits);
                topicStatsStream.writePair("connectedSince", consumerStats.connectedSince);
                topicStatsStream.writePair("msgRateOut", consumerStats.msgRateOut);
                topicStatsStream.writePair("msgThroughputOut", consumerStats.msgThroughputOut);
                topicStatsStream.writePair("msgRateRedeliver", consumerStats.msgRateRedeliver);
                if (SubType.Shared.equals(subscription.getType())) {
                    topicStatsStream.writePair("unackedMessages", consumerStats.unackedMessages);
                    topicStatsStream.writePair("blockedConsumerOnUnackedMsgs", consumerStats.blockedConsumerOnUnackedMsgs);
                }
                if (consumerStats.clientVersion != null) {
                    topicStatsStream.writePair("clientVersion", consumerStats.clientVersion);
                }
                topicStatsStream.endObject();
            }
            // Close Consumer stats
            topicStatsStream.endList();
            // Populate subscription specific stats here
            topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog());
            topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            topicStatsStream.writePair("msgRateOut", subMsgRateOut);
            topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            topicStatsStream.writePair("numberOfEntriesSinceFirstNotAckedMessage", subscription.getNumberOfEntriesSinceFirstNotAckedMessage());
            topicStatsStream.writePair("totalNonContiguousDeletedMessagesRange", subscription.getTotalNonContiguousDeletedMessagesRange());
            topicStatsStream.writePair("type", subscription.getTypeString());
            if (SubType.Shared.equals(subscription.getType())) {
                if (subscription.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subscription.getDispatcher();
                    topicStatsStream.writePair("blockedSubscriptionOnUnackedMsgs", dispatcher.isBlockedDispatcherOnUnackedMsgs());
                    topicStatsStream.writePair("unackedMessages", dispatcher.getTotalUnackedMessages());
                }
            }
            // Close consumers
            topicStatsStream.endObject();
            topicStats.aggMsgRateOut += subMsgRateOut;
            topicStats.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog();
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    topicStatsStream.endObject();
    // Remaining dest stats.
    topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
    topicStatsStream.writePair("producerCount", producers.size());
    topicStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
    topicStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
    topicStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
    topicStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
    topicStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
    topicStatsStream.writePair("storageSize", ledger.getEstimatedBacklogSize());
    topicStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
    nsStats.msgRateIn += topicStats.aggMsgRateIn;
    nsStats.msgRateOut += topicStats.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    nsStats.storageSize += ledger.getEstimatedBacklogSize();
    bundleStats.msgRateIn += topicStats.aggMsgRateIn;
    bundleStats.msgRateOut += topicStats.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
    // Close topic object
    topicStatsStream.endObject();
}

Example 9 with Consumer

use of org.apache.pulsar.broker.service.Consumer in project incubator-pulsar by apache.

the class NonPersistentTopic method publishMessage.

@Override
public void publishMessage(ByteBuf data, PublishContext callback) {
    AtomicInteger msgDeliveryCount = new AtomicInteger(2);
    ENTRIES_ADDED_COUNTER_UPDATER.incrementAndGet(this);
    // retain data for sub/replication because io-thread will release actual payload
    data.retain(2);
    this.executor.submitOrdered(topic, SafeRun.safeRun(() -> {
        subscriptions.forEach((name, subscription) -> {
            ByteBuf duplicateBuffer = data.retainedDuplicate();
            Entry entry = create(0L, 0L, duplicateBuffer);
            // entry internally retains data so, duplicateBuffer should be release here
            duplicateBuffer.release();
            if (subscription.getDispatcher() != null) {
                subscription.getDispatcher().sendMessages(Lists.newArrayList(entry));
            } else {
                // it happens when subscription is created but dispatcher is not created as consumer is not added
                // yet
                entry.release();
            }
        });
        data.release();
        if (msgDeliveryCount.decrementAndGet() == 0) {
            callback.completed(null, 0L, 0L);
        }
    }));
    this.executor.submitOrdered(topic, SafeRun.safeRun(() -> {
        replicators.forEach((name, replicator) -> {
            ByteBuf duplicateBuffer = data.retainedDuplicate();
            Entry entry = create(0L, 0L, duplicateBuffer);
            // entry internally retains data so, duplicateBuffer should be release here
            duplicateBuffer.release();
            ((NonPersistentReplicator) replicator).sendMessage(entry);
        });
        data.release();
        if (msgDeliveryCount.decrementAndGet() == 0) {
            callback.completed(null, 0L, 0L);
        }
    }));
}

Example 10 with Consumer

use of org.apache.pulsar.broker.service.Consumer in project incubator-pulsar by apache.

the class NonPersistentTopic method updateRates.

public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace) {
    TopicStats topicStats = threadLocalTopicStats.get();
    topicStats.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    topicStatsStream.startObject(topic);
    producers.forEach(producer -> {
        producer.updateRates();
        PublisherStats PublisherStats = producer.getStats();
        topicStats.aggMsgRateIn += PublisherStats.msgRateIn;
        topicStats.aggMsgThroughputIn += PublisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStats.remotePublishersStats.put(producer.getRemoteCluster(), PublisherStats);
        }
    });
    // Creating publishers object for backward compatibility
    topicStatsStream.startList("publishers");
    topicStatsStream.endList();
    // Start replicator stats
    topicStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStats.remotePublishersStats.size();
    // Close replication
    topicStatsStream.endObject();
    // Start subscription stats
    topicStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        // Start subscription name & consumers
        try {
            topicStatsStream.startObject(subscriptionName);
            Object[] consumers = subscription.getConsumers().array();
            nsStats.consumerCount += consumers.length;
            bundleStats.consumerCount += consumers.length;
            topicStatsStream.startList("consumers");
            subscription.getDispatcher().getMesssageDropRate().calculateRate();
            for (Object consumerObj : consumers) {
                Consumer consumer = (Consumer) consumerObj;
                consumer.updateRates();
                ConsumerStats consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                // Populate consumer specific stats here
                topicStatsStream.startObject();
                topicStatsStream.writePair("address", consumerStats.address);
                topicStatsStream.writePair("consumerName", consumerStats.consumerName);
                topicStatsStream.writePair("availablePermits", consumerStats.availablePermits);
                topicStatsStream.writePair("connectedSince", consumerStats.connectedSince);
                topicStatsStream.writePair("msgRateOut", consumerStats.msgRateOut);
                topicStatsStream.writePair("msgThroughputOut", consumerStats.msgThroughputOut);
                topicStatsStream.writePair("msgRateRedeliver", consumerStats.msgRateRedeliver);
                if (SubType.Shared.equals(subscription.getType())) {
                    topicStatsStream.writePair("unackedMessages", consumerStats.unackedMessages);
                    topicStatsStream.writePair("blockedConsumerOnUnackedMsgs", consumerStats.blockedConsumerOnUnackedMsgs);
                }
                if (consumerStats.clientVersion != null) {
                    topicStatsStream.writePair("clientVersion", consumerStats.clientVersion);
                }
                topicStatsStream.endObject();
            }
            // Close Consumer stats
            topicStatsStream.endList();
            // Populate subscription specific stats here
            topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog());
            topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            topicStatsStream.writePair("msgRateOut", subMsgRateOut);
            topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            topicStatsStream.writePair("type", subscription.getTypeString());
            if (subscription.getDispatcher() != null) {
                topicStatsStream.writePair("msgDropRate", subscription.getDispatcher().getMesssageDropRate().getRate());
            }
            // Close consumers
            topicStatsStream.endObject();
            topicStats.aggMsgRateOut += subMsgRateOut;
            topicStats.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog();
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    topicStatsStream.endObject();
    // Remaining dest stats.
    topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
    topicStatsStream.writePair("producerCount", producers.size());
    topicStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
    topicStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
    topicStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
    topicStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
    topicStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
    nsStats.msgRateIn += topicStats.aggMsgRateIn;
    nsStats.msgRateOut += topicStats.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    bundleStats.msgRateIn += topicStats.aggMsgRateIn;
    bundleStats.msgRateOut += topicStats.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    // Close topic object
    topicStatsStream.endObject();
}