Examples with Consumer com.yahoo.pulsar.broker.service.Consumer used on opensource projects

Example 11 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicAndSubscriptionDelete.

// @Test
public void testConcurrentTopicAndSubscriptionDelete() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                Thread.sleep(5, 0);
                log.info("deleter outcome is {}", topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                // do subscription delete
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                // Thread.sleep(2,0);
                log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            // assertFalse(ps.delete().isCompletedExceptionally());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 12 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentTopic method updateRates.

public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream destStatsStream, ClusterReplicationMetrics replStats, String namespace) {
    TopicStats topicStats = threadLocalTopicStats.get();
    topicStats.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    destStatsStream.startObject(topic);
    producers.forEach(producer -> {
        producer.updateRates();
        PublisherStats publisherStats = producer.getStats();
        topicStats.aggMsgRateIn += publisherStats.msgRateIn;
        topicStats.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
    });
    // Creating publishers object for backward compatibility
    destStatsStream.startList("publishers");
    destStatsStream.endList();
    // Start replicator stats
    destStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStats.remotePublishersStats.size();
    replicators.forEach((cluster, replicator) -> {
        // Update replicator cursor state
        replicator.updateCursorState();
        // Update replicator stats
        ReplicatorStats rStat = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = topicStats.remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            rStat.msgRateIn = pubStats.msgRateIn;
            rStat.msgThroughputIn = pubStats.msgThroughputIn;
            rStat.inboundConnection = pubStats.address;
            rStat.inboundConnectedSince = pubStats.connectedSince;
        }
        topicStats.aggMsgRateOut += rStat.msgRateOut;
        topicStats.aggMsgThroughputOut += rStat.msgThroughputOut;
        // Populate replicator specific stats here
        destStatsStream.startObject(cluster);
        destStatsStream.writePair("connected", rStat.connected);
        destStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
        destStatsStream.writePair("msgRateIn", rStat.msgRateIn);
        destStatsStream.writePair("msgRateOut", rStat.msgRateOut);
        destStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
        destStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
        destStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
        destStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
        destStatsStream.writePair("inboundConnection", rStat.inboundConnection);
        destStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
        destStatsStream.writePair("outboundConnection", rStat.outboundConnection);
        destStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
        destStatsStream.endObject();
        nsStats.msgReplBacklog += rStat.replicationBacklog;
        if (replStats.isMetricsEnabled()) {
            String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
            ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
            boolean update = false;
            if (replicationMetrics == null) {
                replicationMetrics = ReplicationMetrics.get();
                update = true;
            }
            replicationMetrics.connected += rStat.connected ? 1 : 0;
            replicationMetrics.msgRateOut += rStat.msgRateOut;
            replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
            replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
            if (update) {
                replStats.put(namespaceClusterKey, replicationMetrics);
            }
        }
    });
    // Close replication
    destStatsStream.endObject();
    // Start subscription stats
    destStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        long subUnackedMessages = 0;
        // Start subscription name & consumers
        try {
            destStatsStream.startObject(subscriptionName);
            Object[] consumers = subscription.getConsumers().array();
            nsStats.consumerCount += consumers.length;
            bundleStats.consumerCount += consumers.length;
            destStatsStream.startList("consumers");
            for (Object consumerObj : consumers) {
                Consumer consumer = (Consumer) consumerObj;
                consumer.updateRates();
                ConsumerStats consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                subUnackedMessages += consumerStats.unackedMessages;
                // Populate consumer specific stats here
                destStatsStream.startObject();
                destStatsStream.writePair("address", consumerStats.address);
                destStatsStream.writePair("consumerName", consumerStats.consumerName);
                destStatsStream.writePair("availablePermits", consumerStats.availablePermits);
                destStatsStream.writePair("unackedMessages", consumerStats.unackedMessages);
                destStatsStream.writePair("blockedConsumerOnUnackedMsgs", consumerStats.blockedConsumerOnUnackedMsgs);
                destStatsStream.writePair("connectedSince", consumerStats.connectedSince);
                destStatsStream.writePair("msgRateOut", consumerStats.msgRateOut);
                destStatsStream.writePair("msgThroughputOut", consumerStats.msgThroughputOut);
                destStatsStream.writePair("msgRateRedeliver", consumerStats.msgRateRedeliver);
                destStatsStream.endObject();
            }
            // Close Consumer stats
            destStatsStream.endList();
            // Populate subscription specific stats here
            destStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog());
            destStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            destStatsStream.writePair("msgRateOut", subMsgRateOut);
            destStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            destStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            destStatsStream.writePair("unackedMessages", subUnackedMessages);
            destStatsStream.writePair("type", subscription.getTypeString());
            // Close consumers
            destStatsStream.endObject();
            topicStats.aggMsgRateOut += subMsgRateOut;
            topicStats.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog();
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    destStatsStream.endObject();
    // Remaining dest stats.
    topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
    destStatsStream.writePair("producerCount", producers.size());
    destStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
    destStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
    destStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
    destStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
    destStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
    destStatsStream.writePair("storageSize", ledger.getEstimatedBacklogSize());
    destStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
    nsStats.msgRateIn += topicStats.aggMsgRateIn;
    nsStats.msgRateOut += topicStats.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    nsStats.storageSize += ledger.getEstimatedBacklogSize();
    bundleStats.msgRateIn += topicStats.aggMsgRateIn;
    bundleStats.msgRateOut += topicStats.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
    // Close topic object
    destStatsStream.endObject();
}

Example 13 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentTopic method subscribe.

@Override
public CompletableFuture<Consumer> subscribe(final ServerCnx cnx, String subscriptionName, long consumerId, SubType subType, int priorityLevel, String consumerName) {
    final CompletableFuture<Consumer> future = new CompletableFuture<>();
    if (hasBatchMessagePublished && !cnx.isBatchMessageCompatibleVersion()) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Consumer doesn't support batch-message {}", topic, subscriptionName);
        }
        future.completeExceptionally(new UnsupportedVersionException("Consumer doesn't support batch-message"));
        return future;
    }
    if (subscriptionName.startsWith(replicatorPrefix)) {
        log.warn("[{}] Failed to create subscription for {}", topic, subscriptionName);
        future.completeExceptionally(new NamingException("Subscription with reserved subscription name attempted"));
        return future;
    }
    lock.readLock().lock();
    try {
        if (isFenced) {
            log.warn("[{}] Attempting to subscribe to a fenced topic", topic);
            future.completeExceptionally(new TopicFencedException("Topic is temporarily unavailable"));
            return future;
        }
        USAGE_COUNT_UPDATER.incrementAndGet(this);
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Added consumer -- count: {}", topic, subscriptionName, consumerName, USAGE_COUNT_UPDATER.get(this));
        }
    } finally {
        lock.readLock().unlock();
    }
    ledger.asyncOpenCursor(Codec.encode(subscriptionName), new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{}] Opened cursor for {} {}", topic, subscriptionName, consumerId, consumerName);
            }
            try {
                PersistentSubscription subscription = subscriptions.computeIfAbsent(subscriptionName, name -> new PersistentSubscription(PersistentTopic.this, cursor));
                Consumer consumer = new Consumer(subscription, subType, consumerId, priorityLevel, consumerName, brokerService.pulsar().getConfiguration().getMaxUnackedMessagesPerConsumer(), cnx, cnx.getRole());
                subscription.addConsumer(consumer);
                if (!cnx.isActive()) {
                    consumer.close();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, consumer.consumerName(), USAGE_COUNT_UPDATER.get(PersistentTopic.this));
                    }
                    future.completeExceptionally(new BrokerServiceException("Connection was closed while the opening the cursor "));
                } else {
                    log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId);
                    future.complete(consumer);
                }
            } catch (BrokerServiceException e) {
                if (e instanceof ConsumerBusyException) {
                    log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName);
                } else if (e instanceof SubscriptionBusyException) {
                    log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
                }
                USAGE_COUNT_UPDATER.decrementAndGet(PersistentTopic.this);
                future.completeExceptionally(e);
            }
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            log.warn("[{}] Failed to create subscription for {}", topic, subscriptionName);
            USAGE_COUNT_UPDATER.decrementAndGet(PersistentTopic.this);
            future.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return future;
}

Example 14 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentDispatcherMultipleConsumers method getNextConsumer.

/**
     * <pre>
     * Broker gives more priority while dispatching messages. Here, broker follows descending priorities. (eg:
     * 0=max-priority, 1, 2,..)
     * <p>
     * Broker will first dispatch messages to max priority-level consumers if they
     * have permits, else broker will consider next priority level consumers. 
     * Also on the same priority-level, it selects consumer in round-robin manner. 
     * <p> 
     * If subscription has consumer-A with  priorityLevel 1 and Consumer-B with priorityLevel 2 then broker will dispatch 
     * messages to only consumer-A until it runs out permit and then broker starts dispatching messages to Consumer-B.
     * <p>
     * Consumer PriorityLevel Permits
     * C1       0             2
     * C2       0             1
     * C3       0             1
     * C4       1             2
     * C5       1             1
     * Result of getNextConsumer(): C1, C2, C3, C1, C4, C5, C4
     * </pre>
     * 
     * <pre>
     * <b>Algorithm:</b>
     * 1. sorted-list: consumers stored in sorted-list: max-priority stored first 
     * 2. currentConsumerRoundRobinIndex: it always stores last served consumer-index
     * 3. resultingAvailableConsumerIndex: traversal index. we always start searching availableConsumer from the  
     *    beginning of sorted-list and update resultingAvailableConsumerIndex according searching-traversal
     *    
     * Each time getNextConsumer() is called:<p>
     * 1. It always starts to traverse from the max-priority consumer (first element) from sorted-list
     * 2. Consumers on same priority-level will be treated equally and it tries to pick one of them in round-robin manner
     * 3. If consumer is not available on given priority-level then it will go to the next lower priority-level consumers
     * 4. Optimization: <p>
     *    A. Consumers on same priority-level must be treated equally => dispatch message round-robin to them: 
     *       [if Consumer of resultingAvailableConsumerIndex(current-traversal-index) has the same 
     *       priority-level as consumer of currentConsumerRoundRobinIndex(last-Served-Consumer-Index)] 
     *       <b>Dispatching in Round-Robin:</b> then it means we should do round-robin and skip all the consumers before 
     *          currentConsumerRoundRobinIndex (as they are already served previously)
     *               a. if found: if we found availableConsumer on the same priority-level after currentConsumerRoundRobinIndex 
     *                            then return that consumer and update currentConsumerRoundRobinIndex with that consumer-index
     *               b. else not_found: if we don't find any consumer on that same-priority level after currentConsumerRoundRobinIndex
     *                      - a. check skipped consumers: check skipped consumer (4.A.a) which are on index before than currentConsumerRoundRobinIndex
     *                      - b. next priority-level: if not found in previous step: then it means no consumer available in prior level. So, move to 
     *                           next lower priority level and try to find next-available consumer as per 4.A
     * 
     * </pre>
     * 
     * @return nextAvailableConsumer
     */
public Consumer getNextConsumer() {
    if (consumerList.isEmpty() || IS_CLOSED_UPDATER.get(this) == TRUE) {
        // abort read if no consumers are connected or if disconnect is initiated
        return null;
    }
    if (currentConsumerRoundRobinIndex >= consumerList.size()) {
        currentConsumerRoundRobinIndex = 0;
    }
    // index of resulting consumer which will be returned
    int resultingAvailableConsumerIndex = 0;
    boolean scanFromBeginningIfCurrentConsumerNotAvailable = true;
    int firstConsumerIndexOfCurrentPriorityLevel = -1;
    do {
        int priorityLevel = consumerList.get(currentConsumerRoundRobinIndex).getPriorityLevel() - consumerList.get(resultingAvailableConsumerIndex).getPriorityLevel();
        boolean isSamePriorityLevel = priorityLevel == 0;
        // store first-consumer index with same-priority as currentConsumerRoundRobinIndex
        if (isSamePriorityLevel && firstConsumerIndexOfCurrentPriorityLevel == -1) {
            firstConsumerIndexOfCurrentPriorityLevel = resultingAvailableConsumerIndex;
        }
        // skip already served same-priority-consumer to select consumer in round-robin manner
        resultingAvailableConsumerIndex = (isSamePriorityLevel && currentConsumerRoundRobinIndex > resultingAvailableConsumerIndex) ? currentConsumerRoundRobinIndex : resultingAvailableConsumerIndex;
        // if resultingAvailableConsumerIndex moved ahead of currentConsumerRoundRobinIndex: then we should
        // check skipped consumer which had same priority as currentConsumerRoundRobinIndex consumer
        boolean isLastConsumerBlocked = (currentConsumerRoundRobinIndex == (consumerList.size() - 1) && !isConsumerAvailable(consumerList.get(currentConsumerRoundRobinIndex)));
        boolean shouldScanCurrentLevel = priorityLevel < 0 || /* means moved to next lower-priority-level */
        isLastConsumerBlocked;
        if (shouldScanCurrentLevel && scanFromBeginningIfCurrentConsumerNotAvailable) {
            for (int i = firstConsumerIndexOfCurrentPriorityLevel; i < currentConsumerRoundRobinIndex; i++) {
                Consumer nextConsumer = consumerList.get(i);
                if (isConsumerAvailable(nextConsumer)) {
                    currentConsumerRoundRobinIndex = i + 1;
                    return nextConsumer;
                }
            }
            // now, we have scanned from the beginning: flip the flag to avoid scan again
            scanFromBeginningIfCurrentConsumerNotAvailable = false;
        }
        Consumer nextConsumer = consumerList.get(resultingAvailableConsumerIndex);
        if (isConsumerAvailable(nextConsumer)) {
            currentConsumerRoundRobinIndex = resultingAvailableConsumerIndex + 1;
            return nextConsumer;
        }
        if (++resultingAvailableConsumerIndex >= consumerList.size()) {
            break;
        }
    } while (true);
    // not found unblocked consumer
    return null;
}

Example 15 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentSubscription method resetCursor.

@Override
public CompletableFuture<Void> resetCursor(long timestamp) {
    CompletableFuture<Void> future = new CompletableFuture<>();
    PersistentMessageFinder persistentMessageFinder = new PersistentMessageFinder(topicName, cursor);
    if (log.isDebugEnabled()) {
        log.debug("[{}][{}] Resetting subscription to timestamp {}", topicName, subName, timestamp);
    }
    persistentMessageFinder.findMessages(timestamp, new AsyncCallbacks.FindEntryCallback() {

        @Override
        public void findEntryComplete(Position position, Object ctx) {
            final Position finalPosition;
            if (position == null) {
                // this should not happen ideally unless a reset is requested for a time
                // that spans beyond the retention limits (time/size)
                finalPosition = cursor.getFirstPosition();
                if (finalPosition == null) {
                    log.warn("[{}][{}] Unable to find position for timestamp {}. Unable to reset cursor to first position", topicName, subName, timestamp);
                    future.completeExceptionally(new SubscriptionInvalidCursorPosition("Unable to find position for specified timestamp"));
                    return;
                }
                log.info("[{}][{}] Unable to find position for timestamp {}. Resetting cursor to first position {} in ledger", topicName, subName, timestamp, finalPosition);
            } else {
                finalPosition = position;
            }
            if (!IS_FENCED_UPDATER.compareAndSet(PersistentSubscription.this, FALSE, TRUE)) {
                future.completeExceptionally(new SubscriptionBusyException("Failed to fence subscription"));
                return;
            }
            final CompletableFuture<Void> disconnectFuture;
            if (dispatcher != null && dispatcher.isConsumerConnected()) {
                disconnectFuture = dispatcher.disconnectAllConsumers();
            } else {
                disconnectFuture = CompletableFuture.completedFuture(null);
            }
            disconnectFuture.whenComplete((aVoid, throwable) -> {
                if (throwable != null) {
                    log.error("[{}][{}] Failed to disconnect consumer from subscription", topicName, subName, throwable);
                    IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE);
                    future.completeExceptionally(new SubscriptionBusyException("Failed to disconnect consumers from subscription"));
                    return;
                }
                log.info("[{}][{}] Successfully disconnected consumers from subscription, proceeding with cursor reset", topicName, subName);
                try {
                    cursor.asyncResetCursor(finalPosition, new AsyncCallbacks.ResetCursorCallback() {

                        @Override
                        public void resetComplete(Object ctx) {
                            if (log.isDebugEnabled()) {
                                log.debug("[{}][{}] Successfully reset subscription to timestamp {}", topicName, subName, timestamp);
                            }
                            IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE);
                            future.complete(null);
                        }

                        @Override
                        public void resetFailed(ManagedLedgerException exception, Object ctx) {
                            log.error("[{}][{}] Failed to reset subscription to timestamp {}", topicName, subName, timestamp, exception);
                            IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE);
                            // or should we just ask user to retry one more time?
                            if (exception instanceof InvalidCursorPositionException) {
                                future.completeExceptionally(new SubscriptionInvalidCursorPosition(exception.getMessage()));
                            } else if (exception instanceof ConcurrentFindCursorPositionException) {
                                future.completeExceptionally(new SubscriptionBusyException(exception.getMessage()));
                            } else {
                                future.completeExceptionally(new BrokerServiceException(exception));
                            }
                        }
                    });
                } catch (Exception e) {
                    log.error("[{}][{}] Error while resetting cursor", topicName, subName, e);
                    IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE);
                    future.completeExceptionally(new BrokerServiceException(e));
                }
            });
        }

        @Override
        public void findEntryFailed(ManagedLedgerException exception, Object ctx) {
            // todo - what can go wrong here that needs to be retried?
            if (exception instanceof ConcurrentFindCursorPositionException) {
                future.completeExceptionally(new SubscriptionBusyException(exception.getMessage()));
            } else {
                future.completeExceptionally(new BrokerServiceException(exception));
            }
        }
    });
    return future;
}