Examples with SendMessageInfo org.apache.pulsar.broker.service.Consumer.SendMessageInfo used on opensource projects

Example 1 with SendMessageInfo

use of org.apache.pulsar.broker.service.Consumer.SendMessageInfo in project incubator-pulsar by apache.

the class PersistentDispatcherMultipleConsumers method readEntriesComplete.

@Override
public synchronized void readEntriesComplete(List<Entry> entries, Object ctx) {
    ReadType readType = (ReadType) ctx;
    int start = 0;
    int entriesToDispatch = entries.size();
    if (readType == ReadType.Normal) {
        havePendingRead = false;
    } else {
        havePendingReplayRead = false;
    }
    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}] Increasing read batch size from {} to {}", name, readBatchSize, newReadBatchSize);
        }
        readBatchSize = newReadBatchSize;
    }
    readFailureBackoff.reduceToHalf();
    if (shouldRewindBeforeReadingOrReplaying && readType == ReadType.Normal) {
        // All consumers got disconnected before the completion of the read operation
        entries.forEach(Entry::release);
        cursor.rewind();
        shouldRewindBeforeReadingOrReplaying = false;
        readMoreEntries();
        return;
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size());
    }
    long totalMessagesSent = 0;
    long totalBytesSent = 0;
    while (entriesToDispatch > 0 && totalAvailablePermits > 0 && isAtleastOneConsumerAvailable()) {
        Consumer c = getNextConsumer();
        if (c == null) {
            // Do nothing, cursor will be rewind at reconnection
            log.info("[{}] rewind because no available consumer found from total {}", name, consumerList.size());
            entries.subList(start, entries.size()).forEach(Entry::release);
            cursor.rewind();
            return;
        }
        // round-robin dispatch batch size for this consumer
        int messagesForC = Math.min(Math.min(entriesToDispatch, c.getAvailablePermits()), MaxRoundRobinBatchSize);
        if (messagesForC > 0) {
            // remove positions first from replay list first : sendMessages recycles entries
            if (readType == ReadType.Replay) {
                entries.subList(start, start + messagesForC).forEach(entry -> {
                    messagesToReplay.remove(entry.getLedgerId(), entry.getEntryId());
                });
            }
            SendMessageInfo sentMsgInfo = c.sendMessages(entries.subList(start, start + messagesForC));
            long msgSent = sentMsgInfo.getTotalSentMessages();
            start += messagesForC;
            entriesToDispatch -= messagesForC;
            totalAvailablePermits -= msgSent;
            totalMessagesSent += sentMsgInfo.getTotalSentMessages();
            totalBytesSent += sentMsgInfo.getTotalSentMessageBytes();
        }
    }
    // acquire message-dispatch permits for already delivered messages
    if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) {
        topic.getDispatchRateLimiter().tryDispatchPermit(totalMessagesSent, totalBytesSent);
    }
    if (entriesToDispatch > 0) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] No consumers found with available permits, storing {} positions for later replay", name, entries.size() - start);
        }
        entries.subList(start, entries.size()).forEach(entry -> {
            messagesToReplay.add(entry.getLedgerId(), entry.getEntryId());
            entry.release();
        });
    }
    readMoreEntries();
}

Example 2 with SendMessageInfo

use of org.apache.pulsar.broker.service.Consumer.SendMessageInfo in project incubator-pulsar by apache.

the class PersistentDispatcherSingleActiveConsumer method readEntriesComplete.

@Override
public synchronized void readEntriesComplete(final List<Entry> entries, Object obj) {
    Consumer readConsumer = (Consumer) obj;
    if (log.isDebugEnabled()) {
        log.debug("[{}-{}] Got messages: {}", name, readConsumer, entries.size());
    }
    havePendingRead = false;
    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}-{}] Increasing read batch size from {} to {}", name, readConsumer, readBatchSize, newReadBatchSize);
        }
        readBatchSize = newReadBatchSize;
    }
    readFailureBackoff.reduceToHalf();
    Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
    if (currentConsumer == null || readConsumer != currentConsumer) {
        // re-issue the read request for the new consumer
        if (log.isDebugEnabled()) {
            log.debug("[{}] rewind because no available consumer found", name);
        }
        entries.forEach(Entry::release);
        cursor.rewind();
        if (currentConsumer != null) {
            readMoreEntries(currentConsumer);
        }
    } else {
        SendMessageInfo sentMsgInfo = currentConsumer.sendMessages(entries);
        final long totalMessagesSent = sentMsgInfo.getTotalSentMessages();
        final long totalBytesSent = sentMsgInfo.getTotalSentMessageBytes();
        sentMsgInfo.getChannelPromse().addListener(future -> {
            if (future.isSuccess()) {
                // acquire message-dispatch permits for already delivered messages
                if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) {
                    topic.getDispatchRateLimiter().tryDispatchPermit(totalMessagesSent, totalBytesSent);
                }
                // Schedule a new read batch operation only after the previous batch has been written to the socket
                synchronized (PersistentDispatcherSingleActiveConsumer.this) {
                    Consumer newConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
                    if (newConsumer != null && !havePendingRead) {
                        readMoreEntries(newConsumer);
                    } else {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}-{}] Ignoring write future complete. consumerAvailable={} havePendingRead={}", name, newConsumer, newConsumer != null, havePendingRead);
                        }
                    }
                }
            }
        });
    }
}