Examples with KafkaSpoutMessageId org.apache.storm.kafka.spout.KafkaSpoutMessageId used on opensource projects

Example 1 with KafkaSpoutMessageId

use of org.apache.storm.kafka.spout.KafkaSpoutMessageId in project storm by apache.

the class OffsetManager method findNextCommitOffset.

/**
     * An offset is only committed when all records with lower offset have been
     * acked. This guarantees that all offsets smaller than the committedOffset
     * have been delivered.
     *
     * @return the next OffsetAndMetadata to commit, or null if no offset is
     * ready to commit.
     */
public OffsetAndMetadata findNextCommitOffset() {
    boolean found = false;
    long currOffset;
    long nextCommitOffset = committedOffset;
    // this is a convenience variable to make it faster to create OffsetAndMetadata
    KafkaSpoutMessageId nextCommitMsg = null;
    for (KafkaSpoutMessageId currAckedMsg : ackedMsgs) {
        // complexity is that of a linear scan on a TreeMap
        if ((currOffset = currAckedMsg.offset()) == nextCommitOffset + 1) {
            // found the next offset to commit
            found = true;
            nextCommitMsg = currAckedMsg;
            nextCommitOffset = currOffset;
        } else if (currAckedMsg.offset() > nextCommitOffset + 1) {
            // offset found is not continuous to the offsets listed to go in the next commit, so stop search
            LOG.debug("topic-partition [{}] has non-continuous offset [{}]. It will be processed in a subsequent batch.", tp, currOffset);
            break;
        } else {
            //Received a redundant ack. Ignore and continue processing.
            LOG.warn("topic-partition [{}] has unexpected offset [{}]. Current committed Offset [{}]", tp, currOffset, committedOffset);
        }
    }
    OffsetAndMetadata nextCommitOffsetAndMetadata = null;
    if (found) {
        nextCommitOffsetAndMetadata = new OffsetAndMetadata(nextCommitOffset, nextCommitMsg.getMetadata(Thread.currentThread()));
        LOG.debug("topic-partition [{}] has offsets [{}-{}] ready to be committed", tp, committedOffset + 1, nextCommitOffsetAndMetadata.offset());
    } else {
        LOG.debug("topic-partition [{}] has NO offsets ready to be committed", tp);
    }
    LOG.trace("{}", this);
    return nextCommitOffsetAndMetadata;
}