Examples with MessageImpl org.apache.pulsar.client.impl.MessageImpl used on opensource projects

Example 1 with MessageImpl

use of org.apache.pulsar.client.impl.MessageImpl in project incubator-pulsar by apache.

the class PersistentReplicator method readEntriesComplete.

@Override
public void readEntriesComplete(List<Entry> entries, Object ctx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}][{} -> {}] Read entries complete of {} messages", topicName, localCluster, remoteCluster, entries.size());
    }
    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] Increasing read batch size from {} to {}", topicName, localCluster, remoteCluster, readBatchSize, newReadBatchSize);
        }
        readBatchSize = newReadBatchSize;
    }
    readFailureBackoff.reduceToHalf();
    boolean atLeastOneMessageSentForReplication = false;
    try {
        // This flag is set to true when we skip atleast one local message,
        // in order to skip remaining local messages.
        boolean isLocalMessageSkippedOnce = false;
        for (int i = 0; i < entries.size(); i++) {
            Entry entry = entries.get(i);
            int length = entry.getLength();
            ByteBuf headersAndPayload = entry.getDataBuffer();
            MessageImpl msg;
            try {
                msg = MessageImpl.deserialize(headersAndPayload);
            } catch (Throwable t) {
                log.error("[{}][{} -> {}] Failed to deserialize message at {} (buffer size: {}): {}", topicName, localCluster, remoteCluster, entry.getPosition(), length, t.getMessage(), t);
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                continue;
            }
            if (msg.isReplicated()) {
                // Discard messages that were already replicated into this region
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }
            if (msg.hasReplicateTo() && !msg.getReplicateTo().contains(remoteCluster)) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Skipping message at position {}, replicateTo {}", topicName, localCluster, remoteCluster, entry.getPosition(), msg.getReplicateTo());
                }
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }
            if (msg.isExpired(messageTTLInSeconds)) {
                msgExpired.recordEvent(0);
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Discarding expired message at position {}, replicateTo {}", topicName, localCluster, remoteCluster, entry.getPosition(), msg.getReplicateTo());
                }
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }
            if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) {
                // recovered when the producer is ready
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Dropping read message at {} because producer is not ready", topicName, localCluster, remoteCluster, entry.getPosition());
                }
                isLocalMessageSkippedOnce = true;
                entry.release();
                msg.recycle();
                continue;
            }
            // Increment pending messages for messages produced locally
            PENDING_MESSAGES_UPDATER.incrementAndGet(this);
            msgOut.recordEvent(headersAndPayload.readableBytes());
            msg.setReplicatedFrom(localCluster);
            headersAndPayload.retain();
            producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
            atLeastOneMessageSentForReplication = true;
        }
    } catch (Exception e) {
        log.error("[{}][{} -> {}] Unexpected exception: {}", topicName, localCluster, remoteCluster, e.getMessage(), e);
    }
    HAVE_PENDING_READ_UPDATER.set(this, FALSE);
    if (atLeastOneMessageSentForReplication && !isWritable()) {
        // Don't read any more entries until the current pending entries are persisted
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] Pausing replication traffic. at-least-one: {} is-writable: {}", topicName, localCluster, remoteCluster, atLeastOneMessageSentForReplication, isWritable());
        }
    } else {
        readMoreEntries();
    }
}

Example 2 with MessageImpl

use of org.apache.pulsar.client.impl.MessageImpl in project incubator-pulsar by apache.

the class PersistentTopic method isOldestMessageExpired.

public boolean isOldestMessageExpired(ManagedCursor cursor, long messageTTLInSeconds) {
    MessageImpl msg = null;
    Entry entry = null;
    boolean isOldestMessageExpired = false;
    try {
        entry = cursor.getNthEntry(1, IndividualDeletedEntries.Include);
        if (entry != null) {
            msg = MessageImpl.deserialize(entry.getDataBuffer());
            isOldestMessageExpired = messageTTLInSeconds != 0 && System.currentTimeMillis() > (msg.getPublishTime() + TimeUnit.SECONDS.toMillis((long) (messageTTLInSeconds * MESSAGE_EXPIRY_THRESHOLD)));
        }
    } catch (Exception e) {
        log.warn("[{}] Error while getting the oldest message", topic, e);
    } finally {
        if (entry != null) {
            entry.release();
        }
        if (msg != null) {
            msg.recycle();
        }
    }
    return isOldestMessageExpired;
}

Example 3 with MessageImpl

use of org.apache.pulsar.client.impl.MessageImpl in project incubator-pulsar by apache.

the class NonPersistentReplicator method sendMessage.

public void sendMessage(Entry entry) {
    if ((STATE_UPDATER.get(this) == State.Started) && isWritable()) {
        int length = entry.getLength();
        ByteBuf headersAndPayload = entry.getDataBuffer();
        MessageImpl msg;
        try {
            msg = MessageImpl.deserialize(headersAndPayload);
        } catch (Throwable t) {
            log.error("[{}][{} -> {}] Failed to deserialize message at {} (buffer size: {}): {}", topicName, localCluster, remoteCluster, entry.getPosition(), length, t.getMessage(), t);
            entry.release();
            return;
        }
        if (msg.isReplicated()) {
            // Discard messages that were already replicated into this region
            entry.release();
            msg.recycle();
            return;
        }
        if (msg.hasReplicateTo() && !msg.getReplicateTo().contains(remoteCluster)) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{} -> {}] Skipping message at {} / msg-id: {}: replicateTo {}", topicName, localCluster, remoteCluster, entry.getPosition(), msg.getMessageId(), msg.getReplicateTo());
            }
            entry.release();
            msg.recycle();
            return;
        }
        msgOut.recordEvent(headersAndPayload.readableBytes());
        msg.setReplicatedFrom(localCluster);
        headersAndPayload.retain();
        producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] dropping message because replicator producer is not started/writable", topicName, localCluster, remoteCluster);
        }
        msgDrop.recordEvent();
        entry.release();
    }
}

Example 4 with MessageImpl

use of org.apache.pulsar.client.impl.MessageImpl in project incubator-pulsar by apache.

the class PersistentMessageExpiryMonitor method expireMessages.

public void expireMessages(int messageTTLInSeconds) {
    if (expirationCheckInProgressUpdater.compareAndSet(this, FALSE, TRUE)) {
        log.info("[{}][{}] Starting message expiry check, ttl= {} seconds", topicName, subName, messageTTLInSeconds);
        cursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, entry -> {
            MessageImpl msg = null;
            try {
                msg = MessageImpl.deserialize(entry.getDataBuffer());
                return msg.isExpired(messageTTLInSeconds);
            } catch (Exception e) {
                log.error("[{}][{}] Error deserializing message for expiry check", topicName, subName, e);
            } finally {
                entry.release();
                if (msg != null) {
                    msg.recycle();
                }
            }
            return false;
        }, this, null);
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}][{}] Ignore expire-message scheduled task, last check is still running", topicName, subName);
        }
    }
}