Examples with MessageImpl com.yahoo.pulsar.client.impl.MessageImpl used on opensource projects

Example 1 with MessageImpl

use of com.yahoo.pulsar.client.impl.MessageImpl in project pulsar by yahoo.

the class PersistentTopicsImpl method getMessageFromHttpResponse.

private Message getMessageFromHttpResponse(Response response) throws Exception {
    if (response.getStatus() != Status.OK.getStatusCode()) {
        if (response.getStatus() >= 500) {
            throw new ServerErrorException(response);
        } else if (response.getStatus() >= 400) {
            throw new ClientErrorException(response);
        } else {
            throw new WebApplicationException(response);
        }
    }
    String msgId = response.getHeaderString("X-Pulsar-Message-ID");
    InputStream stream = null;
    try {
        stream = (InputStream) response.getEntity();
        byte[] data = new byte[stream.available()];
        stream.read(data);
        Map<String, String> properties = Maps.newTreeMap();
        MultivaluedMap<String, Object> headers = response.getHeaders();
        Object publishTime = headers.getFirst("X-Pulsar-publish-time");
        if (publishTime != null) {
            properties.put("publish-time", (String) publishTime);
        }
        for (Entry<String, List<Object>> entry : headers.entrySet()) {
            String header = entry.getKey();
            if (header.contains("X-Pulsar-PROPERTY-")) {
                String keyName = header.substring(header.indexOf("X-Pulsar-PROPERTY-") + 1, header.length());
                properties.put(keyName, (String) entry.getValue().get(0));
            }
        }
        return new MessageImpl(msgId, properties, data);
    } finally {
        if (stream != null) {
            stream.close();
        }
    }
}

Example 2 with MessageImpl

use of com.yahoo.pulsar.client.impl.MessageImpl in project pulsar by yahoo.

the class PersistentTopic method isOldestMessageExpired.

public boolean isOldestMessageExpired(ManagedCursor cursor, long messageTTLInSeconds) {
    MessageImpl msg = null;
    Entry entry = null;
    boolean isOldestMessageExpired = false;
    try {
        entry = cursor.getNthEntry(1, IndividualDeletedEntries.Include);
        if (entry != null) {
            msg = MessageImpl.deserialize(entry.getDataBuffer());
            isOldestMessageExpired = messageTTLInSeconds != 0 && System.currentTimeMillis() > (msg.getPublishTime() + TimeUnit.SECONDS.toMillis((long) (messageTTLInSeconds * MESSAGE_EXPIRY_THRESHOLD)));
        }
    } catch (Exception e) {
        log.warn("[{}] Error while getting the oldest message", topic, e);
    } finally {
        if (entry != null) {
            entry.release();
        }
        if (msg != null) {
            msg.recycle();
        }
    }
    return isOldestMessageExpired;
}

Example 3 with MessageImpl

use of com.yahoo.pulsar.client.impl.MessageImpl in project pulsar by yahoo.

the class PersistentMessageExpiryMonitor method expireMessages.

public void expireMessages(int messageTTLInSeconds) {
    if (expirationCheckInProgressUpdater.compareAndSet(this, FALSE, TRUE)) {
        log.info("[{}][{}] Starting message expiry check, ttl= {} seconds", topicName, subName, messageTTLInSeconds);
        cursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, entry -> {
            MessageImpl msg = null;
            try {
                msg = MessageImpl.deserialize(entry.getDataBuffer());
                return msg.isExpired(messageTTLInSeconds);
            } catch (Exception e) {
                log.error("[{}][{}] Error deserializing message for expiry check", topicName, subName, e);
            } finally {
                entry.release();
                if (msg != null) {
                    msg.recycle();
                }
            }
            return false;
        }, this, null);
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}][{}] Ignore expire-message scheduled task, last check is still running", topicName, subName);
        }
    }
}

Example 4 with MessageImpl

use of com.yahoo.pulsar.client.impl.MessageImpl in project pulsar by yahoo.

the class PersistentReplicator method readEntriesComplete.

@Override
public void readEntriesComplete(List<Entry> entries, Object ctx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}][{} -> {}] Read entries complete of {} messages", topicName, localCluster, remoteCluster, entries.size());
    }
    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] Increasing read batch size from {} to {}", topicName, localCluster, remoteCluster, readBatchSize, newReadBatchSize);
        }
        readBatchSize = newReadBatchSize;
    }
    readFailureBackoff.reduceToHalf();
    boolean atLeastOneMessageSentForReplication = false;
    try {
        // This flag is set to true when we skip atleast one local message,
        // in order to skip remaining local messages.
        boolean isLocalMessageSkippedOnce = false;
        for (int i = 0; i < entries.size(); i++) {
            Entry entry = entries.get(i);
            int length = entry.getLength();
            ByteBuf headersAndPayload = entry.getDataBuffer();
            MessageImpl msg;
            try {
                msg = MessageImpl.deserialize(headersAndPayload);
            } catch (Throwable t) {
                log.error("[{}][{} -> {}] Failed to deserialize message at {} (buffer size: {}): {}", topicName, localCluster, remoteCluster, entry.getPosition(), length, t.getMessage(), t);
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                continue;
            }
            if (msg.isReplicated()) {
                // Discard messages that were already replicated into this region
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }
            if (msg.hasReplicateTo() && !msg.getReplicateTo().contains(remoteCluster)) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Skipping message at {} / msg-id: {}: replicateTo {}", topicName, localCluster, remoteCluster, entry.getPosition(), msg.getMessageId(), msg.getReplicateTo());
                }
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }
            if (msg.isExpired(messageTTLInSeconds)) {
                msgExpired.recordEvent(0);
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Discarding expired message at {} / msg-id: {}", topicName, localCluster, remoteCluster, entry.getPosition(), msg.getMessageId());
                }
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }
            if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) {
                // recovered when the producer is ready
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Dropping read message at {} because producer is not ready", topicName, localCluster, remoteCluster, entry.getPosition());
                }
                isLocalMessageSkippedOnce = true;
                entry.release();
                msg.recycle();
                continue;
            }
            // Increment pending messages for messages produced locally
            PENDING_MESSAGES_UPDATER.incrementAndGet(this);
            msgOut.recordEvent(headersAndPayload.readableBytes());
            msg.setReplicatedFrom(localCluster);
            headersAndPayload.retain();
            producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
            atLeastOneMessageSentForReplication = true;
        }
    } catch (Exception e) {
        log.error("[{}][{} -> {}] Unexpected exception: {}", topicName, localCluster, remoteCluster, e.getMessage(), e);
    }
    HAVE_PENDING_READ_UPDATER.set(this, FALSE);
    if (atLeastOneMessageSentForReplication && !isWritable()) {
        // Don't read any more entries until the current pending entries are persisted
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] Pausing replication traffic. at-least-one: {} is-writable: {}", topicName, localCluster, remoteCluster, atLeastOneMessageSentForReplication, isWritable());
        }
    } else {
        readMoreEntries();
    }
}