Examples with MessageIdData com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData used on opensource projects

Example 1 with MessageIdData

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData in project pulsar by yahoo.

the class ConsumerImpl method redeliverUnacknowledgedMessages.

@Override
public void redeliverUnacknowledgedMessages(Set<MessageIdImpl> messageIds) {
    if (conf.getSubscriptionType() != SubscriptionType.Shared) {
        // We cannot redeliver single messages if subscription type is not Shared
        redeliverUnacknowledgedMessages();
        return;
    }
    ClientCnx cnx = cnx();
    if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getNumber()) {
        int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds);
        Iterable<List<MessageIdImpl>> batches = Iterables.partition(messageIds, MAX_REDELIVER_UNACKNOWLEDGED);
        MessageIdData.Builder builder = MessageIdData.newBuilder();
        batches.forEach(ids -> {
            List<MessageIdData> messageIdDatas = ids.stream().map(messageId -> {
                batchMessageAckTracker.remove(messageId);
                builder.setPartition(messageId.getPartitionIndex());
                builder.setLedgerId(messageId.getLedgerId());
                builder.setEntryId(messageId.getEntryId());
                return builder.build();
            }).collect(Collectors.toList());
            ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdDatas);
            cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise());
            messageIdDatas.forEach(MessageIdData::recycle);
        });
        if (messagesFromQueue > 0) {
            increaseAvailablePermits(cnx, messagesFromQueue);
        }
        builder.recycle();
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Redeliver unacked messages and increase {} permits", subscription, topic, consumerName, messagesFromQueue);
        }
        return;
    }
    if (cnx == null || (getState() == State.Connecting)) {
        log.warn("[{}] Client Connection needs to be establised for redelivery of unacknowledged messages", this);
    } else {
        log.warn("[{}] Reconnecting the client to redeliver the messages.", this);
        cnx.ctx().close();
    }
}

Example 2 with MessageIdData

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData in project pulsar by yahoo.

the class MessageIdImpl method toByteArray.

// batchIndex is -1 if message is non-batched message and has the batchIndex for a batch message
protected byte[] toByteArray(int batchIndex) {
    MessageIdData.Builder builder = MessageIdData.newBuilder();
    builder.setLedgerId(ledgerId);
    builder.setEntryId(entryId);
    if (partitionIndex >= 0) {
        builder.setPartition(partitionIndex);
    }
    if (batchIndex != -1) {
        builder.setBatchIndex(batchIndex);
    }
    MessageIdData msgId = builder.build();
    int size = msgId.getSerializedSize();
    ByteBuf serialized = Unpooled.buffer(size, size);
    ByteBufCodedOutputStream stream = ByteBufCodedOutputStream.get(serialized);
    try {
        msgId.writeTo(stream);
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }
    msgId.recycle();
    builder.recycle();
    stream.recycle();
    return serialized.array();
}

Example 3 with MessageIdData

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData in project pulsar by yahoo.

the class Commands method newAck.

public static ByteBuf newAck(long consumerId, long ledgerId, long entryId, AckType ackType, ValidationError validationError) {
    CommandAck.Builder ackBuilder = CommandAck.newBuilder();
    ackBuilder.setConsumerId(consumerId);
    ackBuilder.setAckType(ackType);
    MessageIdData.Builder messageIdDataBuilder = MessageIdData.newBuilder();
    messageIdDataBuilder.setLedgerId(ledgerId);
    messageIdDataBuilder.setEntryId(entryId);
    MessageIdData messageIdData = messageIdDataBuilder.build();
    ackBuilder.setMessageId(messageIdData);
    if (validationError != null) {
        ackBuilder.setValidationError(validationError);
    }
    CommandAck ack = ackBuilder.build();
    ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.ACK).setAck(ack));
    ack.recycle();
    ackBuilder.recycle();
    messageIdDataBuilder.recycle();
    messageIdData.recycle();
    return res;
}

Example 4 with MessageIdData

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData in project pulsar by yahoo.

the class Consumer method sendMessages.

/**
     * Dispatch a list of entries to the consumer.
     *
     * @return a promise that can be use to track when all the data has been written into the socket
     */
public Pair<ChannelPromise, Integer> sendMessages(final List<Entry> entries) {
    final ChannelHandlerContext ctx = cnx.ctx();
    final MutablePair<ChannelPromise, Integer> sentMessages = new MutablePair<ChannelPromise, Integer>();
    final ChannelPromise writePromise = ctx.newPromise();
    sentMessages.setLeft(writePromise);
    if (entries.isEmpty()) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] List of messages is empty, triggering write future immediately for consumerId {}", subscription, consumerId);
        }
        writePromise.setSuccess();
        sentMessages.setRight(0);
        return sentMessages;
    }
    try {
        sentMessages.setRight(updatePermitsAndPendingAcks(entries));
    } catch (PulsarServerException pe) {
        log.warn("[{}] [{}] consumer doesn't support batch-message {}", subscription, consumerId, cnx.getRemoteEndpointProtocolVersion());
        subscription.markTopicWithBatchMessagePublished();
        sentMessages.setRight(0);
        // disconnect consumer: it will update dispatcher's availablePermits and resend pendingAck-messages of this
        // consumer to other consumer
        disconnect();
        return sentMessages;
    }
    ctx.channel().eventLoop().execute(() -> {
        for (int i = 0; i < entries.size(); i++) {
            Entry entry = entries.get(i);
            PositionImpl pos = (PositionImpl) entry.getPosition();
            MessageIdData.Builder messageIdBuilder = MessageIdData.newBuilder();
            MessageIdData messageId = messageIdBuilder.setLedgerId(pos.getLedgerId()).setEntryId(pos.getEntryId()).build();
            ByteBuf metadataAndPayload = entry.getDataBuffer();
            // skip checksum by incrementing reader-index if consumer-client doesn't support checksum verification
            if (cnx.getRemoteEndpointProtocolVersion() < ProtocolVersion.v6.getNumber()) {
                readChecksum(metadataAndPayload);
            }
            // stats
            msgOut.recordEvent(metadataAndPayload.readableBytes());
            if (log.isDebugEnabled()) {
                log.debug("[{}] Sending message to consumerId {}, entry id {}", subscription, consumerId, pos.getEntryId());
            }
            // We only want to pass the "real" promise on the last entry written
            ChannelPromise promise = ctx.voidPromise();
            if (i == (entries.size() - 1)) {
                promise = writePromise;
            }
            ctx.write(Commands.newMessage(consumerId, messageId, metadataAndPayload), promise);
            messageId.recycle();
            messageIdBuilder.recycle();
        }
        ctx.flush();
    });
    return sentMessages;
}

Example 5 with MessageIdData

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData in project pulsar by yahoo.

the class Consumer method redeliverUnacknowledgedMessages.

public void redeliverUnacknowledgedMessages(List<MessageIdData> messageIds) {
    int totalRedeliveryMessages = 0;
    List<PositionImpl> pendingPositions = Lists.newArrayList();
    for (MessageIdData msg : messageIds) {
        PositionImpl position = PositionImpl.get(msg.getLedgerId(), msg.getEntryId());
        Integer batchSize = pendingAcks.remove(position);
        if (batchSize != null) {
            totalRedeliveryMessages += batchSize;
            pendingPositions.add(position);
        }
    }
    UNACKED_MESSAGES_UPDATER.addAndGet(this, -totalRedeliveryMessages);
    blockedConsumerOnUnackedMsgs = false;
    subscription.redeliverUnacknowledgedMessages(this, pendingPositions);
    msgRedeliver.recordMultipleEvents(totalRedeliveryMessages, totalRedeliveryMessages);
    int numberOfBlockedPermits = Math.min(totalRedeliveryMessages, PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.get(this));
    // if permitsReceivedWhileConsumerBlocked has been accumulated then pass it to Dispatcher to flow messages
    if (numberOfBlockedPermits > 0) {
        PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.getAndAdd(this, -numberOfBlockedPermits);
        MESSAGE_PERMITS_UPDATER.getAndAdd(this, numberOfBlockedPermits);
        subscription.consumerFlow(this, numberOfBlockedPermits);
    }
}