Examples with MessageIdData org.apache.pulsar.common.api.proto.MessageIdData used on opensource projects

Example 16 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class Consumer method individualAckNormal.

// this method is for individual ack not carry the transaction
private CompletableFuture<Void> individualAckNormal(CommandAck ack, Map<String, Long> properties) {
    List<Position> positionsAcked = new ArrayList<>();
    for (int i = 0; i < ack.getMessageIdsCount(); i++) {
        MessageIdData msgId = ack.getMessageIdAt(i);
        PositionImpl position;
        long ackedCount = 0;
        long batchSize = getBatchSize(msgId);
        Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId());
        if (msgId.getAckSetsCount() > 0) {
            long[] ackSets = new long[msgId.getAckSetsCount()];
            for (int j = 0; j < msgId.getAckSetsCount(); j++) {
                ackSets[j] = msgId.getAckSetAt(j);
            }
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets);
            ackedCount = getAckedCountForBatchIndexLevelEnabled(position, batchSize, ackSets);
            if (isTransactionEnabled()) {
                // sync the batch position bit set point, in order to delete the position in pending acks
                if (Subscription.isIndividualAckMode(subType)) {
                    ((PersistentSubscription) subscription).syncBatchPositionBitSetForPendingAck(position);
                }
            }
        } else {
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId());
            ackedCount = getAckedCountForMsgIdNoAckSets(batchSize, position);
        }
        addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount);
        positionsAcked.add(position);
        checkCanRemovePendingAcksAndHandle(position, msgId);
        checkAckValidationError(ack, position);
    }
    subscription.acknowledgeMessage(positionsAcked, AckType.Individual, properties);
    CompletableFuture<Void> completableFuture = new CompletableFuture<>();
    completableFuture.complete(null);
    if (isTransactionEnabled() && Subscription.isIndividualAckMode(subType)) {
        completableFuture.whenComplete((v, e) -> positionsAcked.forEach(position -> {
            // the bit set is empty in pending ack handle.
            if (((PositionImpl) position).getAckSet() != null) {
                if (((PersistentSubscription) subscription).checkIsCanDeleteConsumerPendingAck((PositionImpl) position)) {
                    removePendingAcks((PositionImpl) position);
                }
            }
        }));
    }
    return completableFuture;
}

Example 17 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class Consumer method redeliverUnacknowledgedMessages.

public void redeliverUnacknowledgedMessages(List<MessageIdData> messageIds) {
    int totalRedeliveryMessages = 0;
    List<PositionImpl> pendingPositions = Lists.newArrayList();
    for (MessageIdData msg : messageIds) {
        PositionImpl position = PositionImpl.get(msg.getLedgerId(), msg.getEntryId());
        LongPair longPair = pendingAcks.get(position.getLedgerId(), position.getEntryId());
        if (longPair != null) {
            int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(position, longPair.first);
            pendingAcks.remove(position.getLedgerId(), position.getEntryId());
            totalRedeliveryMessages += unAckedCount;
            pendingPositions.add(position);
        }
    }
    addAndGetUnAckedMsgs(this, -totalRedeliveryMessages);
    blockedConsumerOnUnackedMsgs = false;
    if (log.isDebugEnabled()) {
        log.debug("[{}-{}] consumer {} received {} msg-redelivery {}", topicName, subscription, consumerId, totalRedeliveryMessages, pendingPositions.size());
    }
    subscription.redeliverUnacknowledgedMessages(this, pendingPositions);
    msgRedeliver.recordMultipleEvents(totalRedeliveryMessages, totalRedeliveryMessages);
    int numberOfBlockedPermits = PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.getAndSet(this, 0);
    // if permitsReceivedWhileConsumerBlocked has been accumulated then pass it to Dispatcher to flow messages
    if (numberOfBlockedPermits > 0) {
        MESSAGE_PERMITS_UPDATER.getAndAdd(this, numberOfBlockedPermits);
        if (log.isDebugEnabled()) {
            log.debug("[{}-{}] Added {} blockedPermits to broker.service.Consumer's messagePermits for consumer {}", topicName, subscription, numberOfBlockedPermits, consumerId);
        }
        subscription.consumerFlow(this, numberOfBlockedPermits);
    }
}

Example 18 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class CompactedOutBatchMessageTest method testCompactedOutMessages.

@Test
public void testCompactedOutMessages() throws Exception {
    final String topic1 = "persistent://my-property/my-ns/my-topic";
    BrokerEntryMetadata brokerEntryMetadata = new BrokerEntryMetadata().setBrokerTimestamp(1).setBrokerTimestamp(1);
    MessageMetadata metadata = new MessageMetadata().setProducerName("foobar").setSequenceId(1).setPublishTime(1).setNumMessagesInBatch(3);
    // build a buffer with 4 messages, first and last compacted out
    ByteBuf batchBuffer = Unpooled.buffer(1000);
    Commands.serializeSingleMessageInBatchWithPayload(new SingleMessageMetadata().setCompactedOut(true).setPartitionKey("key1"), Unpooled.EMPTY_BUFFER, batchBuffer);
    Commands.serializeSingleMessageInBatchWithPayload(new SingleMessageMetadata().setCompactedOut(true).setPartitionKey("key2"), Unpooled.EMPTY_BUFFER, batchBuffer);
    Commands.serializeSingleMessageInBatchWithPayload(new SingleMessageMetadata().setCompactedOut(false).setPartitionKey("key3"), Unpooled.EMPTY_BUFFER, batchBuffer);
    Commands.serializeSingleMessageInBatchWithPayload(new SingleMessageMetadata().setCompactedOut(true).setPartitionKey("key4"), Unpooled.EMPTY_BUFFER, batchBuffer);
    try (ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe()) {
        // shove it in the sideways
        consumer.receiveIndividualMessagesFromBatch(brokerEntryMetadata, metadata, 0, null, batchBuffer, new MessageIdData().setLedgerId(1234).setEntryId(567), consumer.cnx(), DEFAULT_CONSUMER_EPOCH);
        Message<?> m = consumer.receive();
        assertEquals(((BatchMessageIdImpl) m.getMessageId()).getLedgerId(), 1234);
        assertEquals(((BatchMessageIdImpl) m.getMessageId()).getEntryId(), 567);
        assertEquals(((BatchMessageIdImpl) m.getMessageId()).getBatchIndex(), 2);
        assertEquals(m.getKey(), "key3");
        assertEquals(consumer.numMessagesInQueue(), 0);
    }
}

Example 19 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class RawMessageSerDeserTest method testSerializationAndDeserialization.

@Test
public void testSerializationAndDeserialization() {
    int payload = 0xbeefcafe;
    ByteBuf headersAndPayload = Unpooled.buffer(4);
    headersAndPayload.writeInt(payload);
    MessageIdData id = new MessageIdData().setLedgerId(0xf00).setEntryId(0xbaa).setPartition(10).setBatchIndex(20);
    @Cleanup RawMessage m = new RawMessageImpl(id, headersAndPayload);
    ByteBuf serialized = m.serialize();
    byte[] bytes = new byte[serialized.readableBytes()];
    serialized.readBytes(bytes);
    RawMessage m2 = RawMessageImpl.deserializeFrom(Unpooled.wrappedBuffer(bytes));
    Assert.assertEquals(m2.getMessageIdData().getLedgerId(), m.getMessageIdData().getLedgerId());
    Assert.assertEquals(m2.getMessageIdData().getEntryId(), m.getMessageIdData().getEntryId());
    Assert.assertEquals(m2.getMessageIdData().getPartition(), m.getMessageIdData().getPartition());
    Assert.assertEquals(m2.getMessageIdData().getBatchIndex(), m.getMessageIdData().getBatchIndex());
    Assert.assertEquals(m2.getHeadersAndPayload(), m.getHeadersAndPayload());
}

Example 20 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class CompactedTopicTest method buildCompactedLedger.

/**
 * Build a compacted ledger, and return the id of the ledger, the position of the different
 * entries in the ledger, and a list of gaps, and the entry which should be returned after the gap.
 */
private Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> buildCompactedLedger(BookKeeper bk, int count) throws Exception {
    LedgerHandle lh = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    List<Pair<MessageIdData, Long>> positions = new ArrayList<>();
    List<Pair<MessageIdData, Long>> idsInGaps = new ArrayList<>();
    AtomicLong ledgerIds = new AtomicLong(10L);
    AtomicLong entryIds = new AtomicLong(0L);
    CompletableFuture.allOf(IntStream.range(0, count).mapToObj((i) -> {
        List<MessageIdData> idsInGap = new ArrayList<>();
        if (r.nextInt(10) == 1) {
            long delta = r.nextInt(10) + 1;
            idsInGap.add(new MessageIdData().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1));
            ledgerIds.addAndGet(delta);
            entryIds.set(0);
        }
        long delta = r.nextInt(5);
        if (delta != 0) {
            idsInGap.add(new MessageIdData().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1));
        }
        MessageIdData id = new MessageIdData().setLedgerId(ledgerIds.get()).setEntryId(entryIds.addAndGet(delta + 1));
        @Cleanup RawMessage m = new RawMessageImpl(id, Unpooled.EMPTY_BUFFER);
        CompletableFuture<Void> f = new CompletableFuture<>();
        ByteBuf buffer = m.serialize();
        lh.asyncAddEntry(buffer, (rc, ledger, eid, ctx) -> {
            if (rc != BKException.Code.OK) {
                f.completeExceptionally(BKException.create(rc));
            } else {
                positions.add(Pair.of(id, eid));
                idsInGap.forEach((gid) -> idsInGaps.add(Pair.of(gid, eid)));
                f.complete(null);
            }
        }, null);
        return f;
    }).toArray(CompletableFuture[]::new)).get();
    lh.close();
    return Triple.of(lh.getId(), positions, idsInGaps);
}