Examples with MessageIdData org.apache.pulsar.common.api.proto.MessageIdData used on opensource projects

Example 1 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class Consumer method messageAcked.

public CompletableFuture<Void> messageAcked(CommandAck ack) {
    this.lastAckedTimestamp = System.currentTimeMillis();
    Map<String, Long> properties = Collections.emptyMap();
    if (ack.getPropertiesCount() > 0) {
        properties = ack.getPropertiesList().stream().collect(Collectors.toMap(KeyLongValue::getKey, KeyLongValue::getValue));
    }
    if (ack.getAckType() == AckType.Cumulative) {
        if (ack.getMessageIdsCount() != 1) {
            log.warn("[{}] [{}] Received multi-message ack", subscription, consumerId);
        }
        if (Subscription.isIndividualAckMode(subType)) {
            log.warn("[{}] [{}] Received cumulative ack on shared subscription, ignoring", subscription, consumerId);
        }
        PositionImpl position = PositionImpl.EARLIEST;
        if (ack.getMessageIdsCount() == 1) {
            MessageIdData msgId = ack.getMessageIdAt(0);
            if (msgId.getAckSetsCount() > 0) {
                long[] ackSets = new long[msgId.getAckSetsCount()];
                for (int j = 0; j < msgId.getAckSetsCount(); j++) {
                    ackSets[j] = msgId.getAckSetAt(j);
                }
                position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets);
            } else {
                position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId());
            }
        }
        if (ack.hasTxnidMostBits() && ack.hasTxnidLeastBits()) {
            List<PositionImpl> positionsAcked = Collections.singletonList(position);
            return transactionCumulativeAcknowledge(ack.getTxnidMostBits(), ack.getTxnidLeastBits(), positionsAcked);
        } else {
            List<Position> positionsAcked = Collections.singletonList(position);
            subscription.acknowledgeMessage(positionsAcked, AckType.Cumulative, properties);
            return CompletableFuture.completedFuture(null);
        }
    } else {
        if (ack.hasTxnidLeastBits() && ack.hasTxnidMostBits()) {
            return individualAckWithTransaction(ack);
        } else {
            return individualAckNormal(ack, properties);
        }
    }
}

Example 2 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class Consumer method individualAckWithTransaction.

// this method is for individual ack carry the transaction
private CompletableFuture<Void> individualAckWithTransaction(CommandAck ack) {
    // Individual ack
    List<MutablePair<PositionImpl, Integer>> positionsAcked = new ArrayList<>();
    if (!isTransactionEnabled()) {
        return FutureUtil.failedFuture(new BrokerServiceException.NotAllowedException("Server don't support transaction ack!"));
    }
    for (int i = 0; i < ack.getMessageIdsCount(); i++) {
        MessageIdData msgId = ack.getMessageIdAt(i);
        PositionImpl position;
        long ackedCount = 0;
        long batchSize = getBatchSize(msgId);
        Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId());
        if (msgId.getAckSetsCount() > 0) {
            long[] ackSets = new long[msgId.getAckSetsCount()];
            for (int j = 0; j < msgId.getAckSetsCount(); j++) {
                ackSets[j] = msgId.getAckSetAt(j);
            }
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets);
            ackedCount = getAckedCountForTransactionAck(batchSize, ackSets);
        } else {
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId());
            ackedCount = batchSize;
        }
        positionsAcked.add(new MutablePair<>(position, (int) batchSize));
        addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount);
        checkCanRemovePendingAcksAndHandle(position, msgId);
        checkAckValidationError(ack, position);
    }
    CompletableFuture<Void> completableFuture = transactionIndividualAcknowledge(ack.getTxnidMostBits(), ack.getTxnidLeastBits(), positionsAcked);
    if (Subscription.isIndividualAckMode(subType)) {
        completableFuture.whenComplete((v, e) -> positionsAcked.forEach(positionLongMutablePair -> {
            if (positionLongMutablePair.getLeft().getAckSet() != null) {
                if (((PersistentSubscription) subscription).checkIsCanDeleteConsumerPendingAck(positionLongMutablePair.left)) {
                    removePendingAcks(positionLongMutablePair.left);
                }
            }
        }));
    }
    return completableFuture;
}

Example 3 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class TransactionConsumeTest method appendTransactionMessages.

private List<MessageIdData> appendTransactionMessages(TxnID txnID, PersistentTopic topic, int transactionMsgCnt, List<String> sendMessageList) throws ExecutionException, InterruptedException, PulsarClientException {
    // Change the state of TB to Ready.
    @Cleanup Producer<String> producer = PulsarClient.builder().serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()).enableTransaction(true).build().newProducer(Schema.STRING).topic(CONSUME_TOPIC).sendTimeout(0, TimeUnit.SECONDS).create();
    List<MessageIdData> positionList = new ArrayList<>();
    for (int i = 0; i < transactionMsgCnt; i++) {
        final int j = i;
        MessageMetadata metadata = new MessageMetadata().setProducerName("producerName").setSequenceId(i).setTxnidMostBits(txnID.getMostSigBits()).setTxnidLeastBits(txnID.getLeastSigBits()).setPublishTime(System.currentTimeMillis());
        String msg = TXN_MSG_CONTENT + i;
        sendMessageList.add(msg);
        ByteBuf headerAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, Unpooled.copiedBuffer(msg.getBytes(UTF_8)));
        CompletableFuture<PositionImpl> completableFuture = new CompletableFuture<>();
        topic.publishTxnMessage(txnID, headerAndPayload, new Topic.PublishContext() {

            @Override
            public String getProducerName() {
                return "test";
            }

            public long getSequenceId() {
                return j + 30;
            }

            /**
             * Return the producer name for the original producer.
             *
             * For messages published locally, this will return the same local producer name, though in case of replicated
             * messages, the original producer name will differ
             */
            public String getOriginalProducerName() {
                return "test";
            }

            public long getOriginalSequenceId() {
                return j + 30;
            }

            public long getHighestSequenceId() {
                return j + 30;
            }

            public long getOriginalHighestSequenceId() {
                return j + 30;
            }

            public long getNumberOfMessages() {
                return j + 30;
            }

            @Override
            public void completed(Exception e, long ledgerId, long entryId) {
                completableFuture.complete(PositionImpl.get(ledgerId, entryId));
            }
        });
        positionList.add(new MessageIdData().setLedgerId(completableFuture.get().getLedgerId()).setEntryId(completableFuture.get().getEntryId()));
    }
    log.info("append messages to TB finish.");
    return positionList;
}

Example 4 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class CompactedTopicTest method testEntryLookup.

@Test
public void testEntryLookup() throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null);
    Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> compactedLedgerData = buildCompactedLedger(bk, 500);
    List<Pair<MessageIdData, Long>> positions = compactedLedgerData.getMiddle();
    List<Pair<MessageIdData, Long>> idsInGaps = compactedLedgerData.getRight();
    LedgerHandle lh = bk.openLedger(compactedLedgerData.getLeft(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    long lastEntryId = lh.getLastAddConfirmed();
    AsyncLoadingCache<Long, MessageIdData> cache = CompactedTopicImpl.createCache(lh, 50);
    MessageIdData firstPositionId = positions.get(0).getLeft();
    Pair<MessageIdData, Long> lastPosition = positions.get(positions.size() - 1);
    // check ids before and after ids in compacted ledger
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(0, 0), lastEntryId, cache).get(), Long.valueOf(0));
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(Long.MAX_VALUE, 0), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED));
    // entry 0 is never in compacted ledger due to how we generate dummy
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(firstPositionId.getLedgerId(), 0), lastEntryId, cache).get(), Long.valueOf(0));
    // check next id after last id in compacted ledger
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(lastPosition.getLeft().getLedgerId(), lastPosition.getLeft().getEntryId() + 1), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED));
    // shuffle to make cache work hard
    Collections.shuffle(positions, r);
    Collections.shuffle(idsInGaps, r);
    // Check ids we know are in compacted ledger
    for (Pair<MessageIdData, Long> p : positions) {
        PositionImpl pos = new PositionImpl(p.getLeft().getLedgerId(), p.getLeft().getEntryId());
        Long got = CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get();
        Assert.assertEquals(got, p.getRight());
    }
    // Check ids we know are in the gaps of the compacted ledger
    for (Pair<MessageIdData, Long> gap : idsInGaps) {
        PositionImpl pos = new PositionImpl(gap.getLeft().getLedgerId(), gap.getLeft().getEntryId());
        Assert.assertEquals(CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get(), gap.getRight());
    }
}

Example 5 with MessageIdData

use of org.apache.pulsar.common.api.proto.MessageIdData in project pulsar by apache.

the class ServerCnx method handleSeek.

@Override
protected void handleSeek(CommandSeek seek) {
    checkArgument(state == State.Connected);
    final long requestId = seek.getRequestId();
    CompletableFuture<Consumer> consumerFuture = consumers.get(seek.getConsumerId());
    if (!seek.hasMessageId() && !seek.hasMessagePublishTime()) {
        commandSender.sendErrorResponse(requestId, ServerError.MetadataError, "Message id and message publish time were not present");
        return;
    }
    boolean consumerCreated = consumerFuture != null && consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally();
    if (consumerCreated && seek.hasMessageId()) {
        Consumer consumer = consumerFuture.getNow(null);
        Subscription subscription = consumer.getSubscription();
        MessageIdData msgIdData = seek.getMessageId();
        long[] ackSet = null;
        if (msgIdData.getAckSetsCount() > 0) {
            ackSet = new long[msgIdData.getAckSetsCount()];
            for (int i = 0; i < ackSet.length; i++) {
                ackSet[i] = msgIdData.getAckSetAt(i);
            }
        }
        Position position = new PositionImpl(msgIdData.getLedgerId(), msgIdData.getEntryId(), ackSet);
        subscription.resetCursor(position).thenRun(() -> {
            log.info("[{}] [{}][{}] Reset subscription to message id {}", remoteAddress, subscription.getTopic().getName(), subscription.getName(), position);
            commandSender.sendSuccessResponse(requestId);
        }).exceptionally(ex -> {
            log.warn("[{}][{}] Failed to reset subscription: {}", remoteAddress, subscription, ex.getMessage(), ex);
            commandSender.sendErrorResponse(requestId, ServerError.UnknownError, "Error when resetting subscription: " + ex.getCause().getMessage());
            return null;
        });
    } else if (consumerCreated && seek.hasMessagePublishTime()) {
        Consumer consumer = consumerFuture.getNow(null);
        Subscription subscription = consumer.getSubscription();
        long timestamp = seek.getMessagePublishTime();
        subscription.resetCursor(timestamp).thenRun(() -> {
            log.info("[{}] [{}][{}] Reset subscription to publish time {}", remoteAddress, subscription.getTopic().getName(), subscription.getName(), timestamp);
            commandSender.sendSuccessResponse(requestId);
        }).exceptionally(ex -> {
            log.warn("[{}][{}] Failed to reset subscription: {}", remoteAddress, subscription, ex.getMessage(), ex);
            commandSender.sendErrorResponse(requestId, ServerError.UnknownError, "Reset subscription to publish time error: " + ex.getCause().getMessage());
            return null;
        });
    } else {
        commandSender.sendErrorResponse(requestId, ServerError.MetadataError, "Consumer not found");
    }
}