Examples with MessageIdData org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData used on opensource projects

Example 1 with MessageIdData

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData in project incubator-pulsar by apache.

the class CompactedTopicTest method buildCompactedLedger.

/**
 * Build a compacted ledger, and return the id of the ledger, the position of the different
 * entries in the ledger, and a list of gaps, and the entry which should be returned after the gap.
 */
private Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> buildCompactedLedger(BookKeeper bk, int count) throws Exception {
    LedgerHandle lh = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    List<Pair<MessageIdData, Long>> positions = new ArrayList<>();
    List<Pair<MessageIdData, Long>> idsInGaps = new ArrayList<>();
    AtomicLong ledgerIds = new AtomicLong(10L);
    AtomicLong entryIds = new AtomicLong(0L);
    CompletableFuture.allOf(IntStream.range(0, count).mapToObj((i) -> {
        List<MessageIdData> idsInGap = new ArrayList<MessageIdData>();
        if (r.nextInt(10) == 1) {
            long delta = r.nextInt(10) + 1;
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
            ledgerIds.addAndGet(delta);
            entryIds.set(0);
        }
        long delta = r.nextInt(5);
        if (delta != 0) {
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
        }
        MessageIdData id = MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.addAndGet(delta + 1)).build();
        @Cleanup RawMessage m = new RawMessageImpl(id, Unpooled.EMPTY_BUFFER);
        CompletableFuture<Void> f = new CompletableFuture<>();
        ByteBuf buffer = m.serialize();
        lh.asyncAddEntry(buffer, (rc, ledger, eid, ctx) -> {
            if (rc != BKException.Code.OK) {
                f.completeExceptionally(BKException.create(rc));
            } else {
                positions.add(Pair.of(id, eid));
                idsInGap.forEach((gid) -> idsInGaps.add(Pair.of(gid, eid)));
                f.complete(null);
            }
        }, null);
        buffer.release();
        return f;
    }).toArray(CompletableFuture[]::new)).get();
    lh.close();
    return Triple.of(lh.getId(), positions, idsInGaps);
}

Example 2 with MessageIdData

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData in project incubator-pulsar by apache.

the class CompactedTopicTest method testEntryLookup.

@Test
public void testEntryLookup() throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null);
    Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> compactedLedgerData = buildCompactedLedger(bk, 500);
    List<Pair<MessageIdData, Long>> positions = compactedLedgerData.getMiddle();
    List<Pair<MessageIdData, Long>> idsInGaps = compactedLedgerData.getRight();
    LedgerHandle lh = bk.openLedger(compactedLedgerData.getLeft(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    long lastEntryId = lh.getLastAddConfirmed();
    AsyncLoadingCache<Long, MessageIdData> cache = CompactedTopicImpl.createCache(lh, 50);
    MessageIdData firstPositionId = positions.get(0).getLeft();
    Pair<MessageIdData, Long> lastPosition = positions.get(positions.size() - 1);
    // check ids before and after ids in compacted ledger
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(0, 0), lastEntryId, cache).get(), Long.valueOf(0));
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(Long.MAX_VALUE, 0), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED));
    // entry 0 is never in compacted ledger due to how we generate dummy
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(firstPositionId.getLedgerId(), 0), lastEntryId, cache).get(), Long.valueOf(0));
    // check next id after last id in compacted ledger
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(lastPosition.getLeft().getLedgerId(), lastPosition.getLeft().getEntryId() + 1), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED));
    // shuffle to make cache work hard
    Collections.shuffle(positions, r);
    Collections.shuffle(idsInGaps, r);
    // Check ids we know are in compacted ledger
    for (Pair<MessageIdData, Long> p : positions) {
        PositionImpl pos = new PositionImpl(p.getLeft().getLedgerId(), p.getLeft().getEntryId());
        Long got = CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get();
        Assert.assertEquals(got, Long.valueOf(p.getRight()));
    }
    // Check ids we know are in the gaps of the compacted ledger
    for (Pair<MessageIdData, Long> gap : idsInGaps) {
        PositionImpl pos = new PositionImpl(gap.getLeft().getLedgerId(), gap.getLeft().getEntryId());
        Assert.assertEquals(CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get(), Long.valueOf(gap.getRight()));
    }
}

Example 3 with MessageIdData

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData in project incubator-pulsar by apache.

the class Consumer method redeliverUnacknowledgedMessages.

public void redeliverUnacknowledgedMessages(List<MessageIdData> messageIds) {
    int totalRedeliveryMessages = 0;
    List<PositionImpl> pendingPositions = Lists.newArrayList();
    for (MessageIdData msg : messageIds) {
        PositionImpl position = PositionImpl.get(msg.getLedgerId(), msg.getEntryId());
        LongPair batchSize = pendingAcks.get(position.getLedgerId(), position.getEntryId());
        if (batchSize != null) {
            pendingAcks.remove(position.getLedgerId(), position.getEntryId());
            totalRedeliveryMessages += batchSize.first;
            pendingPositions.add(position);
        }
    }
    addAndGetUnAckedMsgs(this, -totalRedeliveryMessages);
    blockedConsumerOnUnackedMsgs = false;
    if (log.isDebugEnabled()) {
        log.debug("[{}-{}] consumer {} received {} msg-redelivery {}", topicName, subscription, consumerId, totalRedeliveryMessages, pendingPositions.size());
    }
    subscription.redeliverUnacknowledgedMessages(this, pendingPositions);
    msgRedeliver.recordMultipleEvents(totalRedeliveryMessages, totalRedeliveryMessages);
    int numberOfBlockedPermits = Math.min(totalRedeliveryMessages, PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.get(this));
    // if permitsReceivedWhileConsumerBlocked has been accumulated then pass it to Dispatcher to flow messages
    if (numberOfBlockedPermits > 0) {
        PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.getAndAdd(this, -numberOfBlockedPermits);
        MESSAGE_PERMITS_UPDATER.getAndAdd(this, numberOfBlockedPermits);
        subscription.consumerFlow(this, numberOfBlockedPermits);
    }
}

Example 4 with MessageIdData

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData in project incubator-pulsar by apache.

the class Consumer method messageAcked.

void messageAcked(CommandAck ack) {
    MessageIdData msgId = ack.getMessageId();
    PositionImpl position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId());
    if (ack.hasValidationError()) {
        log.error("[{}] [{}] Received ack for corrupted message at {} - Reason: {}", subscription, consumerId, position, ack.getValidationError());
    }
    Map<String, Long> properties = Collections.emptyMap();
    if (ack.getPropertiesCount() > 0) {
        properties = ack.getPropertiesList().stream().collect(Collectors.toMap((e) -> e.getKey(), (e) -> e.getValue()));
    }
    if (subType == SubType.Shared) {
        // On shared subscriptions, cumulative ack is not supported
        checkArgument(ack.getAckType() == AckType.Individual);
        // Only ack a single message
        removePendingAcks(position);
        subscription.acknowledgeMessage(position, AckType.Individual, properties);
    } else {
        subscription.acknowledgeMessage(position, ack.getAckType(), properties);
    }
}

Example 5 with MessageIdData

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData in project incubator-pulsar by apache.

the class ConsumerImpl method connectionOpened.

@Override
public void connectionOpened(final ClientCnx cnx) {
    setClientCnx(cnx);
    cnx.registerConsumer(consumerId, this);
    log.info("[{}][{}] Subscribing to topic on cnx {}", topic, subscription, cnx.ctx().channel());
    long requestId = client.newRequestId();
    int currentSize;
    synchronized (this) {
        currentSize = incomingMessages.size();
        startMessageId = clearReceiverQueue();
        unAckedMessageTracker.clear();
    }
    boolean isDurable = subscriptionMode == SubscriptionMode.Durable;
    MessageIdData startMessageIdData;
    if (isDurable) {
        // For regular durable subscriptions, the message id from where to restart will be determined by the broker.
        startMessageIdData = null;
    } else {
        // For non-durable we are going to restart from the next entry
        MessageIdData.Builder builder = MessageIdData.newBuilder();
        builder.setLedgerId(startMessageId.getLedgerId());
        builder.setEntryId(startMessageId.getEntryId());
        if (startMessageId instanceof BatchMessageIdImpl) {
            builder.setBatchIndex(((BatchMessageIdImpl) startMessageId).getBatchIndex());
        }
        startMessageIdData = builder.build();
        builder.recycle();
    }
    ByteBuf request = Commands.newSubscribe(topic, subscription, consumerId, requestId, getSubType(), priorityLevel, consumerName, isDurable, startMessageIdData, metadata, readCompacted, InitialPosition.valueOf(subscriptionInitialPosition.getValue()));
    if (startMessageIdData != null) {
        startMessageIdData.recycle();
    }
    cnx.sendRequestWithId(request, requestId).thenRun(() -> {
        synchronized (ConsumerImpl.this) {
            if (changeToReadyState()) {
                log.info("[{}][{}] Subscribed to topic on {} -- consumer: {}", topic, subscription, cnx.channel().remoteAddress(), consumerId);
                AVAILABLE_PERMITS_UPDATER.set(this, 0);
                // or queue was not empty: send a flow command
                if (waitingOnReceiveForZeroQueueSize || (conf.getReceiverQueueSize() == 0 && currentSize > 0)) {
                    sendFlowPermitsToBroker(cnx, 1);
                }
            } else {
                // Consumer was closed while reconnecting, close the connection to make sure the broker
                // drops the consumer on its side
                setState(State.Closed);
                cnx.removeConsumer(consumerId);
                cnx.channel().close();
                return;
            }
        }
        resetBackoff();
        boolean firstTimeConnect = subscribeFuture.complete(this);
        // command to receive messages
        if (!(firstTimeConnect && partitionIndex > -1) && conf.getReceiverQueueSize() != 0) {
            sendFlowPermitsToBroker(cnx, conf.getReceiverQueueSize());
        }
    }).exceptionally((e) -> {
        cnx.removeConsumer(consumerId);
        if (getState() == State.Closing || getState() == State.Closed) {
            // Consumer was closed while reconnecting, close the connection to make sure the broker
            // drops the consumer on its side
            cnx.channel().close();
            return null;
        }
        log.warn("[{}][{}] Failed to subscribe to topic on {}", topic, subscription, cnx.channel().remoteAddress());
        if (e.getCause() instanceof PulsarClientException && getConnectionHandler().isRetriableError((PulsarClientException) e.getCause()) && System.currentTimeMillis() < subscribeTimeout) {
            reconnectLater(e.getCause());
            return null;
        }
        if (!subscribeFuture.isDone()) {
            // unable to create new consumer, fail operation
            setState(State.Failed);
            subscribeFuture.completeExceptionally(e);
            client.cleanupConsumer(this);
        } else {
            // consumer was subscribed and connected but we got some error, keep trying
            reconnectLater(e.getCause());
        }
        return null;
    });
}