Examples with BitSetRecyclable org.apache.pulsar.common.util.collections.BitSetRecyclable used on opensource projects

Example 11 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class Consumer method getUnAckedCountForBatchIndexLevelEnabled.

private long getUnAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize) {
    long unAckedCount = batchSize;
    if (isAcknowledgmentAtBatchIndexLevelEnabled) {
        long[] cursorAckSet = getCursorAckSet(position);
        if (cursorAckSet != null) {
            BitSetRecyclable cursorBitSet = BitSetRecyclable.create().resetWords(cursorAckSet);
            unAckedCount = cursorBitSet.cardinality();
            cursorBitSet.recycle();
        }
    }
    return unAckedCount;
}

Example 12 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class EntryBatchIndexesAcksTest method shouldResetStateBeforeReusing.

@Test
void shouldResetStateBeforeReusing() {
    // given
    // a bitset with 95 bits set
    BitSetRecyclable bitSet = BitSetRecyclable.create();
    bitSet.set(0, 95);
    long[] nintyFiveBitsSet = bitSet.toLongArray();
    // and a EntryBatchIndexesAcks for the size of 10
    EntryBatchIndexesAcks acks = EntryBatchIndexesAcks.get(10);
    // when setting 2 indexes with 95/100 bits set in each (5 "acked" in each)
    acks.setIndexesAcks(8, Pair.of(100, nintyFiveBitsSet));
    acks.setIndexesAcks(9, Pair.of(100, nintyFiveBitsSet));
    // then the totalAckedIndexCount should be 10
    assertEquals(acks.getTotalAckedIndexCount(), 10);
    // when recycled and used again
    acks.recycle();
    acks = EntryBatchIndexesAcks.get(2);
    // then there should be no previous state and totalAckedIndexCount should be 0
    assertEquals(acks.getTotalAckedIndexCount(), 0);
}

Example 13 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class ConsumerImpl method receiveIndividualMessagesFromBatch.

void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, MessageMetadata msgMetadata, int redeliveryCount, List<Long> ackSet, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, long consumerEpoch) {
    int batchSize = msgMetadata.getNumMessagesInBatch();
    // create ack tracker for entry aka batch
    MessageIdImpl batchMessage = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), getPartitionIndex());
    List<MessageImpl<T>> possibleToDeadLetter = null;
    if (deadLetterPolicy != null && redeliveryCount >= deadLetterPolicy.getMaxRedeliverCount()) {
        possibleToDeadLetter = new ArrayList<>();
    }
    BatchMessageAcker acker = BatchMessageAcker.newAcker(batchSize);
    BitSetRecyclable ackBitSet = null;
    if (ackSet != null && ackSet.size() > 0) {
        ackBitSet = BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet));
    }
    SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
    int skippedMessages = 0;
    try {
        for (int i = 0; i < batchSize; ++i) {
            final MessageImpl<T> message = newSingleMessage(i, batchSize, brokerEntryMetadata, msgMetadata, singleMessageMetadata, uncompressedPayload, batchMessage, schema, true, ackBitSet, acker, redeliveryCount, consumerEpoch);
            if (message == null) {
                skippedMessages++;
                continue;
            }
            if (possibleToDeadLetter != null) {
                possibleToDeadLetter.add(message);
            }
            executeNotifyCallback(message);
        }
        if (ackBitSet != null) {
            ackBitSet.recycle();
        }
    } catch (IllegalStateException e) {
        log.warn("[{}] [{}] unable to obtain message in batch", subscription, consumerName, e);
        discardCorruptedMessage(messageId, cnx, ValidationError.BatchDeSerializeError);
    }
    if (possibleToDeadLetter != null && possibleSendToDeadLetterTopicMessages != null) {
        possibleSendToDeadLetterTopicMessages.put(batchMessage, possibleToDeadLetter);
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] [{}] enqueued messages in batch. queue size - {}, available queue size - {}", subscription, consumerName, incomingMessages.size(), incomingMessages.remainingCapacity());
    }
    if (skippedMessages > 0) {
        increaseAvailablePermits(cnx, skippedMessages);
    }
}

Example 14 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class ConsumerImpl method seekAsync.

@Override
public CompletableFuture<Void> seekAsync(MessageId messageId) {
    String seekBy = String.format("the message %s", messageId.toString());
    return seekAsyncCheckState(seekBy).orElseGet(() -> {
        long requestId = client.newRequestId();
        ByteBuf seek = null;
        if (messageId instanceof BatchMessageIdImpl) {
            BatchMessageIdImpl msgId = (BatchMessageIdImpl) messageId;
            // Initialize ack set
            BitSetRecyclable ackSet = BitSetRecyclable.create();
            ackSet.set(0, msgId.getBatchSize());
            ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0));
            long[] ackSetArr = ackSet.toLongArray();
            ackSet.recycle();
            seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr);
        } else if (messageId instanceof ChunkMessageIdImpl) {
            ChunkMessageIdImpl msgId = (ChunkMessageIdImpl) messageId;
            seek = Commands.newSeek(consumerId, requestId, msgId.getFirstChunkMessageId().getLedgerId(), msgId.getFirstChunkMessageId().getEntryId(), new long[0]);
        } else {
            MessageIdImpl msgId = (MessageIdImpl) messageId;
            seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), new long[0]);
        }
        return seekAsyncInternal(requestId, seek, messageId, seekBy);
    });
}

Example 15 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class ManagedCursorImpl method asyncMarkDelete.

@Override
public void asyncMarkDelete(final Position position, Map<String, Long> properties, final MarkDeleteCallback callback, final Object ctx) {
    checkNotNull(position);
    checkArgument(position instanceof PositionImpl);
    if (isClosed()) {
        callback.markDeleteFailed(new ManagedLedgerException.CursorAlreadyClosedException("Cursor was already closed"), ctx);
        return;
    }
    if (RESET_CURSOR_IN_PROGRESS_UPDATER.get(this) == TRUE) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] cursor reset in progress - ignoring mark delete on position [{}] for cursor [{}]", ledger.getName(), position, name);
        }
        callback.markDeleteFailed(new ManagedLedgerException("Reset cursor in progress - unable to mark delete position " + position.toString()), ctx);
        return;
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Mark delete cursor {} up to position: {}", ledger.getName(), name, position);
    }
    PositionImpl newPosition = (PositionImpl) position;
    if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) {
        if (newPosition.ackSet != null) {
            batchDeletedIndexes.put(newPosition, BitSetRecyclable.create().resetWords(newPosition.ackSet));
            newPosition = ledger.getPreviousPosition(newPosition);
        }
        Map<PositionImpl, BitSetRecyclable> subMap = batchDeletedIndexes.subMap(PositionImpl.EARLIEST, newPosition);
        subMap.values().forEach(BitSetRecyclable::recycle);
        subMap.clear();
    } else if (newPosition.ackSet != null) {
        newPosition = ledger.getPreviousPosition(newPosition);
        newPosition.ackSet = null;
    }
    if (((PositionImpl) ledger.getLastConfirmedEntry()).compareTo(newPosition) < 0) {
        boolean shouldCursorMoveForward = false;
        try {
            long ledgerEntries = ledger.getLedgerInfo(markDeletePosition.getLedgerId()).get().getEntries();
            Long nextValidLedger = ledger.getNextValidLedger(ledger.getLastConfirmedEntry().getLedgerId());
            shouldCursorMoveForward = (markDeletePosition.getEntryId() + 1 >= ledgerEntries) && (newPosition.getLedgerId() == nextValidLedger);
        } catch (Exception e) {
            log.warn("Failed to get ledger entries while setting mark-delete-position", e);
        }
        if (shouldCursorMoveForward) {
            log.info("[{}] move mark-delete-position from {} to {} since all the entries have been consumed", ledger.getName(), markDeletePosition, newPosition);
        } else {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {}" + " for cursor [{}]", ledger.getName(), position, ledger.getLastConfirmedEntry(), name);
            }
            callback.markDeleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx);
            return;
        }
    }
    lock.writeLock().lock();
    try {
        newPosition = setAcknowledgedPosition(newPosition);
    } catch (IllegalArgumentException e) {
        callback.markDeleteFailed(getManagedLedgerException(e), ctx);
        return;
    } finally {
        lock.writeLock().unlock();
    }
    // Apply rate limiting to mark-delete operations
    if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) {
        isDirty = true;
        lastMarkDeleteEntry = new MarkDeleteEntry(newPosition, properties, null, null);
        callback.markDeleteComplete(ctx);
        return;
    }
    internalAsyncMarkDelete(newPosition, properties, callback, ctx);
}