Examples with BitSetRecyclable org.apache.pulsar.common.util.collections.BitSetRecyclable used on opensource projects

Example 16 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class ManagedCursorImpl method asyncDelete.

@Override
public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallback callback, Object ctx) {
    if (isClosed()) {
        callback.deleteFailed(new ManagedLedgerException.CursorAlreadyClosedException("Cursor was already closed"), ctx);
        return;
    }
    PositionImpl newMarkDeletePosition = null;
    lock.writeLock().lock();
    try {
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] Deleting individual messages at {}. Current status: {} - md-position: {}", ledger.getName(), name, positions, individualDeletedMessages, markDeletePosition);
        }
        for (Position pos : positions) {
            PositionImpl position = (PositionImpl) checkNotNull(pos);
            if (((PositionImpl) ledger.getLastConfirmedEntry()).compareTo(position) < 0) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {} " + "for cursor [{}]", ledger.getName(), position, ledger.getLastConfirmedEntry(), name);
                }
                callback.deleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx);
                return;
            }
            if (individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId()) || position.compareTo(markDeletePosition) <= 0) {
                if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) {
                    BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
                    if (bitSetRecyclable != null) {
                        bitSetRecyclable.recycle();
                    }
                }
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Position was already deleted {}", ledger.getName(), name, position);
                }
                continue;
            }
            if (position.ackSet == null) {
                if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) {
                    BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
                    if (bitSetRecyclable != null) {
                        bitSetRecyclable.recycle();
                    }
                }
                // Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
                // make the RangeSet recognize the "continuity" between adjacent Positions.
                PositionImpl previousPosition = ledger.getPreviousPosition(position);
                individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(), previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId());
                MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Individually deleted messages: {}", ledger.getName(), name, individualDeletedMessages);
                }
            } else if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) {
                BitSetRecyclable bitSet = batchDeletedIndexes.computeIfAbsent(position, (v) -> BitSetRecyclable.create().resetWords(position.ackSet));
                BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(position.ackSet);
                bitSet.and(givenBitSet);
                givenBitSet.recycle();
                if (bitSet.isEmpty()) {
                    PositionImpl previousPosition = ledger.getPreviousPosition(position);
                    individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(), previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId());
                    MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
                    BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
                    if (bitSetRecyclable != null) {
                        bitSetRecyclable.recycle();
                    }
                }
            }
        }
        if (individualDeletedMessages.isEmpty()) {
            // No changes to individually deleted messages, so nothing to do at this point
            return;
        }
        // If the lower bound of the range set is the current mark delete position, then we can trigger a new
        // mark-delete to the upper bound of the first range segment
        Range<PositionImpl> range = individualDeletedMessages.firstRange();
        // individualDeletedMessages are now irrelevant
        if (range.upperEndpoint().compareTo(markDeletePosition) <= 0) {
            individualDeletedMessages.removeAtMost(markDeletePosition.getLedgerId(), markDeletePosition.getEntryId());
            range = individualDeletedMessages.firstRange();
        }
        if (range == null) {
            // The set was completely cleaned up now
            return;
        }
        // If the lowerBound is ahead of MarkDelete, verify if there are any entries in-between
        if (range.lowerEndpoint().compareTo(markDeletePosition) <= 0 || ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, range.lowerEndpoint())) <= 0) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Found a position range to mark delete for cursor {}: {} ", ledger.getName(), name, range);
            }
            newMarkDeletePosition = range.upperEndpoint();
        }
        if (newMarkDeletePosition != null) {
            newMarkDeletePosition = setAcknowledgedPosition(newMarkDeletePosition);
        } else {
            newMarkDeletePosition = markDeletePosition;
        }
    } catch (Exception e) {
        log.warn("[{}] [{}] Error while updating individualDeletedMessages [{}]", ledger.getName(), name, e.getMessage(), e);
        callback.deleteFailed(getManagedLedgerException(e), ctx);
        return;
    } finally {
        lock.writeLock().unlock();
        if (individualDeletedMessages.isEmpty()) {
            callback.deleteComplete(ctx);
        }
    }
    // Apply rate limiting to mark-delete operations
    if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) {
        isDirty = true;
        PositionImpl finalNewMarkDeletePosition = newMarkDeletePosition;
        LAST_MARK_DELETE_ENTRY_UPDATER.updateAndGet(this, last -> new MarkDeleteEntry(finalNewMarkDeletePosition, last.properties, null, null));
        callback.deleteComplete(ctx);
        return;
    }
    try {
        Map<String, Long> properties = lastMarkDeleteEntry != null ? lastMarkDeleteEntry.properties : Collections.emptyMap();
        internalAsyncMarkDelete(newMarkDeletePosition, properties, new MarkDeleteCallback() {

            @Override
            public void markDeleteComplete(Object ctx) {
                callback.deleteComplete(ctx);
            }

            @Override
            public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
                callback.deleteFailed(exception, ctx);
            }
        }, ctx);
    } catch (Exception e) {
        log.warn("[{}] [{}] Error doing asyncDelete [{}]", ledger.getName(), name, e.getMessage(), e);
        if (log.isDebugEnabled()) {
            log.debug("[{}] Consumer {} cursor asyncDelete error, counters: consumed {} mdPos {} rdPos {}", ledger.getName(), name, messagesConsumedCounter, markDeletePosition, readPosition);
        }
        callback.deleteFailed(new ManagedLedgerException(e), ctx);
    }
}

Example 17 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class ManagedCursorImpl method internalResetCursor.

protected void internalResetCursor(PositionImpl position, AsyncCallbacks.ResetCursorCallback resetCursorCallback) {
    if (position.equals(PositionImpl.EARLIEST)) {
        position = ledger.getFirstPosition();
    } else if (position.equals(PositionImpl.LATEST)) {
        position = ledger.getLastPosition().getNext();
    }
    log.info("[{}] Initiate reset position to {} on cursor {}", ledger.getName(), position, name);
    synchronized (pendingMarkDeleteOps) {
        if (!RESET_CURSOR_IN_PROGRESS_UPDATER.compareAndSet(this, FALSE, TRUE)) {
            log.error("[{}] reset requested - position [{}], previous reset in progress - cursor {}", ledger.getName(), position, name);
            resetCursorCallback.resetFailed(new ManagedLedgerException.ConcurrentFindCursorPositionException("reset already in progress"), position);
        }
    }
    final AsyncCallbacks.ResetCursorCallback callback = resetCursorCallback;
    final PositionImpl newPosition = position;
    VoidCallback finalCallback = new VoidCallback() {

        @Override
        public void operationComplete() {
            // modify mark delete and read position since we are able to persist new position for cursor
            lock.writeLock().lock();
            try {
                PositionImpl newMarkDeletePosition = ledger.getPreviousPosition(newPosition);
                if (markDeletePosition.compareTo(newMarkDeletePosition) >= 0) {
                    MSG_CONSUMED_COUNTER_UPDATER.addAndGet(cursorImpl(), -getNumberOfEntries(Range.closedOpen(newMarkDeletePosition, markDeletePosition)));
                } else {
                    MSG_CONSUMED_COUNTER_UPDATER.addAndGet(cursorImpl(), getNumberOfEntries(Range.closedOpen(markDeletePosition, newMarkDeletePosition)));
                }
                markDeletePosition = newMarkDeletePosition;
                lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), null, null);
                individualDeletedMessages.clear();
                if (config.isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) {
                    batchDeletedIndexes.values().forEach(BitSetRecyclable::recycle);
                    batchDeletedIndexes.clear();
                    long[] resetWords = newPosition.ackSet;
                    if (resetWords != null) {
                        BitSetRecyclable ackSet = BitSetRecyclable.create().resetWords(resetWords);
                        batchDeletedIndexes.put(newPosition, ackSet);
                    }
                }
                PositionImpl oldReadPosition = readPosition;
                if (oldReadPosition.compareTo(newPosition) >= 0) {
                    log.info("[{}] reset position to {} before current read position {} on cursor {}", ledger.getName(), newPosition, oldReadPosition, name);
                } else {
                    log.info("[{}] reset position to {} skipping from current read position {} on cursor {}", ledger.getName(), newPosition, oldReadPosition, name);
                }
                readPosition = newPosition;
            } finally {
                lock.writeLock().unlock();
            }
            synchronized (pendingMarkDeleteOps) {
                pendingMarkDeleteOps.clear();
                if (!RESET_CURSOR_IN_PROGRESS_UPDATER.compareAndSet(ManagedCursorImpl.this, TRUE, FALSE)) {
                    log.error("[{}] expected reset position [{}], but another reset in progress on cursor {}", ledger.getName(), newPosition, name);
                }
            }
            callback.resetComplete(newPosition);
            updateLastActive();
        }

        @Override
        public void operationFailed(ManagedLedgerException exception) {
            synchronized (pendingMarkDeleteOps) {
                if (!RESET_CURSOR_IN_PROGRESS_UPDATER.compareAndSet(ManagedCursorImpl.this, TRUE, FALSE)) {
                    log.error("[{}] expected reset position [{}], but another reset in progress on cursor {}", ledger.getName(), newPosition, name);
                }
            }
            callback.resetFailed(new ManagedLedgerException.InvalidCursorPositionException("unable to persist position for cursor reset " + newPosition.toString()), newPosition);
        }
    };
    internalAsyncMarkDelete(newPosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), new MarkDeleteCallback() {

        @Override
        public void markDeleteComplete(Object ctx) {
            finalCallback.operationComplete();
        }

        @Override
        public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
            finalCallback.operationFailed(exception);
        }
    }, null);
}

Example 18 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class PositionAckSetUtil method compareToWithAckSet.

// This method is compare two position which position is bigger than another one.
// When the ledgerId and entryId in this position is same to another one and two position all have ack set, it will
// compare the ack set next bit index is bigger than another one.
public static int compareToWithAckSet(PositionImpl currentPosition, PositionImpl otherPosition) {
    if (currentPosition == null || otherPosition == null) {
        throw new IllegalArgumentException("Two positions can't be null! " + "current position : [" + currentPosition + "] other position : [" + otherPosition + "]");
    }
    int result = ComparisonChain.start().compare(currentPosition.getLedgerId(), otherPosition.getLedgerId()).compare(currentPosition.getEntryId(), otherPosition.getEntryId()).result();
    if (result == 0) {
        BitSetRecyclable otherAckSet;
        BitSetRecyclable currentAckSet;
        if (otherPosition.getAckSet() == null) {
            otherAckSet = BitSetRecyclable.create();
        } else {
            otherAckSet = BitSetRecyclable.valueOf(otherPosition.getAckSet());
        }
        if (currentPosition.getAckSet() == null) {
            currentAckSet = BitSetRecyclable.create();
        } else {
            currentAckSet = BitSetRecyclable.valueOf(currentPosition.getAckSet());
        }
        if (currentAckSet.isEmpty() || otherAckSet.isEmpty()) {
            // when ack set is empty, the nextSetBit will return -1, so we should return the inverse value.
            result = -(currentAckSet.nextSetBit(0) - otherAckSet.nextSetBit(0));
        } else {
            result = currentAckSet.nextSetBit(0) - otherAckSet.nextSetBit(0);
        }
        currentAckSet.recycle();
        otherAckSet.recycle();
    }
    return result;
}

Example 19 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class PendingAckHandleImpl method internalIndividualAcknowledgeMessage.

public void internalIndividualAcknowledgeMessage(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions, CompletableFuture<Void> completableFuture) {
    if (txnID == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    if (positions == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> {
        synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) {
            for (MutablePair<PositionImpl, Integer> positionIntegerMutablePair : positions) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] individualAcknowledgeMessage position: [{}], " + "txnId: [{}], subName: [{}]", topicName, positionIntegerMutablePair.left, txnID, subName);
                }
                PositionImpl position = positionIntegerMutablePair.left;
                // normal acknowledge,throw exception.
                if (((ManagedCursorImpl) persistentSubscription.getCursor()).isMessageDeleted(position)) {
                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before.";
                    log.error(errorMsg);
                    completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                    return;
                }
                if (position.hasAckSet()) {
                    // in order to jude the bit set is over lap, so set the covering
                    // the batch size bit to 1,should know the two
                    // bit set don't have the same point is 0
                    BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet());
                    if (positionIntegerMutablePair.right > bitSetRecyclable.size()) {
                        bitSetRecyclable.set(positionIntegerMutablePair.right);
                    }
                    bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size());
                    long[] ackSetOverlap = bitSetRecyclable.toLongArray();
                    bitSetRecyclable.recycle();
                    if (isAckSetOverlap(ackSetOverlap, ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                    if (individualAckPositions != null && individualAckPositions.containsKey(position) && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack batch message:" + position + " in pending ack status.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                } else {
                    if (individualAckPositions != null && individualAckPositions.containsKey(position)) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " in pending ack status.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                }
            }
            handleIndividualAck(txnID, positions);
            completableFuture.complete(null);
        }
    }).exceptionally(e -> {
        synchronized (PendingAckHandleImpl.this) {
            // we also modify the in memory state when append fail,
            // because we don't know the persistent state, when were replay it,
            // it will produce the wrong operation. so we append fail,
            // we should wait tc time out or client abort this transaction.
            handleIndividualAck(txnID, positions);
            completableFuture.completeExceptionally(e.getCause());
        }
        return null;
    })).exceptionally(e -> {
        completableFuture.completeExceptionally(e);
        return null;
    });
}

Example 20 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class PendingAckHandleImpl method getTransactionInPendingAckStats.

@Override
public TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID) {
    TransactionInPendingAckStats transactionInPendingAckStats = new TransactionInPendingAckStats();
    if (cumulativeAckOfTransaction != null && cumulativeAckOfTransaction.getLeft().equals(txnID)) {
        PositionImpl position = cumulativeAckOfTransaction.getRight();
        StringBuilder stringBuilder = new StringBuilder().append(position.getLedgerId()).append(':').append(position.getEntryId());
        if (cumulativeAckOfTransaction.getRight().hasAckSet()) {
            BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(cumulativeAckOfTransaction.getRight().getAckSet());
            if (!bitSetRecyclable.isEmpty()) {
                stringBuilder.append(":").append(bitSetRecyclable.nextSetBit(0) - 1);
            }
        }
        transactionInPendingAckStats.cumulativeAckPosition = stringBuilder.toString();
    }
    return transactionInPendingAckStats;
}