Examples with PendingAckStore org.apache.pulsar.broker.transaction.pendingack.PendingAckStore used on opensource projects

Example 1 with PendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.PendingAckStore in project pulsar by apache.

the class TransactionTest method testSubscriptionRecreateTopic.

@Test
public void testSubscriptionRecreateTopic() throws PulsarAdminException, NoSuchFieldException, IllegalAccessException, PulsarClientException {
    String topic = "persistent://pulsar/system/testReCreateTopic";
    String subName = "sub_testReCreateTopic";
    int retentionSizeInMbSetTo = 5;
    int retentionSizeInMbSetTopic = 6;
    int retentionSizeInMinutesSetTo = 5;
    int retentionSizeInMinutesSetTopic = 6;
    admin.topics().createNonPartitionedTopic(topic);
    PulsarService pulsarService = super.getPulsarServiceList().get(0);
    pulsarService.getBrokerService().getTopics().clear();
    ManagedLedgerFactory managedLedgerFactory = pulsarService.getBrokerService().getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(managedLedgerFactory);
    ledgers.remove(TopicName.get(topic).getPersistenceNamingEncoding());
    try {
        admin.topics().createNonPartitionedTopic(topic);
        Assert.fail();
    } catch (PulsarAdminException.ConflictException e) {
        log.info("Cann`t create topic again");
    }
    admin.topics().setRetention(topic, new RetentionPolicies(retentionSizeInMinutesSetTopic, retentionSizeInMbSetTopic));
    pulsarClient.newConsumer().topic(topic).subscriptionName(subName).subscribe();
    pulsarService.getBrokerService().getTopicIfExists(topic).thenAccept(option -> {
        if (!option.isPresent()) {
            log.error("Failed o get Topic named: {}", topic);
            Assert.fail();
        }
        PersistentTopic originPersistentTopic = (PersistentTopic) option.get();
        String pendingAckTopicName = MLPendingAckStore.getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName);
        try {
            admin.topics().setRetention(pendingAckTopicName, new RetentionPolicies(retentionSizeInMinutesSetTo, retentionSizeInMbSetTo));
        } catch (PulsarAdminException e) {
            log.error("Failed to get./setRetention of topic with Exception:" + e);
            Assert.fail();
        }
        PersistentSubscription subscription = originPersistentTopic.getSubscription(subName);
        subscription.getPendingAckManageLedger().thenAccept(managedLedger -> {
            long retentionSize = managedLedger.getConfig().getRetentionSizeInMB();
            if (!originPersistentTopic.getTopicPolicies().isPresent()) {
                log.error("Failed to getTopicPolicies of :" + originPersistentTopic);
                Assert.fail();
            }
            TopicPolicies topicPolicies = originPersistentTopic.getTopicPolicies().get();
            Assert.assertEquals(retentionSizeInMbSetTopic, retentionSize);
            MLPendingAckStoreProvider mlPendingAckStoreProvider = new MLPendingAckStoreProvider();
            CompletableFuture<PendingAckStore> future = mlPendingAckStoreProvider.newPendingAckStore(subscription);
            future.thenAccept(pendingAckStore -> {
                ((MLPendingAckStore) pendingAckStore).getManagedLedger().thenAccept(managedLedger1 -> {
                    Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), retentionSizeInMbSetTo);
                });
            });
        });
    });
}

Example 2 with PendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.PendingAckStore in project pulsar by yahoo.

the class PendingAckHandleImpl method internalIndividualAcknowledgeMessage.

public void internalIndividualAcknowledgeMessage(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions, CompletableFuture<Void> completableFuture) {
    if (txnID == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    if (positions == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> {
        synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) {
            for (MutablePair<PositionImpl, Integer> positionIntegerMutablePair : positions) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] individualAcknowledgeMessage position: [{}], " + "txnId: [{}], subName: [{}]", topicName, positionIntegerMutablePair.left, txnID, subName);
                }
                PositionImpl position = positionIntegerMutablePair.left;
                // normal acknowledge,throw exception.
                if (((ManagedCursorImpl) persistentSubscription.getCursor()).isMessageDeleted(position)) {
                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before.";
                    log.error(errorMsg);
                    completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                    return;
                }
                if (position.hasAckSet()) {
                    // in order to jude the bit set is over lap, so set the covering
                    // the batch size bit to 1,should know the two
                    // bit set don't have the same point is 0
                    BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet());
                    if (positionIntegerMutablePair.right > bitSetRecyclable.size()) {
                        bitSetRecyclable.set(positionIntegerMutablePair.right);
                    }
                    bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size());
                    long[] ackSetOverlap = bitSetRecyclable.toLongArray();
                    bitSetRecyclable.recycle();
                    if (isAckSetOverlap(ackSetOverlap, ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                    if (individualAckPositions != null && individualAckPositions.containsKey(position) && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack batch message:" + position + " in pending ack status.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                } else {
                    if (individualAckPositions != null && individualAckPositions.containsKey(position)) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " in pending ack status.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                }
            }
            handleIndividualAck(txnID, positions);
            completableFuture.complete(null);
        }
    }).exceptionally(e -> {
        synchronized (PendingAckHandleImpl.this) {
            // we also modify the in memory state when append fail,
            // because we don't know the persistent state, when were replay it,
            // it will produce the wrong operation. so we append fail,
            // we should wait tc time out or client abort this transaction.
            handleIndividualAck(txnID, positions);
            completableFuture.completeExceptionally(e.getCause());
        }
        return null;
    })).exceptionally(e -> {
        completableFuture.completeExceptionally(e);
        return null;
    });
}

Example 3 with PendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.PendingAckStore in project incubator-pulsar by apache.

the class PendingAckHandleImpl method internalCumulativeAcknowledgeMessage.

public void internalCumulativeAcknowledgeMessage(TxnID txnID, List<PositionImpl> positions, CompletableFuture<Void> completableFuture) {
    if (txnID == null) {
        completableFuture.completeExceptionally(new NotAllowedException("TransactionID can not be null."));
        return;
    }
    if (positions == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    if (positions.size() != 1) {
        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " invalid cumulative ack received with multiple message ids.";
        log.error(errorMsg);
        completableFuture.completeExceptionally(new NotAllowedException(errorMsg));
        return;
    }
    PositionImpl position = positions.get(0);
    this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> {
        if (log.isDebugEnabled()) {
            log.debug("[{}] cumulativeAcknowledgeMessage position: [{}], " + "txnID:[{}], subName: [{}].", topicName, txnID, position, subName);
        }
        if (position.compareTo((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) <= 0) {
            String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to cumulative ack position: " + position + " within range of cursor's " + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition();
            log.error(errorMsg);
            completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
            return;
        }
        if (cumulativeAckOfTransaction != null && (!cumulativeAckOfTransaction.getKey().equals(txnID) || compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) <= 0)) {
            String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to cumulative batch ack position: " + position + " within range of current " + "currentPosition: " + cumulativeAckOfTransaction.getValue();
            log.error(errorMsg);
            completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
            return;
        }
        handleCumulativeAck(txnID, position);
        completableFuture.complete(null);
    }).exceptionally(e -> {
        // we also modify the in memory state when append fail, because we don't know the persistent
        // state, when wereplay it, it will produce the wrong operation. so we append fail, we should
        // wait tc time out or client abort this transaction.
        handleCumulativeAck(txnID, position);
        completableFuture.completeExceptionally(e.getCause());
        return null;
    })).exceptionally(e -> {
        completableFuture.completeExceptionally(e);
        return null;
    });
}

Example 4 with PendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.PendingAckStore in project incubator-pulsar by apache.

the class PendingAckHandleImpl method internalIndividualAcknowledgeMessage.

public void internalIndividualAcknowledgeMessage(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions, CompletableFuture<Void> completableFuture) {
    if (txnID == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    if (positions == null) {
        completableFuture.completeExceptionally(new NotAllowedException("Positions can not be null."));
        return;
    }
    this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> {
        synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) {
            for (MutablePair<PositionImpl, Integer> positionIntegerMutablePair : positions) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] individualAcknowledgeMessage position: [{}], " + "txnId: [{}], subName: [{}]", topicName, positionIntegerMutablePair.left, txnID, subName);
                }
                PositionImpl position = positionIntegerMutablePair.left;
                // normal acknowledge,throw exception.
                if (((ManagedCursorImpl) persistentSubscription.getCursor()).isMessageDeleted(position)) {
                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before.";
                    log.error(errorMsg);
                    completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                    return;
                }
                if (position.hasAckSet()) {
                    // in order to jude the bit set is over lap, so set the covering
                    // the batch size bit to 1,should know the two
                    // bit set don't have the same point is 0
                    BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet());
                    if (positionIntegerMutablePair.right > bitSetRecyclable.size()) {
                        bitSetRecyclable.set(positionIntegerMutablePair.right);
                    }
                    bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size());
                    long[] ackSetOverlap = bitSetRecyclable.toLongArray();
                    bitSetRecyclable.recycle();
                    if (isAckSetOverlap(ackSetOverlap, ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                    if (individualAckPositions != null && individualAckPositions.containsKey(position) && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack batch message:" + position + " in pending ack status.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                } else {
                    if (individualAckPositions != null && individualAckPositions.containsKey(position)) {
                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " in pending ack status.";
                        log.error(errorMsg);
                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
                        return;
                    }
                }
            }
            handleIndividualAck(txnID, positions);
            completableFuture.complete(null);
        }
    }).exceptionally(e -> {
        synchronized (PendingAckHandleImpl.this) {
            // we also modify the in memory state when append fail,
            // because we don't know the persistent state, when were replay it,
            // it will produce the wrong operation. so we append fail,
            // we should wait tc time out or client abort this transaction.
            handleIndividualAck(txnID, positions);
            completableFuture.completeExceptionally(e.getCause());
        }
        return null;
    })).exceptionally(e -> {
        completableFuture.completeExceptionally(e);
        return null;
    });
}

Example 5 with PendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.PendingAckStore in project incubator-pulsar by apache.

the class MLPendingAckStoreProvider method newPendingAckStore.

@Override
public CompletableFuture<PendingAckStore> newPendingAckStore(PersistentSubscription subscription) {
    CompletableFuture<PendingAckStore> pendingAckStoreFuture = new CompletableFuture<>();
    if (subscription == null) {
        pendingAckStoreFuture.completeExceptionally(new TransactionPendingAckException.TransactionPendingAckStoreProviderException("The subscription is null."));
        return pendingAckStoreFuture;
    }
    PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic();
    String pendingAckTopicName = MLPendingAckStore.getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subscription.getName());
    originPersistentTopic.getBrokerService().getManagedLedgerFactory().asyncExists(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding()).thenAccept(exist -> {
        TopicName topicName;
        if (exist) {
            topicName = TopicName.get(pendingAckTopicName);
        } else {
            topicName = TopicName.get(originPersistentTopic.getName());
        }
        originPersistentTopic.getBrokerService().getManagedLedgerConfig(topicName).thenAccept(config -> {
            config.setCreateIfMissing(true);
            originPersistentTopic.getBrokerService().getManagedLedgerFactory().asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), config, new AsyncCallbacks.OpenLedgerCallback() {

                @Override
                public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                    ledger.asyncOpenCursor(MLPendingAckStore.getTransactionPendingAckStoreCursorName(), InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() {

                        @Override
                        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
                            pendingAckStoreFuture.complete(new MLPendingAckStore(ledger, cursor, subscription.getCursor(), originPersistentTopic.getBrokerService().getPulsar().getConfiguration().getTransactionPendingAckLogIndexMinLag()));
                            if (log.isDebugEnabled()) {
                                log.debug("{},{} open MLPendingAckStore cursor success", originPersistentTopic.getName(), subscription.getName());
                            }
                        }

                        @Override
                        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
                            log.error("{},{} open MLPendingAckStore cursor failed.", originPersistentTopic.getName(), subscription.getName(), exception);
                            pendingAckStoreFuture.completeExceptionally(exception);
                        }
                    }, null);
                }

                @Override
                public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                    log.error("{}, {} open MLPendingAckStore managedLedger failed.", originPersistentTopic.getName(), subscription.getName(), exception);
                    pendingAckStoreFuture.completeExceptionally(exception);
                }
            }, () -> true, null);
        });
    }).exceptionally(e -> {
        log.error("Failed to obtain the existence of ManagerLedger with topic and subscription : " + originPersistentTopic.getSubscriptions() + "  " + subscription.getName());
        pendingAckStoreFuture.completeExceptionally(e.getCause());
        return null;
    });
    return pendingAckStoreFuture;
}