Examples with PendingAckHandleImpl org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl used on opensource projects

Example 1 with PendingAckHandleImpl

use of org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl in project pulsar by apache.

the class TransactionLowWaterMarkTest method testPendingAckLowWaterMark.

@Test
public void testPendingAckLowWaterMark() throws Exception {
    String subName = "test";
    Transaction txn = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.SECONDS).build().get();
    @Cleanup Producer<byte[]> producer = pulsarClient.newProducer().topic(TOPIC).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create();
    @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(TOPIC).subscriptionName(subName).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).enableBatchIndexAcknowledgment(true).subscriptionType(SubscriptionType.Failover).subscribe();
    final String TEST1 = "test1";
    final String TEST2 = "test2";
    final String TEST3 = "test3";
    producer.send(TEST1.getBytes());
    producer.send(TEST2.getBytes());
    producer.send(TEST3.getBytes());
    Message<byte[]> message = consumer.receive(2, TimeUnit.SECONDS);
    assertEquals(new String(message.getData()), TEST1);
    consumer.acknowledgeAsync(message.getMessageId(), txn).get();
    LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>> individualAckOfTransaction = null;
    for (int i = 0; i < getPulsarServiceList().size(); i++) {
        Field field = BrokerService.class.getDeclaredField("topics");
        field.setAccessible(true);
        ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>> topics = (ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>>) field.get(getPulsarServiceList().get(i).getBrokerService());
        CompletableFuture<Optional<Topic>> completableFuture = topics.get("persistent://" + TOPIC);
        if (completableFuture != null) {
            Optional<Topic> topic = completableFuture.get();
            if (topic.isPresent()) {
                PersistentSubscription persistentSubscription = (PersistentSubscription) topic.get().getSubscription(subName);
                field = PersistentSubscription.class.getDeclaredField("pendingAckHandle");
                field.setAccessible(true);
                PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription);
                field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction");
                field.setAccessible(true);
                individualAckOfTransaction = (LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>>) field.get(pendingAckHandle);
            }
        }
    }
    assertTrue(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) txn).getTxnIdMostBits(), ((TransactionImpl) txn).getTxnIdLeastBits())));
    txn.commit().get();
    Field field = TransactionImpl.class.getDeclaredField("state");
    field.setAccessible(true);
    field.set(txn, TransactionImpl.State.OPEN);
    assertFalse(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) txn).getTxnIdMostBits(), ((TransactionImpl) txn).getTxnIdLeastBits())));
    message = consumer.receive();
    assertEquals(new String(message.getData()), TEST2);
    consumer.acknowledgeAsync(message.getMessageId(), txn).get();
    assertTrue(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) txn).getTxnIdMostBits(), ((TransactionImpl) txn).getTxnIdLeastBits())));
    PartitionedTopicMetadata partitionedTopicMetadata = ((PulsarClientImpl) pulsarClient).getLookup().getPartitionedTopicMetadata(TopicName.TRANSACTION_COORDINATOR_ASSIGN).get();
    Transaction lowWaterMarkTxn = null;
    for (int i = 0; i < partitionedTopicMetadata.partitions; i++) {
        lowWaterMarkTxn = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.SECONDS).build().get();
        if (((TransactionImpl) lowWaterMarkTxn).getTxnIdMostBits() == ((TransactionImpl) txn).getTxnIdMostBits()) {
            break;
        }
    }
    if (lowWaterMarkTxn != null && ((TransactionImpl) lowWaterMarkTxn).getTxnIdMostBits() == ((TransactionImpl) txn).getTxnIdMostBits()) {
        producer.newMessage(lowWaterMarkTxn).value(TEST3.getBytes()).send();
        message = consumer.receive(2, TimeUnit.SECONDS);
        assertEquals(new String(message.getData()), TEST3);
        consumer.acknowledgeAsync(message.getMessageId(), lowWaterMarkTxn).get();
        assertTrue(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) txn).getTxnIdMostBits(), ((TransactionImpl) txn).getTxnIdLeastBits())));
        assertTrue(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) lowWaterMarkTxn).getTxnIdMostBits(), ((TransactionImpl) lowWaterMarkTxn).getTxnIdLeastBits())));
        lowWaterMarkTxn.commit().get();
        assertFalse(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) txn).getTxnIdMostBits(), ((TransactionImpl) txn).getTxnIdLeastBits())));
        assertFalse(individualAckOfTransaction.containsKey(new TxnID(((TransactionImpl) lowWaterMarkTxn).getTxnIdMostBits(), ((TransactionImpl) lowWaterMarkTxn).getTxnIdLeastBits())));
    } else {
        fail();
    }
}

Example 2 with PendingAckHandleImpl

use of org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl in project pulsar by apache.

the class TransactionTest method testEndTPRecoveringWhenManagerLedgerDisReadable.

@Test
public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception {
    String topic = NAMESPACE1 + "/testEndTPRecoveringWhenManagerLedgerDisReadable";
    admin.topics().createNonPartitionedTopic(topic);
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).producerName("test").enableBatching(false).sendTimeout(0, TimeUnit.SECONDS).topic(topic).create();
    producer.newMessage().send();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic(topic, false).get().get();
    persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true);
    PersistentSubscription persistentSubscription = (PersistentSubscription) persistentTopic.createSubscription("test", CommandSubscribe.InitialPosition.Earliest, false).get();
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn(true).when(managedCursor).hasMoreEntries();
    doReturn(false).when(managedCursor).isClosed();
    doReturn(new PositionImpl(-1, -1)).when(managedCursor).getMarkDeletedPosition();
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class);
    doReturn(CompletableFuture.completedFuture(new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null))).when(pendingAckStoreProvider).newPendingAckStore(any());
    doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any());
    Class<PulsarService> pulsarServiceClass = PulsarService.class;
    Field field = pulsarServiceClass.getDeclaredField("transactionPendingAckStoreProvider");
    field.setAccessible(true);
    field.set(getPulsarServiceList().get(0), pendingAckStoreProvider);
    PendingAckHandleImpl pendingAckHandle1 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle1.getStats().state, "Ready"));
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    PendingAckHandleImpl pendingAckHandle2 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle2.getStats().state, "Ready"));
}

Example 3 with PendingAckHandleImpl

use of org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl in project pulsar by apache.

the class TransactionProduceTest method getPendingAckCount.

private int getPendingAckCount(String topic, String subscriptionName) throws Exception {
    Class<PersistentSubscription> clazz = PersistentSubscription.class;
    int pendingAckCount = 0;
    for (PulsarService pulsarService : getPulsarServiceList()) {
        for (String key : pulsarService.getBrokerService().getTopics().keys()) {
            if (key.contains(topic)) {
                Field field = clazz.getDeclaredField("pendingAckHandle");
                field.setAccessible(true);
                PersistentSubscription subscription = (PersistentSubscription) pulsarService.getBrokerService().getTopics().get(key).get().get().getSubscription(subscriptionName);
                PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field.get(subscription);
                field = PendingAckHandleImpl.class.getDeclaredField("individualAckPositions");
                field.setAccessible(true);
                Map<PositionImpl, MutablePair<PositionImpl, Long>> map = (Map<PositionImpl, MutablePair<PositionImpl, Long>>) field.get(pendingAckHandle);
                if (map != null) {
                    pendingAckCount += map.size();
                }
            }
        }
    }
    log.info("subscriptionName: {}, pendingAckCount: {}", subscriptionName, pendingAckCount);
    return pendingAckCount;
}

Example 4 with PendingAckHandleImpl

use of org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl in project pulsar by apache.

the class PersistentSubscriptionTest method setup.

@BeforeMethod
public void setup() throws Exception {
    executor = OrderedExecutor.newBuilder().numThreads(1).name("persistent-subscription-test").build();
    eventLoopGroup = new NioEventLoopGroup();
    ServiceConfiguration svcConfig = spy(ServiceConfiguration.class);
    svcConfig.setBrokerShutdownTimeoutMs(0L);
    svcConfig.setTransactionCoordinatorEnabled(true);
    svcConfig.setClusterName("pulsar-cluster");
    pulsarMock = spyWithClassAndConstructorArgs(PulsarService.class, svcConfig);
    PulsarResources pulsarResources = mock(PulsarResources.class);
    doReturn(pulsarResources).when(pulsarMock).getPulsarResources();
    NamespaceResources namespaceResources = mock(NamespaceResources.class);
    doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
    doReturn(Optional.of(new Policies())).when(namespaceResources).getPoliciesIfCached(any());
    doReturn(new InMemTransactionBufferProvider()).when(pulsarMock).getTransactionBufferProvider();
    doReturn(new TransactionPendingAckStoreProvider() {

        @Override
        public CompletableFuture<PendingAckStore> newPendingAckStore(PersistentSubscription subscription) {
            return CompletableFuture.completedFuture(new PendingAckStore() {

                @Override
                public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService) {
                    try {
                        Field field = PendingAckHandleState.class.getDeclaredField("state");
                        field.setAccessible(true);
                        field.set(pendingAckHandle, PendingAckHandleState.State.Ready);
                    } catch (NoSuchFieldException | IllegalAccessException e) {
                        fail();
                    }
                }

                @Override
                public CompletableFuture<Void> closeAsync() {
                    return CompletableFuture.completedFuture(null);
                }

                @Override
                public CompletableFuture<Void> appendIndividualAck(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions) {
                    return CompletableFuture.completedFuture(null);
                }

                @Override
                public CompletableFuture<Void> appendCumulativeAck(TxnID txnID, PositionImpl position) {
                    return CompletableFuture.completedFuture(null);
                }

                @Override
                public CompletableFuture<Void> appendCommitMark(TxnID txnID, AckType ackType) {
                    return CompletableFuture.completedFuture(null);
                }

                @Override
                public CompletableFuture<Void> appendAbortMark(TxnID txnID, AckType ackType) {
                    return CompletableFuture.completedFuture(null);
                }
            });
        }

        @Override
        public CompletableFuture<Boolean> checkInitializedBefore(PersistentSubscription subscription) {
            return CompletableFuture.completedFuture(true);
        }
    }).when(pulsarMock).getTransactionPendingAckStoreProvider();
    doReturn(svcConfig).when(pulsarMock).getConfiguration();
    doReturn(mock(Compactor.class)).when(pulsarMock).getCompactor();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    doReturn(mlFactoryMock).when(pulsarMock).getManagedLedgerFactory();
    ZooKeeper zkMock = createMockZooKeeper();
    doReturn(createMockBookKeeper(executor)).when(pulsarMock).getBookKeeperClient();
    store = new ZKMetadataStore(zkMock);
    doReturn(store).when(pulsarMock).getLocalMetadataStore();
    doReturn(store).when(pulsarMock).getConfigurationMetadataStore();
    brokerMock = spyWithClassAndConstructorArgs(BrokerService.class, pulsarMock, eventLoopGroup);
    doNothing().when(brokerMock).unloadNamespaceBundlesGracefully();
    doReturn(brokerMock).when(pulsarMock).getBrokerService();
    ledgerMock = mock(ManagedLedgerImpl.class);
    cursorMock = mock(ManagedCursorImpl.class);
    managedLedgerConfigMock = mock(ManagedLedgerConfig.class);
    doReturn(new ManagedCursorContainer()).when(ledgerMock).getCursors();
    doReturn("mockCursor").when(cursorMock).getName();
    doReturn(new PositionImpl(1, 50)).when(cursorMock).getMarkDeletedPosition();
    doReturn(ledgerMock).when(cursorMock).getManagedLedger();
    doReturn(managedLedgerConfigMock).when(ledgerMock).getConfig();
    doReturn(false).when(managedLedgerConfigMock).isAutoSkipNonRecoverableData();
    topic = new PersistentTopic(successTopicName, ledgerMock, brokerMock);
    consumerMock = mock(Consumer.class);
    persistentSubscription = new PersistentSubscription(topic, subName, cursorMock, false);
}

Example 5 with PendingAckHandleImpl

use of org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl in project pulsar by apache.

the class PendingAckInMemoryDeleteTest method txnAckTestNoBatchAndSharedSubMemoryDeleteTest.

@Test
public void txnAckTestNoBatchAndSharedSubMemoryDeleteTest() throws Exception {
    String normalTopic = NAMESPACE1 + "/normal-topic";
    String subscriptionName = "test";
    @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(normalTopic).isAckReceiptEnabled(true).subscriptionName(subscriptionName).enableBatchIndexAcknowledgment(true).subscriptionType(SubscriptionType.Shared).ackTimeout(2, TimeUnit.SECONDS).acknowledgmentGroupTime(0, TimeUnit.MICROSECONDS).subscribe();
    @Cleanup Producer<byte[]> producer = pulsarClient.newProducer().topic(normalTopic).enableBatching(false).create();
    for (int retryCnt = 0; retryCnt < 2; retryCnt++) {
        int messageCnt = 1000;
        // produce normal messages
        for (int i = 0; i < messageCnt; i++) {
            producer.newMessage().value("hello".getBytes()).sendAsync();
        }
        Message<byte[]> message;
        Transaction commitTxn = getTxn();
        // send 1000 and ack 999, and test the consumer pending ack has already clear 999 messages
        for (int i = 0; i < messageCnt - 1; i++) {
            message = consumer.receive(2, TimeUnit.SECONDS);
            Assert.assertNotNull(message);
            if (i % 2 == 0) {
                consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get();
                log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i);
            } else {
                consumer.acknowledge(message.getMessageId());
                log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i);
            }
        }
        commitTxn.commit().get();
        int count = 0;
        for (int i = 0; i < getPulsarServiceList().size(); i++) {
            Field field = BrokerService.class.getDeclaredField("topics");
            field.setAccessible(true);
            ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>> topics = (ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>>) field.get(getPulsarServiceList().get(i).getBrokerService());
            CompletableFuture<Optional<Topic>> completableFuture = topics.get("persistent://" + normalTopic);
            if (completableFuture != null) {
                Optional<Topic> topic = completableFuture.get();
                if (topic.isPresent()) {
                    PersistentSubscription persistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName);
                    field = PersistentSubscription.class.getDeclaredField("pendingAckHandle");
                    field.setAccessible(true);
                    PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription);
                    field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction");
                    field.setAccessible(true);
                    LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>> individualAckOfTransaction = (LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>>) field.get(pendingAckHandle);
                    assertTrue(individualAckOfTransaction.isEmpty());
                    if (retryCnt == 0) {
                        // one message are not ack
                        assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
                    } else {
                        // two message are not ack
                        assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 2);
                    }
                    count++;
                }
            }
        }
        // make sure the consumer is ownership for a broker server
        assertEquals(count, 1);
    }
}