Examples with BitSetRecyclable org.apache.pulsar.common.util.collections.BitSetRecyclable used on opensource projects

Example 21 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class PositionAckSetUtilTest method andAckSetTest.

@Test
public void andAckSetTest() {
    PositionImpl positionOne = PositionImpl.get(1, 1);
    PositionImpl positionTwo = PositionImpl.get(1, 2);
    BitSet bitSetOne = new BitSet();
    BitSet bitSetTwo = new BitSet();
    bitSetOne.set(0);
    bitSetOne.set(2);
    bitSetOne.set(4);
    bitSetOne.set(6);
    bitSetOne.set(8);
    positionOne.setAckSet(bitSetOne.toLongArray());
    positionTwo.setAckSet(bitSetTwo.toLongArray());
    andAckSet(positionOne, positionTwo);
    BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(positionOne.getAckSet());
    assertTrue(bitSetRecyclable.isEmpty());
    bitSetTwo.set(2);
    bitSetTwo.set(4);
    positionOne.setAckSet(bitSetOne.toLongArray());
    positionTwo.setAckSet(bitSetTwo.toLongArray());
    andAckSet(positionOne, positionTwo);
    bitSetRecyclable = BitSetRecyclable.valueOf(positionOne.getAckSet());
    BitSetRecyclable bitSetRecyclableTwo = BitSetRecyclable.valueOf(bitSetTwo.toLongArray());
    assertEquals(bitSetRecyclable, bitSetRecyclableTwo);
}

Example 22 with BitSetRecyclable

use of org.apache.pulsar.common.util.collections.BitSetRecyclable in project pulsar by apache.

the class PendingAckInMemoryDeleteTest method txnAckTestBatchAndSharedSubMemoryDeleteTest.

@Test
public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception {
    String normalTopic = NAMESPACE1 + "/normal-topic";
    String subscriptionName = "test";
    @Cleanup Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(normalTopic).subscriptionName(subscriptionName).enableBatchIndexAcknowledgment(true).subscriptionType(SubscriptionType.Shared).subscribe();
    @Cleanup Producer<byte[]> producer = pulsarClient.newProducer().topic(normalTopic).enableBatching(true).batchingMaxMessages(200).create();
    PendingAckHandleImpl pendingAckHandle = null;
    LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>> individualAckOfTransaction = null;
    ManagedCursorImpl managedCursor = null;
    MessageId[] messageIds = new MessageId[2];
    for (int retryCnt = 0; retryCnt < 2; retryCnt++) {
        int messageCnt = 1000;
        // produce normal messages
        for (int i = 0; i < messageCnt; i++) {
            producer.newMessage().value("hello".getBytes()).sendAsync();
        }
        Message<byte[]> message;
        // after transaction abort, the messages could be received
        Transaction commitTxn = getTxn();
        // send 1000 and ack 999, and test the consumer pending ack has already clear 999 messages
        for (int i = 0; i < messageCnt; i++) {
            message = consumer.receive(2, TimeUnit.SECONDS);
            Assert.assertNotNull(message);
            // in order to free up 2 position to judge the consumer pending ack delete
            if (i != 500) {
                if (i % 2 == 0) {
                    consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get();
                    log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i);
                } else {
                    consumer.acknowledge(message.getMessageId());
                    log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i);
                }
            } else {
                messageIds[retryCnt] = message.getMessageId();
            }
        }
        commitTxn.commit().get();
        int count = 0;
        for (int i = 0; i < getPulsarServiceList().size(); i++) {
            Field field = BrokerService.class.getDeclaredField("topics");
            field.setAccessible(true);
            ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>> topics = (ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>>) field.get(getPulsarServiceList().get(i).getBrokerService());
            CompletableFuture<Optional<Topic>> completableFuture = topics.get("persistent://" + normalTopic);
            if (completableFuture != null) {
                Optional<Topic> topic = completableFuture.get();
                if (topic.isPresent()) {
                    PersistentSubscription testPersistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName);
                    field = PersistentSubscription.class.getDeclaredField("pendingAckHandle");
                    field.setAccessible(true);
                    pendingAckHandle = (PendingAckHandleImpl) field.get(testPersistentSubscription);
                    field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction");
                    field.setAccessible(true);
                    individualAckOfTransaction = (LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>>) field.get(pendingAckHandle);
                    assertTrue(individualAckOfTransaction.isEmpty());
                    managedCursor = (ManagedCursorImpl) testPersistentSubscription.getCursor();
                    field = ManagedCursorImpl.class.getDeclaredField("batchDeletedIndexes");
                    field.setAccessible(true);
                    final ConcurrentSkipListMap<PositionImpl, BitSetRecyclable> batchDeletedIndexes = (ConcurrentSkipListMap<PositionImpl, BitSetRecyclable>) field.get(managedCursor);
                    if (retryCnt == 0) {
                        // one message are not ack
                        Awaitility.await().until(() -> {
                            return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1;
                        });
                        assertEquals(batchDeletedIndexes.size(), 1);
                        assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
                    } else {
                        // two message are not ack
                        Awaitility.await().until(() -> {
                            return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 2;
                        });
                        Transaction commitTwice = getTxn();
                        // this message is in one batch point
                        consumer.acknowledge(messageIds[0]);
                        Awaitility.await().until(() -> {
                            return batchDeletedIndexes.size() == 1;
                        });
                        assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
                        // this test is for the last message has been cleared in this consumer pending acks
                        // and it won't clear the last message in cursor batch index ack set
                        consumer.acknowledgeAsync(messageIds[1], commitTwice).get();
                        assertEquals(batchDeletedIndexes.size(), 1);
                        assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0);
                        // the messages has been produced were all acked, the memory in broker for the messages has been cleared.
                        commitTwice.commit().get();
                        assertEquals(batchDeletedIndexes.size(), 0);
                        assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0);
                    }
                    count++;
                }
            }
        }
        assertEquals(count, 1);
    }
}