Examples with TopicTransactionBuffer org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer used on opensource projects

Example 1 with TopicTransactionBuffer

use of org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer in project pulsar by apache.

the class TransactionStablePositionTest method testSyncNormalPositionWhenTBRecover.

@Test(dataProvider = "enableTransactionAndState")
public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, TopicTransactionBufferState.State state) throws Exception {
    final String topicName = NAMESPACE1 + "/testSyncNormalPositionWhenTBRecover-" + clientEnableTransaction + state.name();
    pulsarClient = PulsarClient.builder().serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()).statsInterval(0, TimeUnit.SECONDS).enableTransaction(clientEnableTransaction).build();
    @Cleanup Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES).sendTimeout(0, TimeUnit.SECONDS).topic(topicName).create();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic(TopicName.get(topicName).toString(), false).get().get();
    TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) persistentTopic.getTransactionBuffer();
    // wait topic transaction buffer recover success
    checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer);
    Field field = TopicTransactionBufferState.class.getDeclaredField("state");
    field.setAccessible(true);
    field.set(topicTransactionBuffer, state);
    // init maxReadPosition is PositionImpl.EARLIEST
    Position position = topicTransactionBuffer.getMaxReadPosition();
    assertEquals(position, PositionImpl.EARLIEST);
    MessageIdImpl messageId = (MessageIdImpl) producer.send("test".getBytes());
    // send normal message can't change MaxReadPosition when state is None or Initializing
    position = topicTransactionBuffer.getMaxReadPosition();
    assertEquals(position, PositionImpl.EARLIEST);
    // invoke recover
    Method method = TopicTransactionBuffer.class.getDeclaredMethod("recover");
    method.setAccessible(true);
    method.invoke(topicTransactionBuffer);
    // change to None state can recover
    field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None);
    // recover success again
    checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer);
    // change MaxReadPosition to normal message position
    assertEquals(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()), topicTransactionBuffer.getMaxReadPosition());
}

Example 2 with TopicTransactionBuffer

use of org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer in project pulsar by apache.

the class TransactionTest method testNoEntryCanBeReadWhenRecovery.

@Test
public void testNoEntryCanBeReadWhenRecovery() throws Exception {
    String topic = NAMESPACE1 + "/test";
    PersistentTopic persistentTopic = (PersistentTopic) pulsarServiceList.get(0).getBrokerService().getTopic(TopicName.get(topic).toString(), true).get().get();
    Class<PersistentTopic> persistentTopicClass = PersistentTopic.class;
    Field filed1 = persistentTopicClass.getDeclaredField("ledger");
    Field field2 = persistentTopicClass.getDeclaredField("transactionBuffer");
    filed1.setAccessible(true);
    field2.setAccessible(true);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) spy(filed1.get(persistentTopic));
    filed1.set(persistentTopic, managedLedger);
    TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field2.get(persistentTopic);
    Method method = TopicTransactionBuffer.class.getDeclaredMethod("takeSnapshot");
    method.setAccessible(true);
    CompletableFuture<Void> completableFuture = (CompletableFuture<Void>) method.invoke(topicTransactionBuffer);
    completableFuture.get();
    doReturn(PositionImpl.LATEST).when(managedLedger).getLastConfirmedEntry();
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn(false).when(managedCursor).hasMoreEntries();
    doReturn(managedCursor).when(managedLedger).newNonDurableCursor(any(), any());
    TopicTransactionBuffer transactionBuffer = new TopicTransactionBuffer(persistentTopic);
    Awaitility.await().untilAsserted(() -> Assert.assertTrue(transactionBuffer.checkIfReady()));
}

Example 3 with TopicTransactionBuffer

use of org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer in project pulsar by apache.

the class TopicTransactionBufferRecoverTest method testTopicTransactionBufferDeleteAbort.

@Test
private void testTopicTransactionBufferDeleteAbort() throws Exception {
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(ABORT_DELETE).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create();
    @Cleanup Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(ABORT_DELETE).subscriptionName(SUBSCRIPTION_NAME).subscribe();
    Transaction tnx = pulsarClient.newTransaction().withTransactionTimeout(2, TimeUnit.SECONDS).build().get();
    String value = "Hello Pulsar!";
    MessageId messageId1 = producer.newMessage(tnx).value(value).send();
    tnx.abort().get();
    admin.topics().unload(ABORT_DELETE);
    tnx = pulsarClient.newTransaction().withTransactionTimeout(2, TimeUnit.SECONDS).build().get();
    value = "Hello";
    producer.newMessage(tnx).value(value).send();
    tnx.commit().get();
    Message<String> message = consumer.receive(2, TimeUnit.SECONDS);
    System.out.println("consumer receive message" + message.getMessageId());
    assertNotNull(message.getValue(), value);
    consumer.acknowledge(message);
    tnx = pulsarClient.newTransaction().withTransactionTimeout(2, TimeUnit.SECONDS).build().get();
    MessageId messageId2 = producer.newMessage(tnx).value(value).send();
    tnx.abort().get();
    assertTrue(((MessageIdImpl) messageId2).getLedgerId() != ((MessageIdImpl) messageId1).getLedgerId());
    boolean exist = false;
    for (int i = 0; i < getPulsarServiceList().size(); i++) {
        Field field = BrokerService.class.getDeclaredField("topics");
        field.setAccessible(true);
        ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>> topics = (ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>>) field.get(getPulsarServiceList().get(i).getBrokerService());
        CompletableFuture<Optional<Topic>> completableFuture = topics.get("persistent://" + ABORT_DELETE);
        if (completableFuture != null) {
            Optional<Topic> topic = completableFuture.get();
            if (topic.isPresent()) {
                PersistentTopic persistentTopic = (PersistentTopic) topic.get();
                field = ManagedLedgerImpl.class.getDeclaredField("ledgers");
                field.setAccessible(true);
                NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers = (NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo>) field.get(persistentTopic.getManagedLedger());
                ledgers.remove(((MessageIdImpl) messageId1).getLedgerId());
                tnx = pulsarClient.newTransaction().withTransactionTimeout(2, TimeUnit.SECONDS).build().get();
                producer.newMessage(tnx).value(value).send();
                tnx.commit().get();
                field = PersistentTopic.class.getDeclaredField("transactionBuffer");
                field.setAccessible(true);
                TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field.get(persistentTopic);
                field = TopicTransactionBuffer.class.getDeclaredField("aborts");
                field.setAccessible(true);
                LinkedMap<TxnID, PositionImpl> linkedMap = (LinkedMap<TxnID, PositionImpl>) field.get(topicTransactionBuffer);
                assertEquals(linkedMap.size(), 1);
                assertEquals(linkedMap.get(linkedMap.firstKey()).getLedgerId(), ((MessageIdImpl) message.getMessageId()).getLedgerId());
                exist = true;
            }
        }
    }
    assertTrue(exist);
}

Example 4 with TopicTransactionBuffer

use of org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer in project pulsar by apache.

the class TopicTransactionBufferRecoverTest method recoverTest.

@Test(dataProvider = "testTopic")
private void recoverTest(String testTopic) throws Exception {
    PulsarClient pulsarClient = this.pulsarClient;
    Transaction tnx1 = pulsarClient.newTransaction().withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
    Transaction tnx2 = pulsarClient.newTransaction().withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
    @Cleanup Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(testTopic).subscriptionName(SUBSCRIPTION_NAME).subscribe();
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(testTopic).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create();
    int messageCnt = 10;
    String content = "Hello Txn - ";
    for (int i = 0; i < messageCnt; i++) {
        String msg = content + i;
        if (i % 2 == 0) {
            MessageId messageId = producer.newMessage(tnx1).value(msg).send();
            log.info("Txn1 send message : {}, messageId : {}", msg, messageId);
        } else {
            MessageId messageId = producer.newMessage(tnx2).value(msg).send();
            log.info("Txn2 send message : {}, messageId : {}", msg, messageId);
        }
    }
    Message<String> message = consumer.receive(2, TimeUnit.SECONDS);
    assertNull(message);
    tnx1.commit();
    // only can receive message 1
    message = consumer.receive(2, TimeUnit.SECONDS);
    assertNotNull(message);
    log.info("Txn1 commit receive message : {}, messageId : {}", message.getValue(), message.getMessageId());
    consumer.acknowledge(message);
    // can't receive message
    message = consumer.receive(2, TimeUnit.SECONDS);
    assertNull(message);
    admin.topics().unload(testTopic);
    Awaitility.await().until(() -> {
        for (int i = 0; i < getPulsarServiceList().size(); i++) {
            Field field = BrokerService.class.getDeclaredField("topics");
            field.setAccessible(true);
            ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>> topics = (ConcurrentOpenHashMap<String, CompletableFuture<Optional<Topic>>>) field.get(getPulsarServiceList().get(i).getBrokerService());
            CompletableFuture<Optional<Topic>> completableFuture = topics.get("persistent://" + testTopic);
            if (completableFuture != null) {
                Optional<Topic> topic = completableFuture.get();
                if (topic.isPresent()) {
                    PersistentTopic persistentTopic = (PersistentTopic) topic.get();
                    field = PersistentTopic.class.getDeclaredField("transactionBuffer");
                    field.setAccessible(true);
                    TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field.get(persistentTopic);
                    if (topicTransactionBuffer.checkIfReady()) {
                        return true;
                    } else {
                        return false;
                    }
                }
            }
        }
        return false;
    });
    if (testTopic.equals(RECOVER_COMMIT)) {
        tnx2.commit().get();
        for (int i = messageCnt; i > 1; i--) {
            message = consumer.receive();
            log.info("Txn2 commit receive message : {}, messageId : {}", message.getValue(), message.getMessageId());
            consumer.acknowledge(message);
        }
        // can't receive message
        message = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(message);
    } else {
        tnx2.abort().get();
        for (int i = messageCnt / 2; i > 1; i--) {
            message = consumer.receive();
            log.info("Txn2 commit receive message : {}, messageId : {}", message.getValue(), message.getMessageId());
            consumer.acknowledge(message);
        }
        // can't receive message
        message = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(message);
    }
    consumer.close();
    producer.close();
}

Example 5 with TopicTransactionBuffer

use of org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer in project pulsar by yahoo.

the class TransactionTest method testNoEntryCanBeReadWhenRecovery.

@Test
public void testNoEntryCanBeReadWhenRecovery() throws Exception {
    String topic = NAMESPACE1 + "/test";
    PersistentTopic persistentTopic = (PersistentTopic) pulsarServiceList.get(0).getBrokerService().getTopic(TopicName.get(topic).toString(), true).get().get();
    Class<PersistentTopic> persistentTopicClass = PersistentTopic.class;
    Field filed1 = persistentTopicClass.getDeclaredField("ledger");
    Field field2 = persistentTopicClass.getDeclaredField("transactionBuffer");
    filed1.setAccessible(true);
    field2.setAccessible(true);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) spy(filed1.get(persistentTopic));
    filed1.set(persistentTopic, managedLedger);
    TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field2.get(persistentTopic);
    Method method = TopicTransactionBuffer.class.getDeclaredMethod("takeSnapshot");
    method.setAccessible(true);
    CompletableFuture<Void> completableFuture = (CompletableFuture<Void>) method.invoke(topicTransactionBuffer);
    completableFuture.get();
    doReturn(PositionImpl.LATEST).when(managedLedger).getLastConfirmedEntry();
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn(false).when(managedCursor).hasMoreEntries();
    doReturn(managedCursor).when(managedLedger).newNonDurableCursor(any(), any());
    TopicTransactionBuffer transactionBuffer = new TopicTransactionBuffer(persistentTopic);
    Awaitility.await().untilAsserted(() -> Assert.assertTrue(transactionBuffer.checkIfReady()));
}