Examples with MLPendingAckStore org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore used on opensource projects

Example 1 with MLPendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore in project pulsar by apache.

the class TransactionTest method testSubscriptionRecreateTopic.

@Test
public void testSubscriptionRecreateTopic() throws PulsarAdminException, NoSuchFieldException, IllegalAccessException, PulsarClientException {
    String topic = "persistent://pulsar/system/testReCreateTopic";
    String subName = "sub_testReCreateTopic";
    int retentionSizeInMbSetTo = 5;
    int retentionSizeInMbSetTopic = 6;
    int retentionSizeInMinutesSetTo = 5;
    int retentionSizeInMinutesSetTopic = 6;
    admin.topics().createNonPartitionedTopic(topic);
    PulsarService pulsarService = super.getPulsarServiceList().get(0);
    pulsarService.getBrokerService().getTopics().clear();
    ManagedLedgerFactory managedLedgerFactory = pulsarService.getBrokerService().getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(managedLedgerFactory);
    ledgers.remove(TopicName.get(topic).getPersistenceNamingEncoding());
    try {
        admin.topics().createNonPartitionedTopic(topic);
        Assert.fail();
    } catch (PulsarAdminException.ConflictException e) {
        log.info("Cann`t create topic again");
    }
    admin.topics().setRetention(topic, new RetentionPolicies(retentionSizeInMinutesSetTopic, retentionSizeInMbSetTopic));
    pulsarClient.newConsumer().topic(topic).subscriptionName(subName).subscribe();
    pulsarService.getBrokerService().getTopicIfExists(topic).thenAccept(option -> {
        if (!option.isPresent()) {
            log.error("Failed o get Topic named: {}", topic);
            Assert.fail();
        }
        PersistentTopic originPersistentTopic = (PersistentTopic) option.get();
        String pendingAckTopicName = MLPendingAckStore.getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName);
        try {
            admin.topics().setRetention(pendingAckTopicName, new RetentionPolicies(retentionSizeInMinutesSetTo, retentionSizeInMbSetTo));
        } catch (PulsarAdminException e) {
            log.error("Failed to get./setRetention of topic with Exception:" + e);
            Assert.fail();
        }
        PersistentSubscription subscription = originPersistentTopic.getSubscription(subName);
        subscription.getPendingAckManageLedger().thenAccept(managedLedger -> {
            long retentionSize = managedLedger.getConfig().getRetentionSizeInMB();
            if (!originPersistentTopic.getTopicPolicies().isPresent()) {
                log.error("Failed to getTopicPolicies of :" + originPersistentTopic);
                Assert.fail();
            }
            TopicPolicies topicPolicies = originPersistentTopic.getTopicPolicies().get();
            Assert.assertEquals(retentionSizeInMbSetTopic, retentionSize);
            MLPendingAckStoreProvider mlPendingAckStoreProvider = new MLPendingAckStoreProvider();
            CompletableFuture<PendingAckStore> future = mlPendingAckStoreProvider.newPendingAckStore(subscription);
            future.thenAccept(pendingAckStore -> {
                ((MLPendingAckStore) pendingAckStore).getManagedLedger().thenAccept(managedLedger1 -> {
                    Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), retentionSizeInMbSetTo);
                });
            });
        });
    });
}

Example 2 with MLPendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore in project pulsar by apache.

the class TransactionTest method testEndTPRecoveringWhenManagerLedgerDisReadable.

@Test
public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception {
    String topic = NAMESPACE1 + "/testEndTPRecoveringWhenManagerLedgerDisReadable";
    admin.topics().createNonPartitionedTopic(topic);
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).producerName("test").enableBatching(false).sendTimeout(0, TimeUnit.SECONDS).topic(topic).create();
    producer.newMessage().send();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic(topic, false).get().get();
    persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true);
    PersistentSubscription persistentSubscription = (PersistentSubscription) persistentTopic.createSubscription("test", CommandSubscribe.InitialPosition.Earliest, false).get();
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn(true).when(managedCursor).hasMoreEntries();
    doReturn(false).when(managedCursor).isClosed();
    doReturn(new PositionImpl(-1, -1)).when(managedCursor).getMarkDeletedPosition();
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class);
    doReturn(CompletableFuture.completedFuture(new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null))).when(pendingAckStoreProvider).newPendingAckStore(any());
    doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any());
    Class<PulsarService> pulsarServiceClass = PulsarService.class;
    Field field = pulsarServiceClass.getDeclaredField("transactionPendingAckStoreProvider");
    field.setAccessible(true);
    field.set(getPulsarServiceList().get(0), pendingAckStoreProvider);
    PendingAckHandleImpl pendingAckHandle1 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle1.getStats().state, "Ready"));
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    PendingAckHandleImpl pendingAckHandle2 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle2.getStats().state, "Ready"));
}

Example 3 with MLPendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore in project pulsar by yahoo.

the class TransactionTest method testEndTPRecoveringWhenManagerLedgerDisReadable.

@Test
public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception {
    String topic = NAMESPACE1 + "/testEndTPRecoveringWhenManagerLedgerDisReadable";
    admin.topics().createNonPartitionedTopic(topic);
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).producerName("test").enableBatching(false).sendTimeout(0, TimeUnit.SECONDS).topic(topic).create();
    producer.newMessage().send();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic(topic, false).get().get();
    persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true);
    PersistentSubscription persistentSubscription = (PersistentSubscription) persistentTopic.createSubscription("test", CommandSubscribe.InitialPosition.Earliest, false, null).get();
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn(true).when(managedCursor).hasMoreEntries();
    doReturn(false).when(managedCursor).isClosed();
    doReturn(new PositionImpl(-1, -1)).when(managedCursor).getMarkDeletedPosition();
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class);
    doReturn(CompletableFuture.completedFuture(new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null, 500))).when(pendingAckStoreProvider).newPendingAckStore(any());
    doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any());
    Class<PulsarService> pulsarServiceClass = PulsarService.class;
    Field field = pulsarServiceClass.getDeclaredField("transactionPendingAckStoreProvider");
    field.setAccessible(true);
    field.set(getPulsarServiceList().get(0), pendingAckStoreProvider);
    PendingAckHandleImpl pendingAckHandle1 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle1.getStats().state, "Ready"));
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    PendingAckHandleImpl pendingAckHandle2 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle2.getStats().state, "Ready"));
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    PendingAckHandleImpl pendingAckHandle3 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle3.getStats().state, "Ready"));
}

Example 4 with MLPendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore in project pulsar by yahoo.

the class PendingAckMetadataTest method testPendingAckManageLedgerWriteFailState.

@Test
public void testPendingAckManageLedgerWriteFailState() throws Exception {
    ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
    factoryConf.setMaxCacheSize(0);
    String pendingAckTopicName = MLPendingAckStore.getTransactionPendingAckStoreSuffix("test", "test");
    @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf);
    CompletableFuture<ManagedLedger> completableFuture = new CompletableFuture<>();
    factory.asyncOpen(pendingAckTopicName, new AsyncCallbacks.OpenLedgerCallback() {

        @Override
        public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
            completableFuture.complete(ledger);
        }

        @Override
        public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
        }
    }, null);
    ManagedCursor cursor = completableFuture.get().openCursor("test");
    ManagedCursor subCursor = completableFuture.get().openCursor("test");
    MLPendingAckStore pendingAckStore = new MLPendingAckStore(completableFuture.get(), cursor, subCursor, 500);
    Field field = MLPendingAckStore.class.getDeclaredField("managedLedger");
    field.setAccessible(true);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) field.get(pendingAckStore);
    field = ManagedLedgerImpl.class.getDeclaredField("STATE_UPDATER");
    field.setAccessible(true);
    AtomicReferenceFieldUpdater<ManagedLedgerImpl, ManagedLedgerImpl.State> state = (AtomicReferenceFieldUpdater<ManagedLedgerImpl, ManagedLedgerImpl.State>) field.get(managedLedger);
    state.set(managedLedger, WriteFailed);
    try {
        pendingAckStore.appendAbortMark(new TxnID(1, 1), CommandAck.AckType.Cumulative).get();
        fail();
    } catch (ExecutionException e) {
        assertTrue(e.getCause().getCause() instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException);
    }
    pendingAckStore.appendAbortMark(new TxnID(1, 1), CommandAck.AckType.Cumulative).get();
    completableFuture.get().close();
    cursor.close();
    subCursor.close();
}

Example 5 with MLPendingAckStore

use of org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore in project incubator-pulsar by apache.

the class TransactionTest method testEndTPRecoveringWhenManagerLedgerDisReadable.

@Test
public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception {
    String topic = NAMESPACE1 + "/testEndTPRecoveringWhenManagerLedgerDisReadable";
    admin.topics().createNonPartitionedTopic(topic);
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).producerName("test").enableBatching(false).sendTimeout(0, TimeUnit.SECONDS).topic(topic).create();
    producer.newMessage().send();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic(topic, false).get().get();
    persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true);
    PersistentSubscription persistentSubscription = (PersistentSubscription) persistentTopic.createSubscription("test", CommandSubscribe.InitialPosition.Earliest, false, null).get();
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn(true).when(managedCursor).hasMoreEntries();
    doReturn(false).when(managedCursor).isClosed();
    doReturn(new PositionImpl(-1, -1)).when(managedCursor).getMarkDeletedPosition();
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    TransactionPendingAckStoreProvider pendingAckStoreProvider = mock(TransactionPendingAckStoreProvider.class);
    doReturn(CompletableFuture.completedFuture(new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null, 500))).when(pendingAckStoreProvider).newPendingAckStore(any());
    doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any());
    Class<PulsarService> pulsarServiceClass = PulsarService.class;
    Field field = pulsarServiceClass.getDeclaredField("transactionPendingAckStoreProvider");
    field.setAccessible(true);
    field.set(getPulsarServiceList().get(0), pendingAckStoreProvider);
    PendingAckHandleImpl pendingAckHandle1 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle1.getStats().state, "Ready"));
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    PendingAckHandleImpl pendingAckHandle2 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle2.getStats().state, "Ready"));
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    PendingAckHandleImpl pendingAckHandle3 = new PendingAckHandleImpl(persistentSubscription);
    Awaitility.await().untilAsserted(() -> assertEquals(pendingAckHandle3.getStats().state, "Ready"));
}