Examples with ManagedCursorContainer org.apache.bookkeeper.mledger.impl.ManagedCursorContainer used on opensource projects

Example 1 with ManagedCursorContainer

use of org.apache.bookkeeper.mledger.impl.ManagedCursorContainer in project pulsar by apache.

the class PersistentTopic method checkTimeBacklogExceeded.

/**
 * @return determine if backlog quota enforcement needs to be done for topic based on time limit
 */
public CompletableFuture<Boolean> checkTimeBacklogExceeded() {
    TopicName topicName = TopicName.get(getName());
    int backlogQuotaLimitInSecond = getBacklogQuota(BacklogQuotaType.message_age).getLimitTime();
    // If backlog quota by time is not set and we have no durable cursor.
    if (backlogQuotaLimitInSecond <= 0 || ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition() == null) {
        return CompletableFuture.completedFuture(false);
    }
    if (brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck()) {
        CompletableFuture<Boolean> future = new CompletableFuture<>();
        // Check if first unconsumed message(first message after mark delete position)
        // for slowest cursor's has expired.
        PositionImpl position = ((ManagedLedgerImpl) ledger).getNextValidPosition(((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition());
        ((ManagedLedgerImpl) ledger).asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() {

            @Override
            public void readEntryComplete(Entry entry, Object ctx) {
                try {
                    long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer());
                    boolean expired = MessageImpl.isEntryExpired(backlogQuotaLimitInSecond, entryTimestamp);
                    if (expired && log.isDebugEnabled()) {
                        log.debug("Time based backlog quota exceeded, oldest entry in cursor {}'s backlog" + "exceeded quota {}", ((ManagedLedgerImpl) ledger).getSlowestConsumer().getName(), backlogQuotaLimitInSecond);
                    }
                    future.complete(expired);
                } catch (Exception e) {
                    log.error("[{}][{}] Error deserializing message for backlog check", topicName, e);
                    future.complete(false);
                } finally {
                    entry.release();
                }
            }

            @Override
            public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
                log.error("[{}][{}] Error reading entry for precise time based  backlog check", topicName, exception);
                future.complete(false);
            }
        }, null);
        return future;
    } else {
        Long ledgerId = ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition().getLedgerId();
        try {
            org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledger.getLedgerInfo(ledgerId).get();
            if (ledgerInfo != null && ledgerInfo.hasTimestamp() && ledgerInfo.getTimestamp() > 0 && ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp() > backlogQuotaLimitInSecond * 1000) {
                if (log.isDebugEnabled()) {
                    log.debug("Time based backlog quota exceeded, quota {}, age of ledger " + "slowest cursor currently on {}", backlogQuotaLimitInSecond * 1000, ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp());
                }
                return CompletableFuture.completedFuture(true);
            } else {
                return CompletableFuture.completedFuture(false);
            }
        } catch (Exception e) {
            log.error("[{}][{}] Error reading entry for precise time based backlog check", topicName, e);
            return CompletableFuture.completedFuture(false);
        }
    }
}

Example 2 with ManagedCursorContainer

use of org.apache.bookkeeper.mledger.impl.ManagedCursorContainer in project pulsar by apache.

the class ManagedCursorMetrics method aggregate.

/**
 * Aggregation by namespace, ledger, cursor.
 *
 * @return List<Metrics>
 */
private List<Metrics> aggregate() {
    metricsCollection.clear();
    for (Map.Entry<String, ManagedLedgerImpl> e : getManagedLedgers().entrySet()) {
        String ledgerName = e.getKey();
        ManagedLedgerImpl ledger = e.getValue();
        String namespace = parseNamespaceFromLedgerName(ledgerName);
        ManagedCursorContainer cursorContainer = ledger.getCursors();
        Iterator<ManagedCursor> cursorIterator = cursorContainer.iterator();
        while (cursorIterator.hasNext()) {
            ManagedCursorImpl cursor = (ManagedCursorImpl) cursorIterator.next();
            ManagedCursorMXBean cStats = cursor.getStats();
            dimensionMap.clear();
            dimensionMap.put("namespace", namespace);
            dimensionMap.put("ledger_name", ledgerName);
            dimensionMap.put("cursor_name", cursor.getName());
            Metrics metrics = createMetrics(dimensionMap);
            metrics.put("brk_ml_cursor_nonContiguousDeletedMessagesRange", (long) cursor.getTotalNonContiguousDeletedMessagesRange());
            metrics.put("brk_ml_cursor_persistLedgerSucceed", cStats.getPersistLedgerSucceed());
            metrics.put("brk_ml_cursor_persistLedgerErrors", cStats.getPersistLedgerErrors());
            metrics.put("brk_ml_cursor_persistZookeeperSucceed", cStats.getPersistZookeeperSucceed());
            metrics.put("brk_ml_cursor_persistZookeeperErrors", cStats.getPersistZookeeperErrors());
            metrics.put("brk_ml_cursor_writeLedgerSize", cStats.getWriteCursorLedgerSize());
            metrics.put("brk_ml_cursor_writeLedgerLogicalSize", cStats.getWriteCursorLedgerLogicalSize());
            metrics.put("brk_ml_cursor_readLedgerSize", cStats.getReadCursorLedgerSize());
            metricsCollection.add(metrics);
        }
    }
    return metricsCollection;
}

Example 3 with ManagedCursorContainer

use of org.apache.bookkeeper.mledger.impl.ManagedCursorContainer in project pulsar by yahoo.

the class TransactionTest method testEndTBRecoveringWhenManagerLedgerDisReadable.

@Test
public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception {
    String topic = NAMESPACE1 + "/testEndTBRecoveringWhenManagerLedgerDisReadable";
    admin.topics().createNonPartitionedTopic(topic);
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).producerName("test").enableBatching(false).sendTimeout(0, TimeUnit.SECONDS).topic(topic).create();
    Transaction txn = pulsarClient.newTransaction().withTransactionTimeout(10, TimeUnit.SECONDS).build().get();
    producer.newMessage(txn).value("test").send();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic("persistent://" + topic, false).get().get();
    persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true);
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn("transaction-buffer-sub").when(managedCursor).getName();
    doReturn(true).when(managedCursor).hasMoreEntries();
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    Class<ManagedLedgerImpl> managedLedgerClass = ManagedLedgerImpl.class;
    Field field = managedLedgerClass.getDeclaredField("cursors");
    field.setAccessible(true);
    ManagedCursorContainer managedCursors = (ManagedCursorContainer) field.get(persistentTopic.getManagedLedger());
    managedCursors.removeCursor("transaction-buffer-sub");
    managedCursors.add(managedCursor);
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    TransactionBuffer buffer2 = new TopicTransactionBuffer(persistentTopic);
    Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer2.getStats().state, "Ready"));
    managedCursors.removeCursor("transaction-buffer-sub");
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    managedCursors.add(managedCursor);
    TransactionBuffer buffer3 = new TopicTransactionBuffer(persistentTopic);
    Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer3.getStats().state, "Ready"));
    persistentTopic.getInternalStats(false).thenAccept(internalStats -> {
        assertTrue(internalStats.cursors.isEmpty());
    });
    managedCursors.removeCursor("transaction-buffer-sub");
}

Example 4 with ManagedCursorContainer

use of org.apache.bookkeeper.mledger.impl.ManagedCursorContainer in project incubator-pulsar by apache.

the class TransactionTest method testEndTBRecoveringWhenManagerLedgerDisReadable.

@Test
public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception {
    String topic = NAMESPACE1 + "/testEndTBRecoveringWhenManagerLedgerDisReadable";
    admin.topics().createNonPartitionedTopic(topic);
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).producerName("test").enableBatching(false).sendTimeout(0, TimeUnit.SECONDS).topic(topic).create();
    Transaction txn = pulsarClient.newTransaction().withTransactionTimeout(10, TimeUnit.SECONDS).build().get();
    producer.newMessage(txn).value("test").send();
    PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService().getTopic("persistent://" + topic, false).get().get();
    persistentTopic.getManagedLedger().getConfig().setAutoSkipNonRecoverableData(true);
    ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class);
    doReturn("transaction-buffer-sub").when(managedCursor).getName();
    doReturn(true).when(managedCursor).hasMoreEntries();
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    Class<ManagedLedgerImpl> managedLedgerClass = ManagedLedgerImpl.class;
    Field field = managedLedgerClass.getDeclaredField("cursors");
    field.setAccessible(true);
    ManagedCursorContainer managedCursors = (ManagedCursorContainer) field.get(persistentTopic.getManagedLedger());
    managedCursors.removeCursor("transaction-buffer-sub");
    managedCursors.add(managedCursor);
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.ManagedLedgerFencedException(), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    TransactionBuffer buffer2 = new TopicTransactionBuffer(persistentTopic);
    Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer2.getStats().state, "Ready"));
    managedCursors.removeCursor("transaction-buffer-sub");
    doAnswer(invocation -> {
        AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1);
        callback.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("test"), null);
        return null;
    }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any());
    managedCursors.add(managedCursor);
    TransactionBuffer buffer3 = new TopicTransactionBuffer(persistentTopic);
    Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(buffer3.getStats().state, "Ready"));
    persistentTopic.getInternalStats(false).thenAccept(internalStats -> {
        assertTrue(internalStats.cursors.isEmpty());
    });
    managedCursors.removeCursor("transaction-buffer-sub");
}

Example 5 with ManagedCursorContainer

use of org.apache.bookkeeper.mledger.impl.ManagedCursorContainer in project incubator-pulsar by apache.

the class PersistentTopic method checkTimeBacklogExceeded.

/**
 * @return determine if backlog quota enforcement needs to be done for topic based on time limit
 */
public CompletableFuture<Boolean> checkTimeBacklogExceeded() {
    TopicName topicName = TopicName.get(getName());
    int backlogQuotaLimitInSecond = getBacklogQuota(BacklogQuotaType.message_age).getLimitTime();
    // If backlog quota by time is not set and we have no durable cursor.
    if (backlogQuotaLimitInSecond <= 0 || ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition() == null) {
        return CompletableFuture.completedFuture(false);
    }
    if (brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck()) {
        CompletableFuture<Boolean> future = new CompletableFuture<>();
        // Check if first unconsumed message(first message after mark delete position)
        // for slowest cursor's has expired.
        PositionImpl position = ((ManagedLedgerImpl) ledger).getNextValidPosition(((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition());
        ((ManagedLedgerImpl) ledger).asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() {

            @Override
            public void readEntryComplete(Entry entry, Object ctx) {
                try {
                    long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer());
                    boolean expired = MessageImpl.isEntryExpired(backlogQuotaLimitInSecond, entryTimestamp);
                    if (expired && log.isDebugEnabled()) {
                        log.debug("Time based backlog quota exceeded, oldest entry in cursor {}'s backlog" + "exceeded quota {}", ((ManagedLedgerImpl) ledger).getSlowestConsumer().getName(), backlogQuotaLimitInSecond);
                    }
                    future.complete(expired);
                } catch (Exception e) {
                    log.error("[{}][{}] Error deserializing message for backlog check", topicName, e);
                    future.complete(false);
                } finally {
                    entry.release();
                }
            }

            @Override
            public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
                log.error("[{}][{}] Error reading entry for precise time based  backlog check", topicName, exception);
                future.complete(false);
            }
        }, null);
        return future;
    } else {
        PositionImpl slowestPosition = ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition();
        try {
            return slowestReaderTimeBasedBacklogQuotaCheck(slowestPosition);
        } catch (Exception e) {
            log.error("[{}][{}] Error reading entry for precise time based backlog check", topicName, e);
            return CompletableFuture.completedFuture(false);
        }
    }
}