Examples with LedgerHandle org.apache.bookkeeper.client.LedgerHandle used on opensource projects

Example 1 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project pulsar by yahoo.

the class ManagedLedgerOfflineBacklog method calculateCursorBacklogs.

private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final DestinationName dn, final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers, final PersistentOfflineTopicStats offlineTopicStats) throws Exception {
    if (ledgers.size() == 0) {
        return;
    }
    String managedLedgerName = dn.getPersistenceNamingEncoding();
    MetaStore store = factory.getMetaStore();
    BookKeeper bk = factory.getBookKeeper();
    final CountDownLatch allCursorsCounter = new CountDownLatch(1);
    final long errorInReadingCursor = (long) -1;
    ConcurrentOpenHashMap<String, Long> ledgerRetryMap = new ConcurrentOpenHashMap<>();
    final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
    final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
    if (log.isDebugEnabled()) {
        log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition);
    }
    store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback<List<String>>() {

        @Override
        public void operationComplete(List<String> cursors, MetaStore.Stat v) {
            // Load existing cursors
            if (log.isDebugEnabled()) {
                log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size());
            }
            if (cursors.isEmpty()) {
                allCursorsCounter.countDown();
                return;
            }
            final CountDownLatch cursorCounter = new CountDownLatch(cursors.size());
            for (final String cursorName : cursors) {
                // determine subscription position from cursor ledger
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName);
                }
                AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> {
                    long ledgerId = lh.getId();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, cursorName, rc);
                    }
                    if (rc != BKException.Code.OK) {
                        log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                        cursorCounter.countDown();
                        return;
                    }
                    long lac = lh.getLastAddConfirmed();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                    }
                    if (lac == LedgerHandle.INVALID_ENTRY_ID) {
                        // save the ledger id and cursor to retry outside of this call back
                        // since we are trying to read the same cursor ledger, we will block until
                        // this current callback completes, since an attempt to read the entry
                        // will block behind this current operation to complete
                        ledgerRetryMap.put(cursorName, ledgerId);
                        log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                        cursorCounter.countDown();
                        return;
                    }
                    final long entryId = lac;
                    // read last acked message position for subscription
                    lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() {

                        @Override
                        public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
                            try {
                                if (log.isDebugEnabled()) {
                                    log.debug("readComplete rc={} entryId={}", rc, entryId);
                                }
                                if (rc != BKException.Code.OK) {
                                    log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                                    // indicate that this cursor should be excluded
                                    offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                } else {
                                    LedgerEntry entry = seq.nextElement();
                                    MLDataFormats.PositionInfo positionInfo;
                                    try {
                                        positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry());
                                    } catch (InvalidProtocolBufferException e) {
                                        log.warn("[{}] Error reading position from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, e);
                                        offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                        return;
                                    }
                                    final PositionImpl lastAckedMessagePosition = new PositionImpl(positionInfo);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Cursor {} MD {} read last ledger position {}", managedLedgerName, cursorName, lastAckedMessagePosition, lastLedgerPosition);
                                    }
                                    // calculate cursor backlog
                                    Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                                    }
                                    long cursorBacklog = getNumberOfEntries(range, ledgers);
                                    offlineTopicStats.messageBacklog += cursorBacklog;
                                    offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId());
                                }
                            } finally {
                                cursorCounter.countDown();
                            }
                        }
                    }, null);
                };
                // end of cursor meta read callback
                store.asyncGetCursorInfo(managedLedgerName, cursorName, new MetaStore.MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {

                    @Override
                    public void operationComplete(MLDataFormats.ManagedCursorInfo info, MetaStore.Stat version) {
                        long cursorLedgerId = info.getCursorsLedgerId();
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, cursorName, cursorLedgerId);
                        }
                        if (cursorLedgerId != -1) {
                            bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, cursorLedgerOpenCb, null);
                        } else {
                            PositionImpl lastAckedMessagePosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
                            Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                            }
                            long cursorBacklog = getNumberOfEntries(range, ledgers);
                            offlineTopicStats.messageBacklog += cursorBacklog;
                            offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId);
                            cursorCounter.countDown();
                        }
                    }

                    @Override
                    public void operationFailed(ManagedLedgerException.MetaStoreException e) {
                        log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, cursorName, e);
                        cursorCounter.countDown();
                    }
                });
            }
            // for every cursor find backlog
            try {
                if (accurate) {
                    cursorCounter.await();
                } else {
                    cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e);
            } finally {
                allCursorsCounter.countDown();
            }
        }

        @Override
        public void operationFailed(ManagedLedgerException.MetaStoreException e) {
            log.warn("[{}] Failed to get the cursors list", managedLedgerName, e);
            allCursorsCounter.countDown();
        }
    });
    if (accurate) {
        allCursorsCounter.await();
    } else {
        allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
    }
    // go through ledgers where LAC was -1
    if (accurate && ledgerRetryMap.size() > 0) {
        ledgerRetryMap.forEach((cursorName, ledgerId) -> {
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId);
            }
            PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName);
            if (lastAckedMessagePosition == null) {
                log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", managedLedgerName, cursorName, ledgerId);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, cursorName, ledgerId, lastAckedMessagePosition);
                }
                // calculate cursor backlog
                Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                }
                long cursorBacklog = getNumberOfEntries(range, ledgers);
                offlineTopicStats.messageBacklog += cursorBacklog;
                offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
            }
        });
    }
}

Example 2 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project pulsar by yahoo.

the class EntryCacheTest method testReadMissingAfter.

@Test(timeOut = 5000)
void testReadMissingAfter() throws Exception {
    LedgerHandle lh = getLedgerHandle();
    when(lh.getId()).thenReturn((long) 0);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    EntryCache entryCache = cacheManager.getEntryCache(ml);
    byte[] data = new byte[10];
    for (int i = 0; i < 8; i++) {
        entryCache.insert(new EntryImpl(0, i, data));
    }
    final CountDownLatch counter = new CountDownLatch(1);
    entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {

        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            assertEquals(entries.size(), 10);
            counter.countDown();
        }

        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            Assert.fail("should not have failed");
        }
    }, null);
    counter.await();
}

Example 3 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project pulsar by yahoo.

the class EntryCacheTest method testReadMissingBefore.

@Test(timeOut = 5000)
void testReadMissingBefore() throws Exception {
    LedgerHandle lh = getLedgerHandle();
    when(lh.getId()).thenReturn((long) 0);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    EntryCache entryCache = cacheManager.getEntryCache(ml);
    byte[] data = new byte[10];
    for (int i = 3; i < 10; i++) {
        entryCache.insert(new EntryImpl(0, i, data));
    }
    final CountDownLatch counter = new CountDownLatch(1);
    entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {

        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            assertEquals(entries.size(), 10);
            counter.countDown();
        }

        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            Assert.fail("should not have failed");
        }
    }, null);
    counter.await();
}

Example 4 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project pulsar by yahoo.

the class EntryCacheTest method testReadMissingMiddle.

@Test(timeOut = 5000)
void testReadMissingMiddle() throws Exception {
    LedgerHandle lh = getLedgerHandle();
    when(lh.getId()).thenReturn((long) 0);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    EntryCache entryCache = cacheManager.getEntryCache(ml);
    byte[] data = new byte[10];
    entryCache.insert(new EntryImpl(0, 0, data));
    entryCache.insert(new EntryImpl(0, 1, data));
    entryCache.insert(new EntryImpl(0, 8, data));
    entryCache.insert(new EntryImpl(0, 9, data));
    final CountDownLatch counter = new CountDownLatch(1);
    entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {

        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            assertEquals(entries.size(), 10);
            counter.countDown();
        }

        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            Assert.fail("should not have failed");
        }
    }, null);
    counter.await();
}

Example 5 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project pulsar by yahoo.

the class EntryCacheTest method getLedgerHandle.

private static LedgerHandle getLedgerHandle() {
    final LedgerHandle lh = mock(LedgerHandle.class);
    final LedgerEntry ledgerEntry = mock(LedgerEntry.class, Mockito.CALLS_REAL_METHODS);
    doReturn(new byte[10]).when(ledgerEntry).getEntry();
    doReturn(Unpooled.wrappedBuffer(new byte[10])).when(ledgerEntry).getEntryBuffer();
    doReturn((long) 10).when(ledgerEntry).getLength();
    doAnswer(new Answer<Object>() {

        public Object answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            long firstEntry = (Long) args[0];
            long lastEntry = (Long) args[1];
            ReadCallback callback = (ReadCallback) args[2];
            Object ctx = args[3];
            Vector<LedgerEntry> entries = new Vector<LedgerEntry>();
            for (int i = 0; i <= (lastEntry - firstEntry); i++) {
                entries.add(ledgerEntry);
            }
            callback.readComplete(0, lh, entries.elements(), ctx);
            return null;
        }
    }).when(lh).asyncReadEntries(anyLong(), anyLong(), any(ReadCallback.class), any());
    return lh;
}