Examples with LedgerEntry org.apache.bookkeeper.client.LedgerEntry used on opensource projects

Example 1 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project pulsar by yahoo.

the class ManagedLedgerOfflineBacklog method tryGetMDPosition.

private PositionImpl tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) {
    BookKeeperAdmin bookKeeperAdmin = null;
    long lastEntry = LedgerHandle.INVALID_ENTRY_ID;
    PositionImpl lastAckedMessagePosition = null;
    try {
        bookKeeperAdmin = new BookKeeperAdmin(bookKeeper);
        Iterator<LedgerEntry> entries = bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry).iterator();
        while (entries.hasNext()) {
            LedgerEntry ledgerEntry = entries.next();
            lastEntry = ledgerEntry.getEntryId();
            if (log.isDebugEnabled()) {
                log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName);
            }
            MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry());
            lastAckedMessagePosition = new PositionImpl(positionInfo);
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition);
            }
        }
    } catch (Exception e) {
        log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e);
    } finally {
        if (bookKeeperAdmin != null) {
            try {
                bookKeeperAdmin.close();
            } catch (Exception e) {
                log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e);
            }
        }
    }
    return lastAckedMessagePosition;
}

Example 2 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project pulsar by yahoo.

the class ManagedLedgerOfflineBacklog method calculateCursorBacklogs.

private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final DestinationName dn, final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers, final PersistentOfflineTopicStats offlineTopicStats) throws Exception {
    if (ledgers.size() == 0) {
        return;
    }
    String managedLedgerName = dn.getPersistenceNamingEncoding();
    MetaStore store = factory.getMetaStore();
    BookKeeper bk = factory.getBookKeeper();
    final CountDownLatch allCursorsCounter = new CountDownLatch(1);
    final long errorInReadingCursor = (long) -1;
    ConcurrentOpenHashMap<String, Long> ledgerRetryMap = new ConcurrentOpenHashMap<>();
    final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
    final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
    if (log.isDebugEnabled()) {
        log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition);
    }
    store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback<List<String>>() {

        @Override
        public void operationComplete(List<String> cursors, MetaStore.Stat v) {
            // Load existing cursors
            if (log.isDebugEnabled()) {
                log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size());
            }
            if (cursors.isEmpty()) {
                allCursorsCounter.countDown();
                return;
            }
            final CountDownLatch cursorCounter = new CountDownLatch(cursors.size());
            for (final String cursorName : cursors) {
                // determine subscription position from cursor ledger
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName);
                }
                AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> {
                    long ledgerId = lh.getId();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, cursorName, rc);
                    }
                    if (rc != BKException.Code.OK) {
                        log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                        cursorCounter.countDown();
                        return;
                    }
                    long lac = lh.getLastAddConfirmed();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                    }
                    if (lac == LedgerHandle.INVALID_ENTRY_ID) {
                        // save the ledger id and cursor to retry outside of this call back
                        // since we are trying to read the same cursor ledger, we will block until
                        // this current callback completes, since an attempt to read the entry
                        // will block behind this current operation to complete
                        ledgerRetryMap.put(cursorName, ledgerId);
                        log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                        cursorCounter.countDown();
                        return;
                    }
                    final long entryId = lac;
                    // read last acked message position for subscription
                    lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() {

                        @Override
                        public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
                            try {
                                if (log.isDebugEnabled()) {
                                    log.debug("readComplete rc={} entryId={}", rc, entryId);
                                }
                                if (rc != BKException.Code.OK) {
                                    log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                                    // indicate that this cursor should be excluded
                                    offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                } else {
                                    LedgerEntry entry = seq.nextElement();
                                    MLDataFormats.PositionInfo positionInfo;
                                    try {
                                        positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry());
                                    } catch (InvalidProtocolBufferException e) {
                                        log.warn("[{}] Error reading position from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, e);
                                        offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                        return;
                                    }
                                    final PositionImpl lastAckedMessagePosition = new PositionImpl(positionInfo);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Cursor {} MD {} read last ledger position {}", managedLedgerName, cursorName, lastAckedMessagePosition, lastLedgerPosition);
                                    }
                                    // calculate cursor backlog
                                    Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                                    }
                                    long cursorBacklog = getNumberOfEntries(range, ledgers);
                                    offlineTopicStats.messageBacklog += cursorBacklog;
                                    offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId());
                                }
                            } finally {
                                cursorCounter.countDown();
                            }
                        }
                    }, null);
                };
                // end of cursor meta read callback
                store.asyncGetCursorInfo(managedLedgerName, cursorName, new MetaStore.MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {

                    @Override
                    public void operationComplete(MLDataFormats.ManagedCursorInfo info, MetaStore.Stat version) {
                        long cursorLedgerId = info.getCursorsLedgerId();
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, cursorName, cursorLedgerId);
                        }
                        if (cursorLedgerId != -1) {
                            bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, cursorLedgerOpenCb, null);
                        } else {
                            PositionImpl lastAckedMessagePosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
                            Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                            }
                            long cursorBacklog = getNumberOfEntries(range, ledgers);
                            offlineTopicStats.messageBacklog += cursorBacklog;
                            offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId);
                            cursorCounter.countDown();
                        }
                    }

                    @Override
                    public void operationFailed(ManagedLedgerException.MetaStoreException e) {
                        log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, cursorName, e);
                        cursorCounter.countDown();
                    }
                });
            }
            // for every cursor find backlog
            try {
                if (accurate) {
                    cursorCounter.await();
                } else {
                    cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e);
            } finally {
                allCursorsCounter.countDown();
            }
        }

        @Override
        public void operationFailed(ManagedLedgerException.MetaStoreException e) {
            log.warn("[{}] Failed to get the cursors list", managedLedgerName, e);
            allCursorsCounter.countDown();
        }
    });
    if (accurate) {
        allCursorsCounter.await();
    } else {
        allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
    }
    // go through ledgers where LAC was -1
    if (accurate && ledgerRetryMap.size() > 0) {
        ledgerRetryMap.forEach((cursorName, ledgerId) -> {
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId);
            }
            PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName);
            if (lastAckedMessagePosition == null) {
                log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", managedLedgerName, cursorName, ledgerId);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, cursorName, ledgerId, lastAckedMessagePosition);
                }
                // calculate cursor backlog
                Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                }
                long cursorBacklog = getNumberOfEntries(range, ledgers);
                offlineTopicStats.messageBacklog += cursorBacklog;
                offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
            }
        });
    }
}

Example 3 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project pulsar by yahoo.

the class EntryCacheTest method getLedgerHandle.

private static LedgerHandle getLedgerHandle() {
    final LedgerHandle lh = mock(LedgerHandle.class);
    final LedgerEntry ledgerEntry = mock(LedgerEntry.class, Mockito.CALLS_REAL_METHODS);
    doReturn(new byte[10]).when(ledgerEntry).getEntry();
    doReturn(Unpooled.wrappedBuffer(new byte[10])).when(ledgerEntry).getEntryBuffer();
    doReturn((long) 10).when(ledgerEntry).getLength();
    doAnswer(new Answer<Object>() {

        public Object answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            long firstEntry = (Long) args[0];
            long lastEntry = (Long) args[1];
            ReadCallback callback = (ReadCallback) args[2];
            Object ctx = args[3];
            Vector<LedgerEntry> entries = new Vector<LedgerEntry>();
            for (int i = 0; i <= (lastEntry - firstEntry); i++) {
                entries.add(ledgerEntry);
            }
            callback.readComplete(0, lh, entries.elements(), ctx);
            return null;
        }
    }).when(lh).asyncReadEntries(anyLong(), anyLong(), any(ReadCallback.class), any());
    return lh;
}

Example 4 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project pulsar by yahoo.

the class ManagedCursorImpl method recoverFromLedger.

protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallback callback) {
    // Read the acknowledged position from the metadata ledger, then create
    // a new ledger and write the position into it
    ledger.mbean.startCursorLedgerOpenOp();
    long ledgerId = info.getCursorsLedgerId();
    bookkeeper.asyncOpenLedger(ledgerId, config.getDigestType(), config.getPassword(), (rc, lh, ctx) -> {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Opened ledger {} for consumer {}. rc={}", ledger.getName(), ledgerId, name, rc);
        }
        if (isBkErrorNotRecoverable(rc)) {
            log.error("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc));
            // Rewind to oldest entry available
            initialize(getRollbackPosition(info), callback);
            return;
        } else if (rc != BKException.Code.OK) {
            log.warn("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc));
            callback.operationFailed(new ManagedLedgerException(BKException.getMessage(rc)));
            return;
        }
        // Read the last entry in the ledger
        cursorLedger = lh;
        lh.asyncReadLastEntry((rc1, lh1, seq, ctx1) -> {
            if (log.isDebugEnabled()) {
                log.debug("readComplete rc={} entryId={}", rc1, lh1.getLastAddConfirmed());
            }
            if (isBkErrorNotRecoverable(rc1)) {
                log.error("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1));
                // Rewind to oldest entry available
                initialize(getRollbackPosition(info), callback);
                return;
            } else if (rc1 != BKException.Code.OK) {
                log.warn("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1));
                callback.operationFailed(new ManagedLedgerException(BKException.getMessage(rc1)));
                return;
            }
            LedgerEntry entry = seq.nextElement();
            PositionInfo positionInfo;
            try {
                positionInfo = PositionInfo.parseFrom(entry.getEntry());
            } catch (InvalidProtocolBufferException e) {
                callback.operationFailed(new ManagedLedgerException(e));
                return;
            }
            PositionImpl position = new PositionImpl(positionInfo);
            if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
                recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
            }
            recoveredCursor(position);
            callback.operationComplete();
        }, null);
    }, null);
}

Example 5 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project bookkeeper by apache.

the class BKLogSegmentRandomAccessEntryReader method readComplete.

@SuppressWarnings("unchecked")
@Override
public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> entries, Object ctx) {
    CompletableFuture<List<Entry.Reader>> promise = (CompletableFuture<List<Entry.Reader>>) ctx;
    if (BKException.Code.OK == rc) {
        List<Entry.Reader> entryList = Lists.newArrayList();
        while (entries.hasMoreElements()) {
            LedgerEntry entry = entries.nextElement();
            try {
                entryList.add(processReadEntry(entry));
            } catch (IOException ioe) {
                // release the buffers
                while (entries.hasMoreElements()) {
                    LedgerEntry le = entries.nextElement();
                    le.getEntryBuffer().release();
                }
                FutureUtils.completeExceptionally(promise, ioe);
                return;
            } finally {
                entry.getEntryBuffer().release();
            }
        }
        FutureUtils.complete(promise, entryList);
    } else {
        FutureUtils.completeExceptionally(promise, new BKTransmitException("Failed to read entries :", rc));
    }
}