Examples with LedgerEntry org.apache.bookkeeper.client.LedgerEntry used on opensource projects

Example 41 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project bookkeeper by apache.

the class TestSmoke method testBootWriteReadShutdown.

@Test
public void testBootWriteReadShutdown() throws Exception {
    Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion));
    String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker);
    BookKeeper bk = new BookKeeper(zookeeper);
    long ledgerId;
    try (LedgerHandle writelh = bk.createLedger(BookKeeper.DigestType.CRC32, PASSWD)) {
        ledgerId = writelh.getId();
        for (int i = 0; i < 100; i++) {
            writelh.addEntry(("entry-" + i).getBytes());
        }
    }
    try (LedgerHandle readlh = bk.openLedger(ledgerId, BookKeeper.DigestType.CRC32, PASSWD)) {
        long lac = readlh.getLastAddConfirmed();
        int i = 0;
        Enumeration<LedgerEntry> entries = readlh.readEntries(0, lac);
        while (entries.hasMoreElements()) {
            LedgerEntry e = entries.nextElement();
            String readBack = new String(e.getEntry());
            Assert.assertEquals(readBack, "entry-" + i++);
        }
        Assert.assertEquals(i, 100);
    }
    bk.close();
    Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker));
}

Example 42 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project herddb by diennea.

the class BookkeeperCommitLog method followTheLeader.

@Override
public void followTheLeader(LogSequenceNumber skipPast, BiConsumer<LogSequenceNumber, LogEntry> consumer) throws LogNotAvailableException {
    List<Long> actualList = metadataManager.getActualLedgersList(tableSpaceUUID).getActiveLedgers();
    List<Long> toRead = actualList;
    if (skipPast.ledgerId != -1) {
        toRead = toRead.stream().filter(l -> l >= skipPast.ledgerId).collect(Collectors.toList());
    }
    try {
        long nextEntry = skipPast.offset + 1;
        // LOGGER.log(Level.SEVERE, "followTheLeader "+tableSpace+" skipPast:{0} toRead: {1} actualList:{2}, nextEntry:{3}", new Object[]{skipPast, toRead, actualList, nextEntry});
        for (Long previous : toRead) {
            // LOGGER.log(Level.SEVERE, "followTheLeader openLedger " + previous + " nextEntry:" + nextEntry);
            LedgerHandle lh;
            try {
                lh = bookKeeper.openLedgerNoRecovery(previous, BookKeeper.DigestType.CRC32, sharedSecret.getBytes(StandardCharsets.UTF_8));
            } catch (BKException.BKLedgerRecoveryException e) {
                LOGGER.log(Level.SEVERE, "error", e);
                return;
            }
            try {
                long lastAddConfirmed = lh.getLastAddConfirmed();
                LOGGER.log(Level.FINE, "followTheLeader " + tableSpaceUUID + " openLedger {0} -> lastAddConfirmed:{1}, nextEntry:{2}", new Object[] { previous, lastAddConfirmed, nextEntry });
                if (nextEntry > lastAddConfirmed) {
                    nextEntry = 0;
                    continue;
                }
                Enumeration<LedgerEntry> entries = lh.readEntries(nextEntry, lh.getLastAddConfirmed());
                while (entries.hasMoreElements()) {
                    LedgerEntry e = entries.nextElement();
                    long entryId = e.getEntryId();
                    byte[] entryData = e.getEntry();
                    LogEntry statusEdit = LogEntry.deserialize(entryData);
                    // LOGGER.log(Level.SEVERE, "" + tableSpaceUUID + " followentry {0},{1} -> {2}", new Object[]{previous, entryId, statusEdit});
                    LogSequenceNumber number = new LogSequenceNumber(previous, entryId);
                    lastSequenceNumber.accumulateAndGet(number.offset, EnsureLongIncrementAccumulator.INSTANCE);
                    lastLedgerId = number.ledgerId;
                    currentLedgerId = number.ledgerId;
                    consumer.accept(number, statusEdit);
                }
            } finally {
                try {
                    lh.close();
                } catch (BKException err) {
                    LOGGER.log(Level.SEVERE, "error while closing ledger", err);
                } catch (InterruptedException err) {
                    LOGGER.log(Level.SEVERE, "error while closing ledger", err);
                    Thread.currentThread().interrupt();
                }
            }
        }
    } catch (InterruptedException | EOFException | BKException err) {
        LOGGER.log(Level.SEVERE, "internal error", err);
        throw new LogNotAvailableException(err);
    }
}

Example 43 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project herddb by diennea.

the class BookkeeperCommitLog method recovery.

@Override
public void recovery(LogSequenceNumber snapshotSequenceNumber, BiConsumer<LogSequenceNumber, LogEntry> consumer, boolean fencing) throws LogNotAvailableException {
    this.actualLedgersList = metadataManager.getActualLedgersList(tableSpaceUUID);
    LOGGER.log(Level.SEVERE, "Actual ledgers list:" + actualLedgersList + " tableSpace " + tableSpaceUUID);
    this.lastLedgerId = snapshotSequenceNumber.ledgerId;
    this.currentLedgerId = snapshotSequenceNumber.ledgerId;
    this.lastSequenceNumber.set(snapshotSequenceNumber.offset);
    LOGGER.log(Level.SEVERE, "recovery from latest snapshotSequenceNumber:" + snapshotSequenceNumber);
    if (currentLedgerId > 0 && !this.actualLedgersList.getActiveLedgers().contains(currentLedgerId) && !this.actualLedgersList.getActiveLedgers().isEmpty()) {
        // TODO: download snapshot from another remote broker
        throw new FullRecoveryNeededException(new Exception("Actual ledgers list does not include latest snapshot ledgerid:" + currentLedgerId));
    }
    if (snapshotSequenceNumber.isStartOfTime() && !this.actualLedgersList.getActiveLedgers().isEmpty() && !this.actualLedgersList.getActiveLedgers().contains(this.actualLedgersList.getFirstLedger())) {
        throw new FullRecoveryNeededException(new Exception("Local data is absent, and actual ledger list " + this.actualLedgersList.getActiveLedgers() + " does not contain first ledger of ever: " + this.actualLedgersList.getFirstLedger()));
    }
    try {
        for (long ledgerId : actualLedgersList.getActiveLedgers()) {
            if (ledgerId < snapshotSequenceNumber.ledgerId) {
                LOGGER.log(Level.SEVERE, "Skipping ledger " + ledgerId);
                continue;
            }
            LedgerHandle handle;
            if (fencing) {
                handle = bookKeeper.openLedger(ledgerId, BookKeeper.DigestType.CRC32, sharedSecret.getBytes(StandardCharsets.UTF_8));
            } else {
                handle = bookKeeper.openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, sharedSecret.getBytes(StandardCharsets.UTF_8));
            }
            try {
                long first;
                if (ledgerId == snapshotSequenceNumber.ledgerId) {
                    first = snapshotSequenceNumber.offset;
                    LOGGER.log(Level.SEVERE, "Recovering from latest snapshot ledger " + ledgerId + ", starting from entry " + first);
                } else {
                    first = 0;
                    LOGGER.log(Level.SEVERE, "Recovering from ledger " + ledgerId + ", starting from entry " + first);
                }
                long lastAddConfirmed = handle.getLastAddConfirmed();
                LOGGER.log(Level.SEVERE, "Recovering from ledger " + ledgerId + ", first=" + first + " lastAddConfirmed=" + lastAddConfirmed);
                final int BATCH_SIZE = 10000;
                if (lastAddConfirmed >= 0) {
                    for (long b = first; b <= lastAddConfirmed; ) {
                        long start = b;
                        long end = b + BATCH_SIZE;
                        if (end > lastAddConfirmed) {
                            end = lastAddConfirmed;
                        }
                        b = end + 1;
                        double percent = ((start - first) * 100.0 / (lastAddConfirmed + 1));
                        int entriesToRead = (int) (1 + end - start);
                        LOGGER.log(Level.SEVERE, "From entry {0}, to entry {1} ({2} %)", new Object[] { start, end, percent });
                        long _start = System.currentTimeMillis();
                        Enumeration<LedgerEntry> entries = handle.readEntries(start, end);
                        int localEntryCount = 0;
                        while (entries.hasMoreElements()) {
                            LedgerEntry entry = entries.nextElement();
                            long entryId = entry.getEntryId();
                            LogSequenceNumber number = new LogSequenceNumber(ledgerId, entryId);
                            LogEntry statusEdit = LogEntry.deserialize(entry.getEntry());
                            lastLedgerId = ledgerId;
                            currentLedgerId = ledgerId;
                            lastSequenceNumber.set(entryId);
                            if (number.after(snapshotSequenceNumber)) {
                                LOGGER.log(Level.FINEST, "RECOVER ENTRY #" + localEntryCount + " {0}, {1}", new Object[] { number, statusEdit });
                                consumer.accept(number, statusEdit);
                            } else {
                                LOGGER.log(Level.FINEST, "SKIP ENTRY #" + localEntryCount + " {0}<{1}, {2}", new Object[] { number, snapshotSequenceNumber, statusEdit });
                            }
                            localEntryCount++;
                        }
                        LOGGER.log(Level.SEVERE, "read " + localEntryCount + " entries from ledger " + ledgerId + ", expected " + entriesToRead);
                        LOGGER.log(Level.SEVERE, "finished waiting for " + entriesToRead + " entries to be read from ledger " + ledgerId);
                        if (localEntryCount != entriesToRead) {
                            throw new LogNotAvailableException("Read " + localEntryCount + " entries, expected " + entriesToRead);
                        }
                        lastLedgerId = ledgerId;
                        lastSequenceNumber.set(end);
                        long _stop = System.currentTimeMillis();
                        LOGGER.log(Level.SEVERE, "From entry {0}, to entry {1} ({2} %) read time {3}", new Object[] { start, end, percent, (_stop - _start) + " ms" });
                    }
                }
            } finally {
                handle.close();
            }
        }
        LOGGER.severe("After recovery of " + tableSpaceUUID + " lastSequenceNumber " + getLastSequenceNumber());
    } catch (InterruptedException | EOFException | RuntimeException | BKException err) {
        LOGGER.log(Level.SEVERE, "Fatal error during recovery", err);
        signalLogFailed();
        throw new LogNotAvailableException(err);
    }
}

Example 44 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class ManagedLedgerOfflineBacklog method calculateCursorBacklogs.

private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final TopicName topicName, final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers, final PersistentOfflineTopicStats offlineTopicStats) throws Exception {
    if (ledgers.size() == 0) {
        return;
    }
    String managedLedgerName = topicName.getPersistenceNamingEncoding();
    MetaStore store = factory.getMetaStore();
    BookKeeper bk = factory.getBookKeeper();
    final CountDownLatch allCursorsCounter = new CountDownLatch(1);
    final long errorInReadingCursor = (long) -1;
    ConcurrentOpenHashMap<String, Long> ledgerRetryMap = new ConcurrentOpenHashMap<>();
    final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
    final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
    if (log.isDebugEnabled()) {
        log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition);
    }
    store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback<List<String>>() {

        @Override
        public void operationComplete(List<String> cursors, MetaStore.Stat v) {
            // Load existing cursors
            if (log.isDebugEnabled()) {
                log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size());
            }
            if (cursors.isEmpty()) {
                allCursorsCounter.countDown();
                return;
            }
            final CountDownLatch cursorCounter = new CountDownLatch(cursors.size());
            for (final String cursorName : cursors) {
                // determine subscription position from cursor ledger
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName);
                }
                AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> {
                    long ledgerId = lh.getId();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, cursorName, rc);
                    }
                    if (rc != BKException.Code.OK) {
                        log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                        cursorCounter.countDown();
                        return;
                    }
                    long lac = lh.getLastAddConfirmed();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                    }
                    if (lac == LedgerHandle.INVALID_ENTRY_ID) {
                        // save the ledger id and cursor to retry outside of this call back
                        // since we are trying to read the same cursor ledger, we will block until
                        // this current callback completes, since an attempt to read the entry
                        // will block behind this current operation to complete
                        ledgerRetryMap.put(cursorName, ledgerId);
                        log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                        cursorCounter.countDown();
                        return;
                    }
                    final long entryId = lac;
                    // read last acked message position for subscription
                    lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() {

                        @Override
                        public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
                            try {
                                if (log.isDebugEnabled()) {
                                    log.debug("readComplete rc={} entryId={}", rc, entryId);
                                }
                                if (rc != BKException.Code.OK) {
                                    log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                                    // indicate that this cursor should be excluded
                                    offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                } else {
                                    LedgerEntry entry = seq.nextElement();
                                    MLDataFormats.PositionInfo positionInfo;
                                    try {
                                        positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry());
                                    } catch (InvalidProtocolBufferException e) {
                                        log.warn("[{}] Error reading position from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, e);
                                        offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                        return;
                                    }
                                    final PositionImpl lastAckedMessagePosition = new PositionImpl(positionInfo);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Cursor {} MD {} read last ledger position {}", managedLedgerName, cursorName, lastAckedMessagePosition, lastLedgerPosition);
                                    }
                                    // calculate cursor backlog
                                    Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                                    }
                                    long cursorBacklog = getNumberOfEntries(range, ledgers);
                                    offlineTopicStats.messageBacklog += cursorBacklog;
                                    offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId());
                                }
                            } finally {
                                cursorCounter.countDown();
                            }
                        }
                    }, null);
                };
                // end of cursor meta read callback
                store.asyncGetCursorInfo(managedLedgerName, cursorName, new MetaStore.MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {

                    @Override
                    public void operationComplete(MLDataFormats.ManagedCursorInfo info, MetaStore.Stat version) {
                        long cursorLedgerId = info.getCursorsLedgerId();
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, cursorName, cursorLedgerId);
                        }
                        if (cursorLedgerId != -1) {
                            bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, cursorLedgerOpenCb, null);
                        } else {
                            PositionImpl lastAckedMessagePosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
                            Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                            }
                            long cursorBacklog = getNumberOfEntries(range, ledgers);
                            offlineTopicStats.messageBacklog += cursorBacklog;
                            offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId);
                            cursorCounter.countDown();
                        }
                    }

                    @Override
                    public void operationFailed(ManagedLedgerException.MetaStoreException e) {
                        log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, cursorName, e);
                        cursorCounter.countDown();
                    }
                });
            }
            // for every cursor find backlog
            try {
                if (accurate) {
                    cursorCounter.await();
                } else {
                    cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e);
            } finally {
                allCursorsCounter.countDown();
            }
        }

        @Override
        public void operationFailed(ManagedLedgerException.MetaStoreException e) {
            log.warn("[{}] Failed to get the cursors list", managedLedgerName, e);
            allCursorsCounter.countDown();
        }
    });
    if (accurate) {
        allCursorsCounter.await();
    } else {
        allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
    }
    // go through ledgers where LAC was -1
    if (accurate && ledgerRetryMap.size() > 0) {
        ledgerRetryMap.forEach((cursorName, ledgerId) -> {
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId);
            }
            PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName);
            if (lastAckedMessagePosition == null) {
                log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", managedLedgerName, cursorName, ledgerId);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, cursorName, ledgerId, lastAckedMessagePosition);
                }
                // calculate cursor backlog
                Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                }
                long cursorBacklog = getNumberOfEntries(range, ledgers);
                offlineTopicStats.messageBacklog += cursorBacklog;
                offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
            }
        });
    }
}

Example 45 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class ManagedLedgerOfflineBacklog method tryGetMDPosition.

private PositionImpl tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) {
    BookKeeperAdmin bookKeeperAdmin = null;
    long lastEntry = LedgerHandle.INVALID_ENTRY_ID;
    PositionImpl lastAckedMessagePosition = null;
    try {
        bookKeeperAdmin = new BookKeeperAdmin(bookKeeper);
        Iterator<LedgerEntry> entries = bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry).iterator();
        while (entries.hasNext()) {
            LedgerEntry ledgerEntry = entries.next();
            lastEntry = ledgerEntry.getEntryId();
            if (log.isDebugEnabled()) {
                log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName);
            }
            MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry());
            lastAckedMessagePosition = new PositionImpl(positionInfo);
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition);
            }
        }
    } catch (Exception e) {
        log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e);
    } finally {
        if (bookKeeperAdmin != null) {
            try {
                bookKeeperAdmin.close();
            } catch (Exception e) {
                log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e);
            }
        }
    }
    return lastAckedMessagePosition;
}