Examples with LedgerEntry org.apache.bookkeeper.client.LedgerEntry used on opensource projects

Example 46 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class ManagedLedgerOfflineBacklog method tryGetMDPosition.

private PositionImpl tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) {
    BookKeeperAdmin bookKeeperAdmin = null;
    long lastEntry = LedgerHandle.INVALID_ENTRY_ID;
    PositionImpl lastAckedMessagePosition = null;
    try {
        bookKeeperAdmin = new BookKeeperAdmin(bookKeeper);
        Iterator<LedgerEntry> entries = bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry).iterator();
        while (entries.hasNext()) {
            LedgerEntry ledgerEntry = entries.next();
            lastEntry = ledgerEntry.getEntryId();
            if (log.isDebugEnabled()) {
                log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName);
            }
            MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry());
            lastAckedMessagePosition = new PositionImpl(positionInfo);
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition);
            }
        }
    } catch (Exception e) {
        log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e);
    } finally {
        if (bookKeeperAdmin != null) {
            try {
                bookKeeperAdmin.close();
            } catch (Exception e) {
                log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e);
            }
        }
    }
    return lastAckedMessagePosition;
}

Example 47 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class ManagedCursorTest method testOutOfOrderDeletePersistenceIntoLedgerWithClose.

/**
 * Verifies cursor persists individually unack range into cursor-ledger if range count is higher than
 * MaxUnackedRangesToPersistInZk
 *
 * @throws Exception
 */
@Test(timeOut = 20000)
public void testOutOfOrderDeletePersistenceIntoLedgerWithClose() throws Exception {
    final int totalAddEntries = 100;
    String ledgerName = "my_test_ledger";
    String cursorName = "c1";
    ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
    // metaStore is allowed to store only up to 10 deleted entries range
    managedLedgerConfig.setMaxUnackedRangesToPersistInZk(10);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
    List<Position> addedPositions = new ArrayList<>();
    for (int i = 0; i < totalAddEntries; i++) {
        Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
        addedPositions.add(p);
        if (i % 2 == 0) {
            // Acknowledge alternative message to create totalEntries/2 holes
            c1.delete(addedPositions.get(i));
        }
    }
    assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
    // Close ledger to persist individual-deleted positions into cursor-ledger
    ledger.close();
    // verify cursor-ledgerId is updated properly into cursor-metaStore
    CountDownLatch cursorLedgerLatch = new CountDownLatch(1);
    AtomicLong cursorLedgerId = new AtomicLong(0);
    ledger.getStore().asyncGetCursorInfo(ledger.getName(), cursorName, new MetaStoreCallback<ManagedCursorInfo>() {

        @Override
        public void operationComplete(ManagedCursorInfo result, Stat stat) {
            cursorLedgerId.set(result.getCursorsLedgerId());
            cursorLedgerLatch.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            cursorLedgerLatch.countDown();
        }
    });
    cursorLedgerLatch.await();
    assertEquals(cursorLedgerId.get(), c1.getCursorLedger());
    // verify cursor-ledger's last entry has individual-deleted positions
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicInteger individualDeletedMessagesCount = new AtomicInteger(0);
    bkc.asyncOpenLedger(c1.getCursorLedger(), DigestType.CRC32C, "".getBytes(), (rc, lh, ctx) -> {
        if (rc == BKException.Code.OK) {
            long lastEntry = lh.getLastAddConfirmed();
            lh.asyncReadEntries(lastEntry, lastEntry, (rc1, lh1, seq, ctx1) -> {
                try {
                    LedgerEntry entry = seq.nextElement();
                    PositionInfo positionInfo;
                    positionInfo = PositionInfo.parseFrom(entry.getEntry());
                    individualDeletedMessagesCount.set(positionInfo.getIndividualDeletedMessagesCount());
                } catch (Exception e) {
                }
                latch.countDown();
            }, null);
        } else {
            latch.countDown();
        }
    }, null);
    latch.await();
    assertEquals(individualDeletedMessagesCount.get(), totalAddEntries / 2 - 1);
    // Re-Open
    factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
    c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    // verify cursor has been recovered
    assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
    // try to read entries which should only read non-deleted positions
    List<Entry> entries = c1.readEntries(totalAddEntries);
    assertEquals(entries.size(), totalAddEntries / 2);
}

Example 48 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class EntryCacheImpl method asyncReadEntry.

@Override
public void asyncReadEntry(LedgerHandle lh, PositionImpl position, final ReadEntryCallback callback, final Object ctx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] Reading entry ledger {}: {}", ml.getName(), lh.getId(), position.getEntryId());
    }
    EntryImpl entry = entries.get(position);
    if (entry != null) {
        EntryImpl cachedEntry = EntryImpl.create(entry);
        entry.release();
        manager.mlFactoryMBean.recordCacheHit(cachedEntry.getLength());
        callback.readEntryComplete(cachedEntry, ctx);
    } else {
        lh.asyncReadEntries(position.getEntryId(), position.getEntryId(), (rc, ledgerHandle, sequence, obj) -> {
            if (rc != BKException.Code.OK) {
                ml.invalidateLedgerHandle(ledgerHandle, rc);
                callback.readEntryFailed(createManagedLedgerException(rc), obj);
                return;
            }
            if (sequence.hasMoreElements()) {
                LedgerEntry ledgerEntry = sequence.nextElement();
                EntryImpl returnEntry = EntryImpl.create(ledgerEntry);
                // The EntryImpl is now the owner of the buffer, so we can release the original one
                ledgerEntry.getEntryBuffer().release();
                manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength());
                ml.mbean.addReadEntriesSample(1, returnEntry.getLength());
                ml.getExecutor().submitOrdered(ml.getName(), safeRun(() -> {
                    callback.readEntryComplete(returnEntry, obj);
                }));
            } else {
                // got an empty sequence
                callback.readEntryFailed(new ManagedLedgerException("Could not read given position"), obj);
            }
        }, ctx);
    }
}

Example 49 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class EntryCacheImpl method asyncReadEntry.

@Override
@SuppressWarnings({ "unchecked", "rawtypes" })
public void asyncReadEntry(LedgerHandle lh, long firstEntry, long lastEntry, boolean isSlowestReader, final ReadEntriesCallback callback, Object ctx) {
    final long ledgerId = lh.getId();
    final int entriesToRead = (int) (lastEntry - firstEntry) + 1;
    final PositionImpl firstPosition = PositionImpl.get(lh.getId(), firstEntry);
    final PositionImpl lastPosition = PositionImpl.get(lh.getId(), lastEntry);
    if (log.isDebugEnabled()) {
        log.debug("[{}] Reading entries range ledger {}: {} to {}", ml.getName(), ledgerId, firstEntry, lastEntry);
    }
    Collection<EntryImpl> cachedEntries = entries.getRange(firstPosition, lastPosition);
    if (cachedEntries.size() == entriesToRead) {
        long totalCachedSize = 0;
        final List<EntryImpl> entriesToReturn = Lists.newArrayListWithExpectedSize(entriesToRead);
        // All entries found in cache
        for (EntryImpl entry : cachedEntries) {
            entriesToReturn.add(EntryImpl.create(entry));
            totalCachedSize += entry.getLength();
            entry.release();
        }
        manager.mlFactoryMBean.recordCacheHits(entriesToReturn.size(), totalCachedSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}] Ledger {} -- Found in cache entries: {}-{}", ml.getName(), ledgerId, firstEntry, lastEntry);
        }
        callback.readEntriesComplete((List) entriesToReturn, ctx);
    } else {
        if (!cachedEntries.isEmpty()) {
            cachedEntries.forEach(entry -> entry.release());
        }
        // Read all the entries from bookkeeper
        lh.asyncReadEntries(firstEntry, lastEntry, (rc, lh1, sequence, cb) -> {
            if (rc != BKException.Code.OK) {
                if (rc == BKException.Code.TooManyRequestsException) {
                    callback.readEntriesFailed(createManagedLedgerException(rc), ctx);
                } else {
                    ml.invalidateLedgerHandle(lh1, rc);
                    ManagedLedgerException mlException = createManagedLedgerException(rc);
                    callback.readEntriesFailed(mlException, ctx);
                }
                return;
            }
            checkNotNull(ml.getName());
            checkNotNull(ml.getExecutor());
            ml.getExecutor().submitOrdered(ml.getName(), safeRun(() -> {
                // We got the entries, we need to transform them to a List<> type
                long totalSize = 0;
                final List<EntryImpl> entriesToReturn = Lists.newArrayListWithExpectedSize(entriesToRead);
                while (sequence.hasMoreElements()) {
                    // Insert the entries at the end of the list (they will be unsorted for now)
                    LedgerEntry ledgerEntry = sequence.nextElement();
                    EntryImpl entry = EntryImpl.create(ledgerEntry);
                    ledgerEntry.getEntryBuffer().release();
                    entriesToReturn.add(entry);
                    totalSize += entry.getLength();
                }
                manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize);
                ml.getMBean().addReadEntriesSample(entriesToReturn.size(), totalSize);
                callback.readEntriesComplete((List) entriesToReturn, ctx);
            }));
        }, callback);
    }
}

Example 50 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class ManagedCursorImpl method recoverFromLedger.

protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallback callback) {
    // Read the acknowledged position from the metadata ledger, then create
    // a new ledger and write the position into it
    ledger.mbean.startCursorLedgerOpenOp();
    long ledgerId = info.getCursorsLedgerId();
    bookkeeper.asyncOpenLedger(ledgerId, config.getDigestType(), config.getPassword(), (rc, lh, ctx) -> {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Opened ledger {} for consumer {}. rc={}", ledger.getName(), ledgerId, name, rc);
        }
        if (isBkErrorNotRecoverable(rc)) {
            log.error("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc));
            // Rewind to oldest entry available
            initialize(getRollbackPosition(info), callback);
            return;
        } else if (rc != BKException.Code.OK) {
            log.warn("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc));
            callback.operationFailed(new ManagedLedgerException(BKException.getMessage(rc)));
            return;
        }
        // Read the last entry in the ledger
        long lastEntryInLedger = lh.getLastAddConfirmed();
        lh.asyncReadEntries(lastEntryInLedger, lastEntryInLedger, (rc1, lh1, seq, ctx1) -> {
            if (log.isDebugEnabled()) {
                log.debug("[{}} readComplete rc={} entryId={}", ledger.getName(), rc1, lh1.getLastAddConfirmed());
            }
            if (isBkErrorNotRecoverable(rc1)) {
                log.error("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1));
                // Rewind to oldest entry available
                initialize(getRollbackPosition(info), callback);
                return;
            } else if (rc1 != BKException.Code.OK) {
                log.warn("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1));
                callback.operationFailed(createManagedLedgerException(rc1));
                return;
            }
            LedgerEntry entry = seq.nextElement();
            PositionInfo positionInfo;
            try {
                positionInfo = PositionInfo.parseFrom(entry.getEntry());
            } catch (InvalidProtocolBufferException e) {
                callback.operationFailed(new ManagedLedgerException(e));
                return;
            }
            Map<String, Long> recoveredProperties = Collections.emptyMap();
            if (positionInfo.getPropertiesCount() > 0) {
                // Recover properties map
                recoveredProperties = Maps.newHashMap();
                for (int i = 0; i < positionInfo.getPropertiesCount(); i++) {
                    LongProperty property = positionInfo.getProperties(i);
                    recoveredProperties.put(property.getName(), property.getValue());
                }
            }
            PositionImpl position = new PositionImpl(positionInfo);
            if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
                recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
            }
            recoveredCursor(position, recoveredProperties, lh);
            callback.operationComplete();
        }, null);
    }, null);
}