Examples with ReadHandle org.apache.bookkeeper.client.api.ReadHandle used on opensource projects

Example 1 with ReadHandle

use of org.apache.bookkeeper.client.api.ReadHandle in project herddb by diennea.

the class BookKeeperDataStorageManager method readIndexPage.

@Override
public <X> X readIndexPage(String tableSpace, String indexName, Long pageId, DataReader<X> reader) throws DataStorageManagerException {
    long _start = System.currentTimeMillis();
    TableSpacePagesMapping tableSpacePagesMapping = getTableSpacePagesMapping(tableSpace);
    Long ledgerId = tableSpacePagesMapping.getIndexPagesMapping(indexName).getLedgerIdForPage(pageId);
    if (ledgerId == null) {
        throw new DataPageDoesNotExistException("No such page for index : " + tableSpace + "_" + indexName + "." + pageId);
    }
    byte[] data;
    try (ReadHandle read = FutureUtils.result(bk.getBookKeeper().newOpenLedgerOp().withLedgerId(ledgerId).withPassword(EMPTY_ARRAY).execute(), BKException.HANDLER)) {
        try (LedgerEntries entries = read.readUnconfirmed(0, 0)) {
            data = entries.getEntry(0).getEntryBytes();
        }
        X result = readIndexPage(data, reader);
        long _stop = System.currentTimeMillis();
        long delta = _stop - _start;
        LOGGER.log(Level.FINE, "readIndexPage {0}.{1} {2} ms", new Object[] { tableSpace, indexName, delta + "" });
        indexPageReads.registerSuccessfulEvent(delta, TimeUnit.MILLISECONDS);
        return result;
    } catch (BKException.BKNoSuchLedgerExistsException err) {
        throw new DataStorageManagerException(err);
    } catch (BKException.BKNoSuchLedgerExistsOnMetadataServerException err) {
        throw new DataStorageManagerException(err);
    } catch (org.apache.bookkeeper.client.api.BKException err) {
        throw new DataStorageManagerException(err);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    } catch (InterruptedException err) {
        Thread.currentThread().interrupt();
        throw new DataStorageManagerException(err);
    }
}

Example 2 with ReadHandle

use of org.apache.bookkeeper.client.api.ReadHandle in project herddb by diennea.

the class BookkeeperCommitLogManager method scanRawLedger.

public static void scanRawLedger(long ledgerId, long fromId, long toId, herddb.client.ClientConfiguration clientConfiguration, ZookeeperMetadataStorageManager metadataStorageManager, Consumer<LogEntryWithSequenceNumber> consumer) throws Exception {
    ClientConfiguration config = new ClientConfiguration();
    config.setZkServers(metadataStorageManager.getZkAddress());
    config.setZkTimeout(metadataStorageManager.getZkSessionTimeout());
    config.setZkLedgersRootPath(clientConfiguration.getString(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH, ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH_DEFAULT));
    config.setEnableParallelRecoveryRead(true);
    config.setEnableDigestTypeAutodetection(true);
    try (org.apache.bookkeeper.client.api.BookKeeper bookKeeper = org.apache.bookkeeper.client.api.BookKeeper.newBuilder(config).build()) {
        try (ReadHandle lh = bookKeeper.newOpenLedgerOp().withRecovery(false).withLedgerId(ledgerId).withPassword(BookkeeperCommitLog.SHARED_SECRET.getBytes(StandardCharsets.UTF_8)).execute().get()) {
            long lastAddConfirmed = lh.readLastAddConfirmed();
            if (toId < 0) {
                toId = lastAddConfirmed;
            }
            LOG.log(Level.INFO, "Scanning Ledger {0} from {1} to {2} LAC {3}", new Object[] { ledgerId, fromId, toId, lastAddConfirmed });
            for (long id = fromId; id <= toId; id++) {
                try (LedgerEntries entries = lh.readUnconfirmed(id, id)) {
                    LedgerEntry entry = entries.getEntry(id);
                    LogEntry lEntry = LogEntry.deserialize(entry.getEntryBytes());
                    LogEntryWithSequenceNumber e = new LogEntryWithSequenceNumber(new LogSequenceNumber(ledgerId, id), lEntry);
                    consumer.accept(e);
                }
            }
        }
    }
}

Example 3 with ReadHandle

use of org.apache.bookkeeper.client.api.ReadHandle in project herddb by diennea.

the class BookkeeperCommitLog method followTheLeader.

@Override
public void followTheLeader(LogSequenceNumber lastPosition, EntryAcceptor consumer, FollowerContext context) throws LogNotAvailableException {
    if (LOGGER.isLoggable(Level.FINER)) {
        LOGGER.finer(tableSpaceDescription() + " followTheLeader lastPosition:" + lastPosition);
    }
    BKFollowerContext fContext = (BKFollowerContext) context;
    try {
        fContext.ensureOpenReader(lastPosition);
        if (fContext.currentLedger == null) {
            // no data to read
            if (LOGGER.isLoggable(Level.FINER)) {
                LOGGER.finer(tableSpaceDescription() + " no more data to read for now");
            }
            return;
        }
        long nextEntry = fContext.nextEntryToRead;
        long lastAddConfirmed = fContext.currentLedger.getLastAddConfirmed();
        if (LOGGER.isLoggable(Level.FINER)) {
            LOGGER.finer(tableSpaceDescription() + " next entry to read " + nextEntry + " from ledger " + fContext.currentLedger.getId() + " lastAddConfiremd " + lastAddConfirmed);
        }
        if (lastAddConfirmed < nextEntry) {
            if (LOGGER.isLoggable(Level.FINER)) {
                LOGGER.finer(tableSpaceDescription() + " ledger not closed but there is nothing to read by now");
            }
            return;
        }
        ReadHandle lh = fContext.currentLedger;
        try (LastConfirmedAndEntry entryAndLac = lh.readLastAddConfirmedAndEntry(nextEntry, LONG_POLL_TIMEOUT, false)) {
            if (entryAndLac.hasEntry()) {
                org.apache.bookkeeper.client.api.LedgerEntry e = entryAndLac.getEntry();
                boolean canContinue = acceptEntryForFollower(e, consumer);
                if (!canContinue) {
                    LOGGER.log(Level.INFO, "exit follower {0}", tableSpaceDescription());
                    return;
                }
                long startEntry = nextEntry + 1;
                long endEntry = entryAndLac.getLastAddConfirmed();
                if (startEntry > endEntry) {
                    return;
                }
                // to be read then read them
                if (endEntry - startEntry > MAX_ENTRY_TO_TAIL) {
                    // put a bound on the max entries to read per round
                    endEntry = startEntry + MAX_ENTRY_TO_TAIL;
                }
                try (LedgerEntries entries = lh.read(startEntry, endEntry)) {
                    for (org.apache.bookkeeper.client.api.LedgerEntry ee : entries) {
                        acceptEntryForFollower(ee, consumer);
                    }
                }
            }
        }
    } catch (BKClientClosedException err) {
        LOGGER.log(Level.FINE, "stop following " + tableSpaceDescription(), err);
    } catch (org.apache.bookkeeper.client.api.BKException err) {
        LOGGER.log(Level.SEVERE, tableSpaceDescription() + " internal BK error", err);
        throw new LogNotAvailableException(err);
    } catch (InterruptedException err) {
        LOGGER.log(Level.SEVERE, tableSpaceDescription() + " interrupted", err);
        Thread.currentThread().interrupt();
        throw new LogNotAvailableException(err);
    } catch (LogNotAvailableException err) {
        LOGGER.log(Level.SEVERE, tableSpaceDescription() + " internal error", err);
        throw err;
    } catch (Exception err) {
        LOGGER.log(Level.SEVERE, tableSpaceDescription() + " internal error", err);
        throw new LogNotAvailableException(err);
    }
}

Example 4 with ReadHandle

use of org.apache.bookkeeper.client.api.ReadHandle in project pravega by pravega.

the class BookKeeperLogReconcileCommand method execute.

@Override
public void execute() throws Exception {
    ensureArgCount(1);
    int logId = getIntArg(0);
    // Ensure that the Bookkeeper log is disabled; abort otherwise.
    @Cleanup val context = createContext();
    @Cleanup val log = context.logFactory.createDebugLogWrapper(logId);
    // Display a summary of the BookKeeperLog.
    val m = log.fetchMetadata();
    outputLogSummary(logId, m);
    if (m == null || m.isEnabled()) {
        String message = (m == null) ? "BookKeeperLog '%s' does not exist." : "BookKeeperLog '%s' is enabled. Please, disable it before executing this command.";
        output(message, logId);
        return;
    }
    // Once the Bookkeeper log is disabled, list all ledgers from this log. This implies to query all the ledgers
    // in Bookkeeper and filter out the ones related to BookkeeperLog id passed by parameter.
    ClientConfiguration config = new ClientConfiguration().setMetadataServiceUri("zk://" + this.getServiceConfig().getZkURL() + context.bookKeeperConfig.getBkLedgerPath());
    @Cleanup BookKeeper bkClient = BookKeeper.forConfig(config).build();
    @Cleanup LedgerManager manager = bkClient.getLedgerManager();
    LedgerManager.LedgerRangeIterator ledgerRangeIterator = manager.getLedgerRanges(Long.MAX_VALUE);
    List<ReadHandle> candidateLedgers = new ArrayList<>();
    try {
        while (ledgerRangeIterator.hasNext()) {
            LedgerManager.LedgerRange lr = ledgerRangeIterator.next();
            for (long ledgerId : lr.getLedgers()) {
                ReadHandle readHandle = Ledgers.openRead(ledgerId, bkClient, context.bookKeeperConfig);
                if (Ledgers.getBookKeeperLogId(readHandle) == logId) {
                    candidateLedgers.add(readHandle);
                }
            }
        }
        // If there are no candidate ledgers, just return.
        if (candidateLedgers.isEmpty()) {
            output("No candidate ledgers to reconcile.");
            return;
        }
        // Confirm with user prior executing the command.
        output("Candidate ledgers for reconciliation: %s", candidateLedgers.stream().map(String::valueOf).collect(Collectors.joining(",")));
        output("BookKeeperLog '%s' reconciliation is about to be executed.", logId);
        if (!confirmContinue()) {
            output("Not reconciling anything at this time.");
            return;
        }
        // Executing BookkeeperLog reconciliation.
        output("BookKeeperLog '%s': starting ledger reconciliation.", logId);
        log.reconcileLedgers(candidateLedgers);
        output("BookKeeperLog '%s': ledger reconciliation completed.", logId);
    } finally {
        // Closing opened ledgers.
        closeBookkeeperReadHandles(candidateLedgers);
    }
}

Example 5 with ReadHandle

use of org.apache.bookkeeper.client.api.ReadHandle in project pravega by pravega.

the class DebugBookKeeperLogWrapper method reconcileLedgers.

/**
 * Performs a {@link BookKeeperLog}-{@link LedgerHandle} reconciliation for this {@link BookKeeperLog} subject to the
 * following rules:
 * - Any {@link LedgerHandle}s that list this {@link BookKeeperLog} as their owner will be added to this {@link BookKeeperLog}'s
 * list of ledgers (if they're non-empty and haven't been truncated out).
 * - Any {@link LedgerMetadata} instances in this {@link BookKeeperLog} that point to inexistent {@link LedgerHandle}s
 * will be removed.
 *
 * @param candidateLedgers A List of {@link LedgerHandle}s that contain all the Ledgers that this {@link BookKeeperLog}
 *                         should contain. This could be the list of all BookKeeper Ledgers or a subset, as long as
 *                         it contains all Ledgers that list this {@link BookKeeperLog} as their owner.
 * @return True if something changed (and the metadata is updated), false otherwise.
 * @throws IllegalStateException   If this BookKeeperLog is not disabled.
 * @throws DurableDataLogException If an exception occurred while updating the metadata.
 */
public boolean reconcileLedgers(List<? extends ReadHandle> candidateLedgers) throws DurableDataLogException {
    // Load metadata and verify if disabled (metadata may be null if it doesn't exist).
    LogMetadata metadata = this.log.loadMetadata();
    final long highestLedgerId;
    if (metadata != null) {
        Preconditions.checkState(!metadata.isEnabled(), "BookKeeperLog is enabled; cannot reconcile ledgers.");
        int ledgerCount = metadata.getLedgers().size();
        if (ledgerCount > 0) {
            // Get the highest Ledger id from the list of ledgers.
            highestLedgerId = metadata.getLedgers().get(ledgerCount - 1).getLedgerId();
        } else if (metadata.getTruncationAddress() != null) {
            // All Ledgers have been truncated out. Get it from the Truncation Address.
            highestLedgerId = metadata.getTruncationAddress().getLedgerId();
        } else {
            // No information.
            highestLedgerId = Ledgers.NO_LEDGER_ID;
        }
    } else {
        // No metadata.
        highestLedgerId = Ledgers.NO_LEDGER_ID;
    }
    // First, we filter out any Ledger that does not reference this Log as their owner or that are empty.
    candidateLedgers = candidateLedgers.stream().filter(lh -> Ledgers.getBookKeeperLogId(lh) == this.log.getLogId() && lh.getLength() > 0).collect(Collectors.toList());
    // Begin reconstructing the Ledger List by eliminating references to inexistent ledgers.
    val newLedgerList = new ArrayList<LedgerMetadata>();
    if (metadata != null) {
        val candidateLedgerIds = candidateLedgers.stream().map(Handle::getId).collect(Collectors.toSet());
        metadata.getLedgers().stream().filter(lm -> candidateLedgerIds.contains(lm.getLedgerId())).forEach(newLedgerList::add);
    }
    // Find ledgers that should be in the log but are not referenced. Only select ledgers which have their Id greater
    // than the Id of the last ledger used in this Log (Id are assigned monotonically increasing, and we don't want
    // to add already truncated out ledgers).
    val seq = new AtomicInteger(newLedgerList.isEmpty() ? 0 : newLedgerList.get(newLedgerList.size() - 1).getSequence());
    candidateLedgers.stream().filter(lh -> lh.getId() > highestLedgerId).forEach(lh -> newLedgerList.add(new LedgerMetadata(lh.getId(), seq.incrementAndGet())));
    // Make sure the ledgers are properly sorted.
    newLedgerList.sort(Comparator.comparingLong(LedgerMetadata::getLedgerId));
    // Determine if anything changed.
    boolean changed = metadata == null || metadata.getLedgers().size() != newLedgerList.size();
    if (!changed) {
        for (int i = 0; i < newLedgerList.size(); i++) {
            if (metadata.getLedgers().get(i).getLedgerId() != newLedgerList.get(i).getLedgerId()) {
                changed = true;
                break;
            }
        }
    }
    // Update metadata in ZooKeeper, but only if it has changed.
    if (changed) {
        val newMetadata = LogMetadata.builder().enabled(false).epoch(getOrDefault(metadata, LogMetadata::getEpoch, LogMetadata.INITIAL_EPOCH) + 1).truncationAddress(getOrDefault(metadata, LogMetadata::getTruncationAddress, LogMetadata.INITIAL_TRUNCATION_ADDRESS)).updateVersion(getOrDefault(metadata, LogMetadata::getUpdateVersion, LogMetadata.INITIAL_VERSION)).ledgers(newLedgerList).build();
        this.log.overWriteMetadata(newMetadata);
    }
    return changed;
}