Search in sources :

Example 1 with Handle

use of org.apache.bookkeeper.client.api.Handle in project pravega by pravega.

the class DebugBookKeeperLogWrapper method reconcileLedgers.

/**
 * Performs a {@link BookKeeperLog}-{@link LedgerHandle} reconciliation for this {@link BookKeeperLog} subject to the
 * following rules:
 * - Any {@link LedgerHandle}s that list this {@link BookKeeperLog} as their owner will be added to this {@link BookKeeperLog}'s
 * list of ledgers (if they're non-empty and haven't been truncated out).
 * - Any {@link LedgerMetadata} instances in this {@link BookKeeperLog} that point to inexistent {@link LedgerHandle}s
 * will be removed.
 *
 * @param candidateLedgers A List of {@link LedgerHandle}s that contain all the Ledgers that this {@link BookKeeperLog}
 *                         should contain. This could be the list of all BookKeeper Ledgers or a subset, as long as
 *                         it contains all Ledgers that list this {@link BookKeeperLog} as their owner.
 * @return True if something changed (and the metadata is updated), false otherwise.
 * @throws IllegalStateException   If this BookKeeperLog is not disabled.
 * @throws DurableDataLogException If an exception occurred while updating the metadata.
 */
public boolean reconcileLedgers(List<? extends ReadHandle> candidateLedgers) throws DurableDataLogException {
    // Load metadata and verify if disabled (metadata may be null if it doesn't exist).
    LogMetadata metadata = this.log.loadMetadata();
    final long highestLedgerId;
    if (metadata != null) {
        Preconditions.checkState(!metadata.isEnabled(), "BookKeeperLog is enabled; cannot reconcile ledgers.");
        int ledgerCount = metadata.getLedgers().size();
        if (ledgerCount > 0) {
            // Get the highest Ledger id from the list of ledgers.
            highestLedgerId = metadata.getLedgers().get(ledgerCount - 1).getLedgerId();
        } else if (metadata.getTruncationAddress() != null) {
            // All Ledgers have been truncated out. Get it from the Truncation Address.
            highestLedgerId = metadata.getTruncationAddress().getLedgerId();
        } else {
            // No information.
            highestLedgerId = Ledgers.NO_LEDGER_ID;
        }
    } else {
        // No metadata.
        highestLedgerId = Ledgers.NO_LEDGER_ID;
    }
    // First, we filter out any Ledger that does not reference this Log as their owner or that are empty.
    candidateLedgers = candidateLedgers.stream().filter(lh -> Ledgers.getBookKeeperLogId(lh) == this.log.getLogId() && lh.getLength() > 0).collect(Collectors.toList());
    // Begin reconstructing the Ledger List by eliminating references to inexistent ledgers.
    val newLedgerList = new ArrayList<LedgerMetadata>();
    if (metadata != null) {
        val candidateLedgerIds = candidateLedgers.stream().map(Handle::getId).collect(Collectors.toSet());
        metadata.getLedgers().stream().filter(lm -> candidateLedgerIds.contains(lm.getLedgerId())).forEach(newLedgerList::add);
    }
    // Find ledgers that should be in the log but are not referenced. Only select ledgers which have their Id greater
    // than the Id of the last ledger used in this Log (Id are assigned monotonically increasing, and we don't want
    // to add already truncated out ledgers).
    val seq = new AtomicInteger(newLedgerList.isEmpty() ? 0 : newLedgerList.get(newLedgerList.size() - 1).getSequence());
    candidateLedgers.stream().filter(lh -> lh.getId() > highestLedgerId).forEach(lh -> newLedgerList.add(new LedgerMetadata(lh.getId(), seq.incrementAndGet())));
    // Make sure the ledgers are properly sorted.
    newLedgerList.sort(Comparator.comparingLong(LedgerMetadata::getLedgerId));
    // Determine if anything changed.
    boolean changed = metadata == null || metadata.getLedgers().size() != newLedgerList.size();
    if (!changed) {
        for (int i = 0; i < newLedgerList.size(); i++) {
            if (metadata.getLedgers().get(i).getLedgerId() != newLedgerList.get(i).getLedgerId()) {
                changed = true;
                break;
            }
        }
    }
    // Update metadata in ZooKeeper, but only if it has changed.
    if (changed) {
        val newMetadata = LogMetadata.builder().enabled(false).epoch(getOrDefault(metadata, LogMetadata::getEpoch, LogMetadata.INITIAL_EPOCH) + 1).truncationAddress(getOrDefault(metadata, LogMetadata::getTruncationAddress, LogMetadata.INITIAL_TRUNCATION_ADDRESS)).updateVersion(getOrDefault(metadata, LogMetadata::getUpdateVersion, LogMetadata.INITIAL_VERSION)).ledgers(newLedgerList).build();
        this.log.overWriteMetadata(newMetadata);
    }
    return changed;
}
Also used : lombok.val(lombok.val) ThrottleSourceListener(io.pravega.segmentstore.storage.ThrottleSourceListener) RequiredArgsConstructor(lombok.RequiredArgsConstructor) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) LogAddress(io.pravega.segmentstore.storage.LogAddress) Function(java.util.function.Function) ArrayList(java.util.ArrayList) AccessLevel(lombok.AccessLevel) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Duration(java.time.Duration) DataLogInitializationException(io.pravega.segmentstore.storage.DataLogInitializationException) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) DurableDataLogException(io.pravega.segmentstore.storage.DurableDataLogException) BookKeeper(org.apache.bookkeeper.client.api.BookKeeper) DebugDurableDataLogWrapper(io.pravega.segmentstore.storage.DebugDurableDataLogWrapper) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) CloseableIterator(io.pravega.common.util.CloseableIterator) Handle(org.apache.bookkeeper.client.api.Handle) DurableDataLog(io.pravega.segmentstore.storage.DurableDataLog) lombok.val(lombok.val) ReadOnlyLogMetadata(io.pravega.segmentstore.storage.ReadOnlyLogMetadata) Collectors(java.util.stream.Collectors) ReadHandle(org.apache.bookkeeper.client.api.ReadHandle) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) QueueStats(io.pravega.segmentstore.storage.QueueStats) Preconditions(com.google.common.base.Preconditions) Comparator(java.util.Comparator) WriteSettings(io.pravega.segmentstore.storage.WriteSettings) CompositeArrayView(io.pravega.common.util.CompositeArrayView) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ArrayList(java.util.ArrayList) ReadOnlyLogMetadata(io.pravega.segmentstore.storage.ReadOnlyLogMetadata)

Aggregations

Preconditions (com.google.common.base.Preconditions)1 CloseableIterator (io.pravega.common.util.CloseableIterator)1 CompositeArrayView (io.pravega.common.util.CompositeArrayView)1 DataLogInitializationException (io.pravega.segmentstore.storage.DataLogInitializationException)1 DebugDurableDataLogWrapper (io.pravega.segmentstore.storage.DebugDurableDataLogWrapper)1 DurableDataLog (io.pravega.segmentstore.storage.DurableDataLog)1 DurableDataLogException (io.pravega.segmentstore.storage.DurableDataLogException)1 LogAddress (io.pravega.segmentstore.storage.LogAddress)1 QueueStats (io.pravega.segmentstore.storage.QueueStats)1 ReadOnlyLogMetadata (io.pravega.segmentstore.storage.ReadOnlyLogMetadata)1 ThrottleSourceListener (io.pravega.segmentstore.storage.ThrottleSourceListener)1 WriteSettings (io.pravega.segmentstore.storage.WriteSettings)1 Duration (java.time.Duration)1 ArrayList (java.util.ArrayList)1 Comparator (java.util.Comparator)1 List (java.util.List)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1