Search in sources :

Example 1 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method initialize.

synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callback, final Object ctx) {
    log.info("Opening managed ledger {}", name);
    // Fetch the list of existing ledgers in the managed ledger
    store.getManagedLedgerInfo(name, new MetaStoreCallback<ManagedLedgerInfo>() {

        @Override
        public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
            ledgersStat = stat;
            for (LedgerInfo ls : mlInfo.getLedgerInfoList()) {
                ledgers.put(ls.getLedgerId(), ls);
            }
            // Last ledger stat may be zeroed, we must update it
            if (ledgers.size() > 0) {
                final long id = ledgers.lastKey();
                OpenCallback opencb = (rc, lh, ctx1) -> {
                    executor.submitOrdered(name, safeRun(() -> {
                        mbean.endDataLedgerOpenOp();
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] Opened ledger {}: ", name, id, BKException.getMessage(rc));
                        }
                        if (rc == BKException.Code.OK) {
                            LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(id).setEntries(lh.getLastAddConfirmed() + 1).setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build();
                            ledgers.put(id, info);
                            initializeBookKeeper(callback);
                        } else if (rc == BKException.Code.NoSuchLedgerExistsException) {
                            log.warn("[{}] Ledger not found: {}", name, ledgers.lastKey());
                            ledgers.remove(ledgers.lastKey());
                            initializeBookKeeper(callback);
                        } else {
                            log.error("[{}] Failed to open ledger {}: {}", name, id, BKException.getMessage(rc));
                            callback.initializeFailed(new ManagedLedgerException(BKException.getMessage(rc)));
                            return;
                        }
                    }));
                };
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Opening legder {}", name, id);
                }
                mbean.startDataLedgerOpenOp();
                bookKeeper.asyncOpenLedger(id, config.getDigestType(), config.getPassword(), opencb, null);
            } else {
                initializeBookKeeper(callback);
            }
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            callback.initializeFailed(new ManagedLedgerException(e));
        }
    });
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) OpenCallback(org.apache.bookkeeper.client.AsyncCallback.OpenCallback) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo)

Example 2 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method internalTrimConsumedLedgers.

/**
     * Checks whether there are ledger that have been fully consumed and deletes them
     *
     * @throws Exception
     */
void internalTrimConsumedLedgers() {
    // Ensure only one trimming operation is active
    if (!trimmerMutex.tryLock()) {
        scheduleDeferredTrimming();
        return;
    }
    List<LedgerInfo> ledgersToDelete = Lists.newArrayList();
    synchronized (this) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(), TOTAL_SIZE_UPDATER.get(this));
        }
        if (STATE_UPDATER.get(this) == State.Closed) {
            log.debug("[{}] Ignoring trimming request since the managed ledger was already closed", name);
            trimmerMutex.unlock();
            return;
        }
        long slowestReaderLedgerId = -1;
        if (cursors.isEmpty()) {
            // At this point the lastLedger will be pointing to the
            // ledger that has just been closed, therefore the +1 to
            // include lastLedger in the trimming.
            slowestReaderLedgerId = currentLedger.getId() + 1;
        } else {
            PositionImpl slowestReaderPosition = cursors.getSlowestReaderPosition();
            if (slowestReaderPosition != null) {
                slowestReaderLedgerId = slowestReaderPosition.getLedgerId();
            } else {
                trimmerMutex.unlock();
                return;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Slowest consumer ledger id: {}", name, slowestReaderLedgerId);
        }
        // skip ledger if retention constraint met
        for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) {
            boolean expired = hasLedgerRetentionExpired(ls.getTimestamp());
            boolean overRetentionQuota = TOTAL_SIZE_UPDATER.get(this) > ((long) config.getRetentionSizeInMB()) * 1024 * 1024;
            if (ls.getLedgerId() == currentLedger.getId() || (!expired && !overRetentionQuota)) {
                if (log.isDebugEnabled()) {
                    if (!expired) {
                        log.debug("[{}] ledger id skipped for deletion as unexpired: {}", name, ls.getLedgerId());
                    }
                    if (!overRetentionQuota) {
                        log.debug("[{}] ledger id: {} skipped for deletion as size: {} under quota: {} MB", name, ls.getLedgerId(), TOTAL_SIZE_UPDATER.get(this), config.getRetentionSizeInMB());
                    }
                }
                break;
            }
            ledgersToDelete.add(ls);
            ledgerCache.remove(ls.getLedgerId());
        }
        if (ledgersToDelete.isEmpty()) {
            trimmerMutex.unlock();
            return;
        }
        if (// Give up now and schedule a new trimming
        STATE_UPDATER.get(this) == State.CreatingLedger || !ledgersListMutex.tryLock()) {
            // Avoid deadlocks with other operations updating the ledgers list
            scheduleDeferredTrimming();
            trimmerMutex.unlock();
            return;
        }
        // Update metadata
        for (LedgerInfo ls : ledgersToDelete) {
            ledgers.remove(ls.getLedgerId());
            NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, -ls.getEntries());
            TOTAL_SIZE_UPDATER.addAndGet(this, -ls.getSize());
            entryCache.invalidateAllEntries(ls.getLedgerId());
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Updating of ledgers list after trimming", name);
        }
        ManagedLedgerInfo mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()).build();
        store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, new MetaStoreCallback<Void>() {

            @Override
            public void operationComplete(Void result, Stat stat) {
                log.info("[{}] End TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.size(), TOTAL_SIZE_UPDATER.get(ManagedLedgerImpl.this));
                ledgersStat = stat;
                ledgersListMutex.unlock();
                trimmerMutex.unlock();
                for (LedgerInfo ls : ledgersToDelete) {
                    log.info("[{}] Removing ledger {} - size: {}", name, ls.getLedgerId(), ls.getSize());
                    bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx) -> {
                        if (rc == BKException.Code.NoSuchLedgerExistsException) {
                            log.warn("[{}] Ledger was already deleted {}", name, ls.getLedgerId());
                        } else if (rc != BKException.Code.OK) {
                            log.error("[{}] Error deleting ledger {}", name, ls.getLedgerId(), BKException.getMessage(rc));
                        } else {
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Deleted ledger {}", name, ls.getLedgerId());
                            }
                        }
                    }, null);
                }
            }

            @Override
            public void operationFailed(MetaStoreException e) {
                log.warn("[{}] Failed to update the list of ledgers after trimming", name, e);
                ledgersListMutex.unlock();
                trimmerMutex.unlock();
            }
        });
    }
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) OpenCallback(org.apache.bookkeeper.client.AsyncCallback.OpenCallback) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) LoggerFactory(org.slf4j.LoggerFactory) Random(java.util.Random) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) Pair(org.apache.bookkeeper.mledger.util.Pair) Unpooled(io.netty.buffer.Unpooled) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Map(java.util.Map) ManagedLedgerMXBean(org.apache.bookkeeper.mledger.ManagedLedgerMXBean) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) Commands(com.yahoo.pulsar.common.api.Commands) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) Range(com.google.common.collect.Range) Futures(org.apache.bookkeeper.mledger.util.Futures) Math.min(java.lang.Math.min) Position(org.apache.bookkeeper.mledger.Position) NavigableMap(java.util.NavigableMap) BookKeeper(org.apache.bookkeeper.client.BookKeeper) BKException(org.apache.bookkeeper.client.BKException) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Queues(com.google.common.collect.Queues) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) BoundType(com.google.common.collect.BoundType) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) ConcurrentLongHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentLongHashMap) Queue(java.util.Queue) BadVersionException(org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) CreateCallback(org.apache.bookkeeper.client.AsyncCallback.CreateCallback) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) Entry(org.apache.bookkeeper.mledger.Entry) CompletableFuture(java.util.concurrent.CompletableFuture) ReadEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback) RateLimiter(com.google.common.util.concurrent.RateLimiter) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Lists(com.google.common.collect.Lists) ByteBuf(io.netty.buffer.ByteBuf) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) OrderedSafeExecutor(org.apache.bookkeeper.util.OrderedSafeExecutor) java.util.concurrent.atomic(java.util.concurrent.atomic) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) MessageMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) CallbackMutex(org.apache.bookkeeper.mledger.util.CallbackMutex) VoidCallback(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback) UnboundArrayBlockingQueue(org.apache.bookkeeper.util.UnboundArrayBlockingQueue) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo)

Example 3 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method deleteAllLedgers.

private void deleteAllLedgers(DeleteLedgerCallback callback, Object ctx) {
    List<LedgerInfo> ledgers = Lists.newArrayList(ManagedLedgerImpl.this.ledgers.values());
    AtomicInteger ledgersToDelete = new AtomicInteger(ledgers.size());
    if (ledgers.isEmpty()) {
        // No ledgers to delete, proceed with deleting metadata
        deleteMetadata(callback, ctx);
        return;
    }
    for (LedgerInfo ls : ledgers) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Deleting ledger {}", name, ls);
        }
        bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx1) -> {
            switch(rc) {
                case BKException.Code.NoSuchLedgerExistsException:
                    log.warn("[{}] Ledger {} not found when deleting it", name, ls.getLedgerId());
                case BKException.Code.OK:
                    if (ledgersToDelete.decrementAndGet() == 0) {
                        // All ledgers deleted, now remove ML metadata
                        deleteMetadata(callback, ctx);
                    }
                    break;
                default:
                    // Handle error
                    log.warn("[{}] Failed to delete ledger {} -- {}", name, ls.getLedgerId(), BKException.getMessage(rc));
                    int toDelete = ledgersToDelete.get();
                    if (toDelete != -1 && ledgersToDelete.compareAndSet(toDelete, -1)) {
                        // Trigger callback only once
                        callback.deleteLedgerFailed(new ManagedLedgerException(BKException.getMessage(rc)), ctx);
                    }
            }
        }, null);
    }
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException)

Example 4 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method initializeBookKeeper.

private synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers);
    }
    // Calculate total entries and size
    Iterator<LedgerInfo> iterator = ledgers.values().iterator();
    while (iterator.hasNext()) {
        LedgerInfo li = iterator.next();
        if (li.getEntries() > 0) {
            NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, li.getEntries());
            TOTAL_SIZE_UPDATER.addAndGet(this, li.getSize());
        } else {
            iterator.remove();
            bookKeeper.asyncDeleteLedger(li.getLedgerId(), (rc, ctx) -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Deleted empty ledger ledgerId={} rc={}", name, li.getLedgerId(), rc);
                }
            }, null);
        }
    }
    final MetaStoreCallback<Void> storeLedgersCb = new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void v, Stat stat) {
            ledgersStat = stat;
            initializeCursors(callback);
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            callback.initializeFailed(new ManagedLedgerException(e));
        }
    };
    // Create a new ledger to start writing
    this.lastLedgerCreationInitiationTimestamp = System.nanoTime();
    mbean.startDataLedgerCreateOp();
    bookKeeper.asyncCreateLedger(config.getEnsembleSize(), config.getWriteQuorumSize(), config.getAckQuorumSize(), config.getDigestType(), config.getPassword(), (rc, lh, ctx) -> {
        executor.submitOrdered(name, safeRun(() -> {
            mbean.endDataLedgerCreateOp();
            if (rc != BKException.Code.OK) {
                callback.initializeFailed(new ManagedLedgerException(BKException.getMessage(rc)));
                return;
            }
            log.info("[{}] Created ledger {}", name, lh.getId());
            STATE_UPDATER.set(this, State.LedgerOpened);
            lastLedgerCreatedTimestamp = System.currentTimeMillis();
            currentLedger = lh;
            lastConfirmedEntry = new PositionImpl(lh.getId(), -1);
            LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lh.getId()).setTimestamp(0).build();
            ledgers.put(lh.getId(), info);
            // Save it back to ensure all nodes exist
            ManagedLedgerInfo mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()).build();
            store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, storeLedgersCb);
        }));
    }, null);
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo)

Example 5 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerTest method testTimestampOnWorkingLedger.

@Test
public void testTimestampOnWorkingLedger() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ManagedLedgerConfig conf = new ManagedLedgerConfig();
    conf.setMaxEntriesPerLedger(1);
    conf.setRetentionSizeInMB(10);
    conf.setRetentionTime(1, TimeUnit.HOURS);
    ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("my_test_ledger", conf);
    ml.openCursor("c1");
    ml.addEntry("msg1".getBytes());
    Iterator<LedgerInfo> iter = ml.getLedgersInfoAsList().iterator();
    long ts = -1;
    while (iter.hasNext()) {
        LedgerInfo i = iter.next();
        if (iter.hasNext()) {
            assertTrue(ts <= i.getTimestamp(), i.toString());
            ts = i.getTimestamp();
        } else {
            // the last timestamp can be
            // 0 if it is still opened
            // >0 if it is closed after the addEntry see OpAddEntry#addComplete()
            assertTrue(i.getTimestamp() == 0 || ts <= i.getTimestamp(), i.toString());
        }
    }
    ml.addEntry("msg02".getBytes());
    ml.close();
    // Thread.sleep(1000);
    iter = ml.getLedgersInfoAsList().iterator();
    ts = -1;
    while (iter.hasNext()) {
        LedgerInfo i = iter.next();
        if (iter.hasNext()) {
            assertTrue(ts <= i.getTimestamp(), i.toString());
            ts = i.getTimestamp();
        } else {
            assertTrue(i.getTimestamp() > 0, "well closed LedgerInfo should set a timestamp > 0");
        }
    }
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Test(org.testng.annotations.Test)

Aggregations

ManagedLedgerInfo (org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo)11 LedgerInfo (org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo)11 MetaStoreException (org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException)7 Stat (org.apache.bookkeeper.mledger.impl.MetaStore.Stat)7 CountDownLatch (java.util.concurrent.CountDownLatch)5 MetaStoreCallback (org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback)5 OpenCallback (org.apache.bookkeeper.client.AsyncCallback.OpenCallback)4 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)4 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 BoundType (com.google.common.collect.BoundType)3 Lists (com.google.common.collect.Lists)3 Maps (com.google.common.collect.Maps)3 Queues (com.google.common.collect.Queues)3 Range (com.google.common.collect.Range)3 RateLimiter (com.google.common.util.concurrent.RateLimiter)3 Commands (com.yahoo.pulsar.common.api.Commands)3 MessageMetadata (com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata)3 ConcurrentLongHashMap (com.yahoo.pulsar.common.util.collections.ConcurrentLongHashMap)3 ByteBuf (io.netty.buffer.ByteBuf)3 Unpooled (io.netty.buffer.Unpooled)3