Search in sources :

Example 71 with ManagedLedgerException

use of org.apache.bookkeeper.mledger.ManagedLedgerException in project pulsar by yahoo.

the class ManagedLedgerImpl method asyncDeleteCursor.

@Override
public synchronized void asyncDeleteCursor(final String consumerName, final DeleteCursorCallback callback, final Object ctx) {
    final ManagedCursorImpl cursor = (ManagedCursorImpl) cursors.get(consumerName);
    if (cursor == null) {
        callback.deleteCursorFailed(new ManagedLedgerException("ManagedCursor not found: " + consumerName), ctx);
        return;
    }
    // First remove the consumer form the MetaStore. If this operation succeeds and the next one (removing the
    // ledger from BK) don't, we end up having a loose ledger leaked but the state will be consistent.
    store.asyncRemoveCursor(ManagedLedgerImpl.this.name, consumerName, new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void result, Stat stat) {
            cursor.asyncDeleteCursorLedger();
            cursors.removeCursor(consumerName);
            // Redo invalidation of entries in cache
            PositionImpl slowestConsumerPosition = cursors.getSlowestReaderPosition();
            if (slowestConsumerPosition != null) {
                if (log.isDebugEnabled()) {
                    log.debug("Doing cache invalidation up to {}", slowestConsumerPosition);
                }
                entryCache.invalidateEntries(slowestConsumerPosition);
            } else {
                entryCache.clear();
            }
            trimConsumedLedgersInBackground();
            log.info("[{}] [{}] Deleted cursor", name, consumerName);
            callback.deleteCursorComplete(ctx);
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            callback.deleteCursorFailed(e, ctx);
        }
    });
}
Also used : MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat)

Example 72 with ManagedLedgerException

use of org.apache.bookkeeper.mledger.ManagedLedgerException in project pulsar by yahoo.

the class ManagedLedgerImpl method asyncClose.

@Override
public synchronized void asyncClose(final CloseCallback callback, final Object ctx) {
    State state = STATE_UPDATER.get(this);
    if (state == State.Fenced) {
        factory.close(this);
        callback.closeFailed(new ManagedLedgerFencedException(), ctx);
        return;
    } else if (state == State.Closed) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Ignoring request to close a closed managed ledger", name);
        }
        callback.closeComplete(ctx);
        return;
    }
    log.info("[{}] Closing managed ledger", name);
    factory.close(this);
    STATE_UPDATER.set(this, State.Closed);
    LedgerHandle lh = currentLedger;
    if (log.isDebugEnabled()) {
        log.debug("[{}] Closing current writing ledger {}", name, lh.getId());
    }
    mbean.startDataLedgerCloseOp();
    lh.asyncClose((rc, lh1, ctx1) -> {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Close complete for ledger {}: rc = {}", name, lh.getId(), rc);
        }
        mbean.endDataLedgerCloseOp();
        if (rc != BKException.Code.OK) {
            callback.closeFailed(new ManagedLedgerException(BKException.getMessage(rc)), ctx);
            return;
        }
        // Close all cursors in parallel
        List<CompletableFuture<Void>> futures = Lists.newArrayList();
        for (ManagedCursor cursor : cursors) {
            Futures.CloseFuture closeFuture = new Futures.CloseFuture();
            cursor.asyncClose(closeFuture, null);
            futures.add(closeFuture);
        }
        Futures.waitForAll(futures).thenRun(() -> {
            callback.closeComplete(ctx);
        }).exceptionally(exception -> {
            callback.closeFailed(new ManagedLedgerException(exception), ctx);
            return null;
        });
    }, null);
}
Also used : ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CompletableFuture(java.util.concurrent.CompletableFuture) Futures(org.apache.bookkeeper.mledger.util.Futures) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor)

Example 73 with ManagedLedgerException

use of org.apache.bookkeeper.mledger.ManagedLedgerException in project pulsar by yahoo.

the class ManagedLedgerImpl method checkBackloggedCursors.

@Override
public void checkBackloggedCursors() {
    // activate caught up cursors
    cursors.forEach(cursor -> {
        if (cursor.getNumberOfEntries() < maxActiveCursorBacklogEntries) {
            cursor.setActive();
        }
    });
    // deactivate backlog cursors
    Iterator<ManagedCursor> cursors = activeCursors.iterator();
    while (cursors.hasNext()) {
        ManagedCursor cursor = cursors.next();
        long backlogEntries = cursor.getNumberOfEntries();
        if (backlogEntries > maxActiveCursorBacklogEntries) {
            PositionImpl readPosition = (PositionImpl) cursor.getReadPosition();
            readPosition = isValidPosition(readPosition) ? readPosition : getNextValidPosition(readPosition);
            if (readPosition == null) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Couldn't find valid read position [{}] {}", name, cursor.getName(), cursor.getReadPosition());
                }
                continue;
            }
            try {
                asyncReadEntry(readPosition, new ReadEntryCallback() {

                    @Override
                    public void readEntryFailed(ManagedLedgerException e, Object ctx) {
                        log.warn("[{}] Failed while reading entries on [{}] {}", name, cursor.getName(), e.getMessage(), e);
                    }

                    @Override
                    public void readEntryComplete(Entry entry, Object ctx) {
                        MessageMetadata msgMetadata = null;
                        try {
                            msgMetadata = Commands.parseMessageMetadata(entry.getDataBuffer());
                            long msgTimeSincePublish = (System.currentTimeMillis() - msgMetadata.getPublishTime());
                            if (msgTimeSincePublish > maxMessageCacheRetentionTimeMillis) {
                                cursor.setInactive();
                            }
                        } finally {
                            if (msgMetadata != null) {
                                msgMetadata.recycle();
                            }
                            entry.release();
                        }
                    }
                }, null);
            } catch (Exception e) {
                log.warn("[{}] Failed while reading entries from cache on [{}] {}", name, cursor.getName(), e.getMessage(), e);
            }
        }
    }
}
Also used : ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Entry(org.apache.bookkeeper.mledger.Entry) MessageMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata) ReadEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback) BKException(org.apache.bookkeeper.client.BKException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) BadVersionException(org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor)

Example 74 with ManagedLedgerException

use of org.apache.bookkeeper.mledger.ManagedLedgerException in project pulsar by yahoo.

the class ManagedLedgerImpl method asyncReadEntries.

void asyncReadEntries(OpReadEntry opReadEntry) {
    final State state = STATE_UPDATER.get(this);
    if (state == State.Fenced || state == State.Closed) {
        opReadEntry.readEntriesFailed(new ManagedLedgerFencedException(), opReadEntry.ctx);
        return;
    }
    long ledgerId = opReadEntry.readPosition.getLedgerId();
    LedgerHandle currentLedger = this.currentLedger;
    if (ledgerId == currentLedger.getId()) {
        // Current writing ledger is not in the cache (since we don't want
        // it to be automatically evicted), and we cannot use 2 different
        // ledger handles (read & write)for the same ledger.
        internalReadFromLedger(currentLedger, opReadEntry);
    } else {
        LedgerInfo ledgerInfo = ledgers.get(ledgerId);
        if (ledgerInfo == null || ledgerInfo.getEntries() == 0) {
            // Cursor is pointing to a empty ledger, there's no need to try opening it. Skip this ledger and
            // move to the next one
            opReadEntry.updateReadPosition(new PositionImpl(opReadEntry.readPosition.getLedgerId() + 1, 0));
            opReadEntry.checkReadCompletion();
            return;
        }
        // Get a ledger handle to read from
        getLedgerHandle(ledgerId).thenAccept(ledger -> {
            internalReadFromLedger(ledger, opReadEntry);
        }).exceptionally(ex -> {
            log.error("[{}] Error opening ledger for reading at position {} - {}", name, opReadEntry.readPosition, ex.getMessage());
            opReadEntry.readEntriesFailed(new ManagedLedgerException(ex), opReadEntry.ctx);
            return null;
        });
    }
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) OpenCallback(org.apache.bookkeeper.client.AsyncCallback.OpenCallback) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) LoggerFactory(org.slf4j.LoggerFactory) Random(java.util.Random) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) Pair(org.apache.bookkeeper.mledger.util.Pair) Unpooled(io.netty.buffer.Unpooled) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Map(java.util.Map) ManagedLedgerMXBean(org.apache.bookkeeper.mledger.ManagedLedgerMXBean) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) Commands(com.yahoo.pulsar.common.api.Commands) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) Range(com.google.common.collect.Range) Futures(org.apache.bookkeeper.mledger.util.Futures) Math.min(java.lang.Math.min) Position(org.apache.bookkeeper.mledger.Position) NavigableMap(java.util.NavigableMap) BookKeeper(org.apache.bookkeeper.client.BookKeeper) BKException(org.apache.bookkeeper.client.BKException) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Queues(com.google.common.collect.Queues) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) BoundType(com.google.common.collect.BoundType) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) ConcurrentLongHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentLongHashMap) Queue(java.util.Queue) BadVersionException(org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) CreateCallback(org.apache.bookkeeper.client.AsyncCallback.CreateCallback) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) Entry(org.apache.bookkeeper.mledger.Entry) CompletableFuture(java.util.concurrent.CompletableFuture) ReadEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback) RateLimiter(com.google.common.util.concurrent.RateLimiter) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Lists(com.google.common.collect.Lists) ByteBuf(io.netty.buffer.ByteBuf) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) OrderedSafeExecutor(org.apache.bookkeeper.util.OrderedSafeExecutor) java.util.concurrent.atomic(java.util.concurrent.atomic) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) MessageMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) CallbackMutex(org.apache.bookkeeper.mledger.util.CallbackMutex) VoidCallback(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback) UnboundArrayBlockingQueue(org.apache.bookkeeper.util.UnboundArrayBlockingQueue) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException)

Example 75 with ManagedLedgerException

use of org.apache.bookkeeper.mledger.ManagedLedgerException in project pulsar by yahoo.

the class ManagedLedgerImpl method deleteCursor.

@Override
public void deleteCursor(String name) throws InterruptedException, ManagedLedgerException {
    final CountDownLatch counter = new CountDownLatch(1);
    class Result {

        ManagedLedgerException exception = null;
    }
    final Result result = new Result();
    asyncDeleteCursor(name, new DeleteCursorCallback() {

        @Override
        public void deleteCursorComplete(Object ctx) {
            counter.countDown();
        }

        @Override
        public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
            result.exception = exception;
            counter.countDown();
        }
    }, null);
    if (!counter.await(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS)) {
        throw new ManagedLedgerException("Timeout during delete-cursors operation");
    }
    if (result.exception != null) {
        log.error("Deleting cursor", result.exception);
        throw result.exception;
    }
}
Also used : ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) CountDownLatch(java.util.concurrent.CountDownLatch)

Aggregations

ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)133 Test (org.testng.annotations.Test)70 ManagedLedger (org.apache.bookkeeper.mledger.ManagedLedger)65 CountDownLatch (java.util.concurrent.CountDownLatch)63 ManagedCursor (org.apache.bookkeeper.mledger.ManagedCursor)46 Entry (org.apache.bookkeeper.mledger.Entry)36 ManagedLedgerConfig (org.apache.bookkeeper.mledger.ManagedLedgerConfig)33 Position (org.apache.bookkeeper.mledger.Position)33 CompletableFuture (java.util.concurrent.CompletableFuture)23 AddEntryCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)21 AsyncCallbacks (org.apache.bookkeeper.mledger.AsyncCallbacks)19 List (java.util.List)18 ReadEntriesCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback)18 BKException (org.apache.bookkeeper.client.BKException)15 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)13 LedgerHandle (org.apache.bookkeeper.client.LedgerHandle)13 CloseCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback)13 MarkDeleteCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback)13 Logger (org.slf4j.Logger)13 LoggerFactory (org.slf4j.LoggerFactory)13