Examples with MetaStoreCallback org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback used on opensource projects

Example 1 with MetaStoreCallback

use of org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback in project pulsar by yahoo.

the class ManagedLedgerImpl method createComplete.

// //////////////////////////////////////////////////////////////////////
// Callbacks
@Override
public synchronized void createComplete(int rc, final LedgerHandle lh, Object ctx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] createComplete rc={} ledger={}", name, rc, lh != null ? lh.getId() : -1);
    }
    mbean.endDataLedgerCreateOp();
    if (rc != BKException.Code.OK) {
        log.error("[{}] Error creating ledger rc={} {}", name, rc, BKException.getMessage(rc));
        ManagedLedgerException status = new ManagedLedgerException(BKException.getMessage(rc));
        // Empty the list of pending requests and make all of them fail
        clearPendingAddEntries(status);
        lastLedgerCreationFailureTimestamp = System.currentTimeMillis();
        STATE_UPDATER.set(this, State.ClosedLedger);
    } else {
        log.info("[{}] Created new ledger {}", name, lh.getId());
        ledgers.put(lh.getId(), LedgerInfo.newBuilder().setLedgerId(lh.getId()).setTimestamp(0).build());
        currentLedger = lh;
        currentLedgerEntries = 0;
        currentLedgerSize = 0;
        final MetaStoreCallback<Void> cb = new MetaStoreCallback<Void>() {

            @Override
            public void operationComplete(Void v, Stat stat) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Updating of ledgers list after create complete. version={}", name, stat);
                }
                ledgersStat = stat;
                ledgersListMutex.unlock();
                updateLedgersIdsComplete(stat);
                synchronized (ManagedLedgerImpl.this) {
                    mbean.addLedgerSwitchLatencySample(System.nanoTime() - lastLedgerCreationInitiationTimestamp, TimeUnit.NANOSECONDS);
                }
            }

            @Override
            public void operationFailed(MetaStoreException e) {
                if (e instanceof BadVersionException) {
                    synchronized (ManagedLedgerImpl.this) {
                        log.error("[{}] Failed to udpate ledger list. z-node version mismatch. Closing managed ledger", name);
                        STATE_UPDATER.set(ManagedLedgerImpl.this, State.Fenced);
                        clearPendingAddEntries(e);
                        return;
                    }
                }
                log.warn("[{}] Error updating meta data with the new list of ledgers: {}", name, e.getMessage());
                // Remove the ledger, since we failed to update the list
                ledgers.remove(lh.getId());
                mbean.startDataLedgerDeleteOp();
                bookKeeper.asyncDeleteLedger(lh.getId(), (rc1, ctx1) -> {
                    mbean.endDataLedgerDeleteOp();
                    if (rc1 != BKException.Code.OK) {
                        log.warn("[{}] Failed to delete ledger {}: {}", name, lh.getId(), BKException.getMessage(rc1));
                    }
                }, null);
                ledgersListMutex.unlock();
                synchronized (ManagedLedgerImpl.this) {
                    lastLedgerCreationFailureTimestamp = System.currentTimeMillis();
                    STATE_UPDATER.set(ManagedLedgerImpl.this, State.ClosedLedger);
                    clearPendingAddEntries(e);
                }
            }
        };
        updateLedgersListAfterRollover(cb);
    }
}

Example 2 with MetaStoreCallback

use of org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback in project pulsar by yahoo.

the class ManagedLedgerImpl method internalTrimConsumedLedgers.

/**
     * Checks whether there are ledger that have been fully consumed and deletes them
     *
     * @throws Exception
     */
void internalTrimConsumedLedgers() {
    // Ensure only one trimming operation is active
    if (!trimmerMutex.tryLock()) {
        scheduleDeferredTrimming();
        return;
    }
    List<LedgerInfo> ledgersToDelete = Lists.newArrayList();
    synchronized (this) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(), TOTAL_SIZE_UPDATER.get(this));
        }
        if (STATE_UPDATER.get(this) == State.Closed) {
            log.debug("[{}] Ignoring trimming request since the managed ledger was already closed", name);
            trimmerMutex.unlock();
            return;
        }
        long slowestReaderLedgerId = -1;
        if (cursors.isEmpty()) {
            // At this point the lastLedger will be pointing to the
            // ledger that has just been closed, therefore the +1 to
            // include lastLedger in the trimming.
            slowestReaderLedgerId = currentLedger.getId() + 1;
        } else {
            PositionImpl slowestReaderPosition = cursors.getSlowestReaderPosition();
            if (slowestReaderPosition != null) {
                slowestReaderLedgerId = slowestReaderPosition.getLedgerId();
            } else {
                trimmerMutex.unlock();
                return;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Slowest consumer ledger id: {}", name, slowestReaderLedgerId);
        }
        // skip ledger if retention constraint met
        for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) {
            boolean expired = hasLedgerRetentionExpired(ls.getTimestamp());
            boolean overRetentionQuota = TOTAL_SIZE_UPDATER.get(this) > ((long) config.getRetentionSizeInMB()) * 1024 * 1024;
            if (ls.getLedgerId() == currentLedger.getId() || (!expired && !overRetentionQuota)) {
                if (log.isDebugEnabled()) {
                    if (!expired) {
                        log.debug("[{}] ledger id skipped for deletion as unexpired: {}", name, ls.getLedgerId());
                    }
                    if (!overRetentionQuota) {
                        log.debug("[{}] ledger id: {} skipped for deletion as size: {} under quota: {} MB", name, ls.getLedgerId(), TOTAL_SIZE_UPDATER.get(this), config.getRetentionSizeInMB());
                    }
                }
                break;
            }
            ledgersToDelete.add(ls);
            ledgerCache.remove(ls.getLedgerId());
        }
        if (ledgersToDelete.isEmpty()) {
            trimmerMutex.unlock();
            return;
        }
        if (// Give up now and schedule a new trimming
        STATE_UPDATER.get(this) == State.CreatingLedger || !ledgersListMutex.tryLock()) {
            // Avoid deadlocks with other operations updating the ledgers list
            scheduleDeferredTrimming();
            trimmerMutex.unlock();
            return;
        }
        // Update metadata
        for (LedgerInfo ls : ledgersToDelete) {
            ledgers.remove(ls.getLedgerId());
            NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, -ls.getEntries());
            TOTAL_SIZE_UPDATER.addAndGet(this, -ls.getSize());
            entryCache.invalidateAllEntries(ls.getLedgerId());
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Updating of ledgers list after trimming", name);
        }
        ManagedLedgerInfo mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()).build();
        store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, new MetaStoreCallback<Void>() {

            @Override
            public void operationComplete(Void result, Stat stat) {
                log.info("[{}] End TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.size(), TOTAL_SIZE_UPDATER.get(ManagedLedgerImpl.this));
                ledgersStat = stat;
                ledgersListMutex.unlock();
                trimmerMutex.unlock();
                for (LedgerInfo ls : ledgersToDelete) {
                    log.info("[{}] Removing ledger {} - size: {}", name, ls.getLedgerId(), ls.getSize());
                    bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx) -> {
                        if (rc == BKException.Code.NoSuchLedgerExistsException) {
                            log.warn("[{}] Ledger was already deleted {}", name, ls.getLedgerId());
                        } else if (rc != BKException.Code.OK) {
                            log.error("[{}] Error deleting ledger {}", name, ls.getLedgerId(), BKException.getMessage(rc));
                        } else {
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Deleted ledger {}", name, ls.getLedgerId());
                            }
                        }
                    }, null);
                }
            }

            @Override
            public void operationFailed(MetaStoreException e) {
                log.warn("[{}] Failed to update the list of ledgers after trimming", name, e);
                ledgersListMutex.unlock();
                trimmerMutex.unlock();
            }
        });
    }
}

Example 3 with MetaStoreCallback

use of org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback in project pulsar by yahoo.

the class ManagedLedgerImpl method initializeBookKeeper.

private synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers);
    }
    // Calculate total entries and size
    Iterator<LedgerInfo> iterator = ledgers.values().iterator();
    while (iterator.hasNext()) {
        LedgerInfo li = iterator.next();
        if (li.getEntries() > 0) {
            NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, li.getEntries());
            TOTAL_SIZE_UPDATER.addAndGet(this, li.getSize());
        } else {
            iterator.remove();
            bookKeeper.asyncDeleteLedger(li.getLedgerId(), (rc, ctx) -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Deleted empty ledger ledgerId={} rc={}", name, li.getLedgerId(), rc);
                }
            }, null);
        }
    }
    final MetaStoreCallback<Void> storeLedgersCb = new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void v, Stat stat) {
            ledgersStat = stat;
            initializeCursors(callback);
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            callback.initializeFailed(new ManagedLedgerException(e));
        }
    };
    // Create a new ledger to start writing
    this.lastLedgerCreationInitiationTimestamp = System.nanoTime();
    mbean.startDataLedgerCreateOp();
    bookKeeper.asyncCreateLedger(config.getEnsembleSize(), config.getWriteQuorumSize(), config.getAckQuorumSize(), config.getDigestType(), config.getPassword(), (rc, lh, ctx) -> {
        executor.submitOrdered(name, safeRun(() -> {
            mbean.endDataLedgerCreateOp();
            if (rc != BKException.Code.OK) {
                callback.initializeFailed(new ManagedLedgerException(BKException.getMessage(rc)));
                return;
            }
            log.info("[{}] Created ledger {}", name, lh.getId());
            STATE_UPDATER.set(this, State.LedgerOpened);
            lastLedgerCreatedTimestamp = System.currentTimeMillis();
            currentLedger = lh;
            lastConfirmedEntry = new PositionImpl(lh.getId(), -1);
            LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lh.getId()).setTimestamp(0).build();
            ledgers.put(lh.getId(), info);
            // Save it back to ensure all nodes exist
            ManagedLedgerInfo mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()).build();
            store.asyncUpdateLedgerIds(name, mlInfo, ledgersStat, storeLedgersCb);
        }));
    }, null);
}

Example 4 with MetaStoreCallback

use of org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback in project pulsar by yahoo.

the class MetaStoreImplZookeeperTest method updatingMLNode.

@Test(timeOut = 20000)
void updatingMLNode() throws Exception {
    final MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
    zkc.create("/managed-ledgers/my_test", "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    final CountDownLatch latch = new CountDownLatch(1);
    store.getManagedLedgerInfo("my_test", new MetaStoreCallback<ManagedLedgerInfo>() {

        public void operationFailed(MetaStoreException e) {
            fail("should have succeeded");
        }

        public void operationComplete(ManagedLedgerInfo mlInfo, Stat version) {
            // Update again using the version
            zkc.failNow(Code.BADVERSION);
            store.asyncUpdateLedgerIds("my_test", mlInfo, version, new MetaStoreCallback<Void>() {

                public void operationFailed(MetaStoreException e) {
                    // ok
                    latch.countDown();
                }

                @Override
                public void operationComplete(Void result, Stat version) {
                    fail("should have failed");
                }
            });
        }
    });
    latch.await();
}

Example 5 with MetaStoreCallback

use of org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback in project pulsar by yahoo.

the class MetaStoreImplZookeeperTest method updatingCursorNode.

@Test(timeOut = 20000)
void updatingCursorNode() throws Exception {
    final MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
    zkc.create("/managed-ledgers/my_test", "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    final CountDownLatch latch = new CountDownLatch(1);
    ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(1).build();
    store.asyncUpdateCursorInfo("my_test", "c1", info, null, new MetaStoreCallback<Void>() {

        public void operationFailed(MetaStoreException e) {
            fail("should have succeeded");
        }

        public void operationComplete(Void result, Stat version) {
            // Update again using the version
            zkc.failNow(Code.CONNECTIONLOSS);
            ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(2).build();
            store.asyncUpdateCursorInfo("my_test", "c1", info, version, new MetaStoreCallback<Void>() {

                public void operationFailed(MetaStoreException e) {
                    // ok
                    latch.countDown();
                }

                @Override
                public void operationComplete(Void result, Stat version) {
                    fail("should have failed");
                }
            });
        }
    });
    latch.await();
}