Examples with LedgerManager org.apache.bookkeeper.meta.LedgerManager used on opensource projects

Example 16 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project pravega by pravega.

the class BookKeeperListAllLedgersCommand method execute.

@Override
public void execute() throws Exception {
    ensureArgCount(0);
    @Cleanup val context = createContext();
    ClientConfiguration config = new ClientConfiguration().setMetadataServiceUri("zk://" + this.getServiceConfig().getZkURL() + context.bookKeeperConfig.getBkLedgerPath());
    @Cleanup BookKeeper bkClient = BookKeeper.forConfig(config).build();
    @Cleanup LedgerManager manager = bkClient.getLedgerManager();
    LedgerManager.LedgerRangeIterator ledgerRangeIterator = manager.getLedgerRanges(Long.MAX_VALUE);
    List<ReadHandle> candidateLedgers = new ArrayList<>();
    try {
        while (ledgerRangeIterator.hasNext()) {
            LedgerManager.LedgerRange lr = ledgerRangeIterator.next();
            for (long ledgerId : lr.getLedgers()) {
                candidateLedgers.add(Ledgers.openRead(ledgerId, bkClient, context.bookKeeperConfig));
            }
        }
        // Output all the ledgers found.
        output("List of ledgers in the system: ");
        for (ReadHandle rh : candidateLedgers) {
            output("%s, length: %d, lastEntryConfirmed: %d, ledgerMetadata: %s, bookieLogID: %d", rh.toString(), rh.getLength(), rh.readLastAddConfirmed(), rh.getLedgerMetadata().toSafeString(), Ledgers.getBookKeeperLogId(rh));
        }
    } finally {
        // Closing opened ledgers.
        closeBookkeeperReadHandles(candidateLedgers);
    }
}

Example 17 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project distributedlog by twitter.

the class DistributedLogInputFormat method getSplits.

@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    List<InputSplit> inputSplits = Lists.newArrayListWithCapacity(segments.size());
    BookKeeper bk = namespace.getReaderBKC().get();
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bk);
    final AtomicInteger rcHolder = new AtomicInteger(0);
    final AtomicReference<LedgerMetadata> metadataHolder = new AtomicReference<LedgerMetadata>(null);
    for (LogSegmentMetadata segment : segments) {
        final CountDownLatch latch = new CountDownLatch(1);
        lm.readLedgerMetadata(segment.getLedgerId(), new BookkeeperInternalCallbacks.GenericCallback<LedgerMetadata>() {

            @Override
            public void operationComplete(int rc, LedgerMetadata ledgerMetadata) {
                metadataHolder.set(ledgerMetadata);
                rcHolder.set(rc);
                latch.countDown();
            }
        });
        latch.await();
        if (BKException.Code.OK != rcHolder.get()) {
            throw new IOException("Faild to get log segment metadata for " + segment + " : " + BKException.getMessage(rcHolder.get()));
        }
        inputSplits.add(new LogSegmentSplit(segment, metadataHolder.get()));
    }
    return inputSplits;
}

Example 18 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project distributedlog by twitter.

the class DLAuditor method collectLedgersFromBK.

/**
 * Find leak ledgers phase 1: collect ledgers set.
 */
private Set<Long> collectLedgersFromBK(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final Set<Long> ledgers = new HashSet<Long>();
    final SettableFuture<Void> doneFuture = SettableFuture.create();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(Long lid, final AsyncCallback.VoidCallback cb) {
            synchronized (ledgers) {
                ledgers.add(lid);
                if (0 == ledgers.size() % 1000) {
                    logger.info("Collected {} ledgers", ledgers.size());
                }
            }
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    cb.processResult(BKException.Code.OK, null, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.set(null);
            } else {
                doneFuture.setException(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("Collected total {} ledgers", ledgers.size());
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on collecting ledgers : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to collect ledgers : ", e.getCause());
        }
    }
    return ledgers;
}

Example 19 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project distributedlog by twitter.

the class DLAuditor method calculateLedgerSpaceUsage.

private long calculateLedgerSpaceUsage(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    final AtomicLong totalBytes = new AtomicLong(0);
    final AtomicLong totalEntries = new AtomicLong(0);
    final AtomicLong numLedgers = new AtomicLong(0);
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final SettableFuture<Void> doneFuture = SettableFuture.create();
    final BookKeeper bk = bkc.get();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(final Long lid, final AsyncCallback.VoidCallback cb) {
            numLedgers.incrementAndGet();
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    bk.asyncOpenLedgerNoRecovery(lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8), new org.apache.bookkeeper.client.AsyncCallback.OpenCallback() {

                        @Override
                        public void openComplete(int rc, LedgerHandle lh, Object ctx) {
                            final int cbRc;
                            if (BKException.Code.OK == rc) {
                                totalBytes.addAndGet(lh.getLength());
                                totalEntries.addAndGet(lh.getLastAddConfirmed() + 1);
                                cbRc = rc;
                            } else {
                                cbRc = BKException.Code.ZKException;
                            }
                            executorService.submit(new Runnable() {

                                @Override
                                public void run() {
                                    cb.processResult(cbRc, null, null);
                                }
                            });
                        }
                    }, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.set(null);
            } else {
                doneFuture.setException(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("calculated {} ledgers\n\ttotal bytes = {}\n\ttotal entries = {}", new Object[] { numLedgers.get(), totalBytes.get(), totalEntries.get() });
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on calculating ledger space : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to calculate ledger space : ", e.getCause());
        }
    }
    return totalBytes.get();
}