Examples with LedgerManager org.apache.bookkeeper.meta.LedgerManager used on opensource projects

Example 1 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class DLAuditor method calculateLedgerSpaceUsage.

private long calculateLedgerSpaceUsage(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    final AtomicLong totalBytes = new AtomicLong(0);
    final AtomicLong totalEntries = new AtomicLong(0);
    final AtomicLong numLedgers = new AtomicLong(0);
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final CompletableFuture<Void> doneFuture = FutureUtils.createFuture();
    final BookKeeper bk = bkc.get();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(final Long lid, final AsyncCallback.VoidCallback cb) {
            numLedgers.incrementAndGet();
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    bk.asyncOpenLedgerNoRecovery(lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8), new org.apache.bookkeeper.client.AsyncCallback.OpenCallback() {

                        @Override
                        public void openComplete(int rc, LedgerHandle lh, Object ctx) {
                            final int cbRc;
                            if (BKException.Code.OK == rc) {
                                totalBytes.addAndGet(lh.getLength());
                                totalEntries.addAndGet(lh.getLastAddConfirmed() + 1);
                                cbRc = rc;
                            } else {
                                cbRc = BKException.Code.ZKException;
                            }
                            executorService.submit(new Runnable() {

                                @Override
                                public void run() {
                                    cb.processResult(cbRc, null, null);
                                }
                            });
                        }
                    }, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.complete(null);
            } else {
                doneFuture.completeExceptionally(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("calculated {} ledgers\n\ttotal bytes = {}\n\ttotal entries = {}", new Object[] { numLedgers.get(), totalBytes.get(), totalEntries.get() });
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on calculating ledger space : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to calculate ledger space : ", e.getCause());
        }
    }
    return totalBytes.get();
}

Example 2 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class AuditorPeriodicBookieCheckTest method testPeriodicBookieCheckInterval.

/**
 * Test that the periodic bookie checker works.
 */
@Test
public void testPeriodicBookieCheckInterval() throws Exception {
    bsConfs.get(0).setZkServers(zkUtil.getZooKeeperConnectString());
    runFunctionWithLedgerManagerFactory(bsConfs.get(0), mFactory -> {
        try (LedgerManager ledgerManager = mFactory.newLedgerManager()) {
            @Cleanup final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
            LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
            LedgerMetadata md = LedgerHandleAdapter.getLedgerMetadata(lh);
            List<BookieSocketAddress> ensemble = md.getEnsembles().get(0L);
            ensemble.set(0, new BookieSocketAddress("1.1.1.1", 1000));
            TestCallbacks.GenericCallbackFuture<Void> cb = new TestCallbacks.GenericCallbackFuture<Void>();
            ledgerManager.writeLedgerMetadata(lh.getId(), md, cb);
            cb.get();
            long underReplicatedLedger = -1;
            for (int i = 0; i < 10; i++) {
                underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
                if (underReplicatedLedger != -1) {
                    break;
                }
                Thread.sleep(CHECK_INTERVAL * 1000);
            }
            assertEquals("Ledger should be under replicated", lh.getId(), underReplicatedLedger);
        } catch (Exception e) {
            throw new UncheckedExecutionException(e.getMessage(), e);
        }
        return null;
    });
}

Example 3 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class ListLedgerService method handle.

@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    // parameter could be like: print_metadata=true&page=PageIndex
    if (HttpServer.Method.GET == request.getMethod()) {
        Map<String, String> params = request.getParams();
        // default not print metadata
        boolean printMeta = (params != null) && params.containsKey("print_metadata") && params.get("print_metadata").equals("true");
        // Page index should start from 1;
        int pageIndex = (printMeta && params.containsKey("page")) ? Integer.parseInt(params.get("page")) : -1;
        LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
        LedgerManager manager = mFactory.newLedgerManager();
        LedgerManager.LedgerRangeIterator iter = manager.getLedgerRanges();
        // output <ledgerId: ledgerMetadata>
        LinkedHashMap<String, String> output = Maps.newLinkedHashMap();
        // futures for readLedgerMetadata for each page.
        List<ReadLedgerMetadataCallback> futures = Lists.newArrayListWithExpectedSize(LIST_LEDGER_BATCH_SIZE);
        if (printMeta) {
            int ledgerIndex = 0;
            // start and end ledger index for wanted page.
            int startLedgerIndex = 0;
            int endLedgerIndex = 0;
            if (pageIndex > 0) {
                startLedgerIndex = (pageIndex - 1) * LIST_LEDGER_BATCH_SIZE;
                endLedgerIndex = startLedgerIndex + LIST_LEDGER_BATCH_SIZE - 1;
            }
            // get metadata
            while (iter.hasNext()) {
                LedgerManager.LedgerRange r = iter.next();
                for (Long lid : r.getLedgers()) {
                    ledgerIndex++;
                    if (// no actual page parameter provided
                    endLedgerIndex == 0 || (ledgerIndex >= startLedgerIndex && ledgerIndex <= endLedgerIndex)) {
                        ReadLedgerMetadataCallback cb = new ReadLedgerMetadataCallback(lid);
                        manager.readLedgerMetadata(lid, cb);
                        futures.add(cb);
                    }
                }
                if (futures.size() >= LIST_LEDGER_BATCH_SIZE) {
                    while (futures.size() > 0) {
                        ReadLedgerMetadataCallback cb = futures.remove(0);
                        keepLedgerMetadata(cb, output);
                    }
                }
            }
            while (futures.size() > 0) {
                ReadLedgerMetadataCallback cb = futures.remove(0);
                keepLedgerMetadata(cb, output);
            }
        } else {
            while (iter.hasNext()) {
                LedgerManager.LedgerRange r = iter.next();
                for (Long lid : r.getLedgers()) {
                    output.put(lid.toString(), null);
                }
            }
        }
        manager.close();
        String jsonResponse = JsonUtil.toJson(output);
        LOG.debug("output body:" + jsonResponse);
        response.setBody(jsonResponse);
        response.setCode(HttpServer.StatusCode.OK);
        return response;
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be GET method");
        return response;
    }
}

Example 4 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class MockBookKeeperTestCase method setup.

@Before
public void setup() throws Exception {
    mockLedgerMetadataRegistry = new ConcurrentHashMap<>();
    mockLedgerData = new ConcurrentHashMap<>();
    mockNextLedgerId = new AtomicLong(1);
    fencedLedgers = new ConcurrentSkipListSet<>();
    scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(4).name("bk-test").build();
    executor = OrderedExecutor.newBuilder().build();
    bookieWatcher = mock(BookieWatcher.class);
    bookieClient = mock(BookieClient.class);
    ledgerManager = mock(LedgerManager.class);
    ledgerIdGenerator = mock(LedgerIdGenerator.class);
    bk = mock(BookKeeper.class);
    NullStatsLogger nullStatsLogger = setupLoggers();
    failedBookies = new ArrayList<>();
    availableBookies = new HashSet<>();
    when(bk.getCloseLock()).thenReturn(new ReentrantReadWriteLock());
    when(bk.isClosed()).thenReturn(false);
    when(bk.getBookieWatcher()).thenReturn(bookieWatcher);
    when(bk.getDisableEnsembleChangeFeature()).thenReturn(mock(Feature.class));
    when(bk.getExplicitLacInterval()).thenReturn(0);
    when(bk.getMainWorkerPool()).thenReturn(executor);
    when(bk.getBookieClient()).thenReturn(bookieClient);
    when(bk.getScheduler()).thenReturn(scheduler);
    when(bk.getReadSpeculativeRequestPolicy()).thenReturn(Optional.absent());
    mockBookKeeperGetConf(new ClientConfiguration());
    when(bk.getStatsLogger()).thenReturn(nullStatsLogger);
    when(bk.getLedgerManager()).thenReturn(ledgerManager);
    when(bk.getLedgerIdGenerator()).thenReturn(ledgerIdGenerator);
    when(bk.getReturnRc(anyInt())).thenAnswer(invocationOnMock -> invocationOnMock.getArgument(0));
    setupLedgerIdGenerator();
    setupCreateLedgerMetadata();
    setupReadLedgerMetadata();
    setupWriteLedgerMetadata();
    setupRemoveLedgerMetadata();
    setupRegisterLedgerMetadataListener();
    setupBookieWatcherForNewEnsemble();
    setupBookieWatcherForEnsembleChange();
    setupBookieClientReadEntry();
    setupBookieClientAddEntry();
}

Example 5 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class TestWatchEnsembleChange method testWatchMetadataRemoval.

private void testWatchMetadataRemoval(LedgerManagerFactory factory) throws Exception {
    @Cleanup final LedgerManager manager = factory.newLedgerManager();
    @Cleanup LedgerIdGenerator idGenerator = factory.newLedgerIdGenerator();
    final ByteBuffer bbLedgerId = ByteBuffer.allocate(8);
    final CountDownLatch createLatch = new CountDownLatch(1);
    final CountDownLatch removeLatch = new CountDownLatch(1);
    idGenerator.generateLedgerId(new GenericCallback<Long>() {

        @Override
        public void operationComplete(int rc, final Long lid) {
            manager.createLedgerMetadata(lid, new LedgerMetadata(4, 2, 2, digestType, "fpj was here".getBytes()), new BookkeeperInternalCallbacks.GenericCallback<Void>() {

                @Override
                public void operationComplete(int rc, Void result) {
                    bbLedgerId.putLong(lid);
                    bbLedgerId.flip();
                    createLatch.countDown();
                }
            });
        }
    });
    assertTrue(createLatch.await(2000, TimeUnit.MILLISECONDS));
    final long createdLid = bbLedgerId.getLong();
    manager.registerLedgerMetadataListener(createdLid, new LedgerMetadataListener() {

        @Override
        public void onChanged(long ledgerId, LedgerMetadata metadata) {
            assertEquals(ledgerId, createdLid);
            assertEquals(metadata, null);
            removeLatch.countDown();
        }
    });
    manager.removeLedgerMetadata(createdLid, Version.ANY, new BookkeeperInternalCallbacks.GenericCallback<Void>() {

        @Override
        public void operationComplete(int rc, Void result) {
            assertEquals(rc, BKException.Code.OK);
        }
    });
    assertTrue(removeLatch.await(2000, TimeUnit.MILLISECONDS));
}