Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 21 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class MD5DigestBookKeeperTest method entryCount.

/**
 * check if the entry exists. Restart the bookie to allow access
 */
private int entryCount(long ledgerId, ServerConfiguration bookieConf, ClientConfiguration clientConf) throws Exception {
    LOG.info("Counting entries in {}", ledgerId);
    for (ServerConfiguration conf : bsConfs) {
        bookieConf.setBookieAuthProviderFactoryClass(SASLBookieAuthProviderFactory.class.getName());
        bookieConf.setProperty(JAAS_CLIENT_ALLOWED_IDS, ".*hd.*");
    }
    clientConf.setClientAuthProviderFactoryClass(SASLClientProviderFactory.class.getName());
    restartBookies();
    try (BookKeeper bkc = new BookKeeper(clientConf, zkc);
        LedgerHandle lh = bkc.openLedger(ledgerId, DigestType.CRC32, PASSWD)) {
        if (lh.getLastAddConfirmed() < 0) {
            return 0;
        }
        Enumeration<LedgerEntry> e = lh.readEntries(0, lh.getLastAddConfirmed());
        int count = 0;
        while (e.hasMoreElements()) {
            count++;
            assertTrue("Should match what we wrote", Arrays.equals(e.nextElement().getEntry(), ENTRY));
        }
        return count;
    }
}

Example 22 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class Auditor method checkAllLedgers.

/**
 * List all the ledgers and check them individually. This should not
 * be run very often.
 */
void checkAllLedgers() throws BKAuditException, BKException, IOException, InterruptedException, KeeperException {
    ZooKeeper newzk = ZooKeeperClient.newBuilder().connectString(conf.getZkServers()).sessionTimeoutMs(conf.getZkTimeout()).build();
    final BookKeeper client = new BookKeeper(new ClientConfiguration(conf), newzk);
    final BookKeeperAdmin admin = new BookKeeperAdmin(client, statsLogger);
    try {
        final LedgerChecker checker = new LedgerChecker(client);
        final AtomicInteger returnCode = new AtomicInteger(BKException.Code.OK);
        final CountDownLatch processDone = new CountDownLatch(1);
        Processor<Long> checkLedgersProcessor = new Processor<Long>() {

            @Override
            public void process(final Long ledgerId, final AsyncCallback.VoidCallback callback) {
                try {
                    if (!ledgerUnderreplicationManager.isLedgerReplicationEnabled()) {
                        LOG.info("Ledger rereplication has been disabled, aborting periodic check");
                        processDone.countDown();
                        return;
                    }
                } catch (ReplicationException.UnavailableException ue) {
                    LOG.error("Underreplication manager unavailable running periodic check", ue);
                    processDone.countDown();
                    return;
                }
                LedgerHandle lh = null;
                try {
                    lh = admin.openLedgerNoRecovery(ledgerId);
                    checker.checkLedger(lh, new ProcessLostFragmentsCb(lh, callback), conf.getAuditorLedgerVerificationPercentage());
                    // we collect the following stats to get a measure of the
                    // distribution of a single ledger within the bk cluster
                    // the higher the number of fragments/bookies, the more distributed it is
                    numFragmentsPerLedger.registerSuccessfulValue(lh.getNumFragments());
                    numBookiesPerLedger.registerSuccessfulValue(lh.getNumBookies());
                    numLedgersChecked.inc();
                } catch (BKException.BKNoSuchLedgerExistsException bknsle) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ledger was deleted before we could check it", bknsle);
                    }
                    callback.processResult(BKException.Code.OK, null, null);
                    return;
                } catch (BKException bke) {
                    LOG.error("Couldn't open ledger " + ledgerId, bke);
                    callback.processResult(BKException.Code.BookieHandleNotAvailableException, null, null);
                    return;
                } catch (InterruptedException ie) {
                    LOG.error("Interrupted opening ledger", ie);
                    Thread.currentThread().interrupt();
                    callback.processResult(BKException.Code.InterruptedException, null, null);
                    return;
                } finally {
                    if (lh != null) {
                        try {
                            lh.close();
                        } catch (BKException bke) {
                            LOG.warn("Couldn't close ledger " + ledgerId, bke);
                        } catch (InterruptedException ie) {
                            LOG.warn("Interrupted closing ledger " + ledgerId, ie);
                            Thread.currentThread().interrupt();
                        }
                    }
                }
            }
        };
        ledgerManager.asyncProcessLedgers(checkLedgersProcessor, new AsyncCallback.VoidCallback() {

            @Override
            public void processResult(int rc, String s, Object obj) {
                returnCode.set(rc);
                processDone.countDown();
            }
        }, null, BKException.Code.OK, BKException.Code.ReadException);
        try {
            processDone.await();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new BKAuditException("Exception while checking ledgers", e);
        }
        if (returnCode.get() != BKException.Code.OK) {
            throw BKException.create(returnCode.get());
        }
    } finally {
        admin.close();
        client.close();
        newzk.close();
    }
}

Example 23 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class Auditor method initialize.

private void initialize(ServerConfiguration conf, ZooKeeper zkc) throws UnavailableException {
    try {
        ClientConfiguration clientConfiguration = new ClientConfiguration(conf);
        clientConfiguration.setClientRole(ClientConfiguration.CLIENT_ROLE_SYSTEM);
        LOG.info("AuthProvider used by the Auditor is {}", clientConfiguration.getClientAuthProviderFactoryClass());
        this.bkc = new BookKeeper(clientConfiguration, zkc);
        LedgerManagerFactory ledgerManagerFactory = AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, bkc.getMetadataClientDriver().getLayoutManager());
        ledgerManager = ledgerManagerFactory.newLedgerManager();
        this.bookieLedgerIndexer = new BookieLedgerIndexer(ledgerManager);
        this.ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager();
        this.admin = new BookKeeperAdmin(bkc, statsLogger);
        if (this.ledgerUnderreplicationManager.initializeLostBookieRecoveryDelay(conf.getLostBookieRecoveryDelay())) {
            LOG.info("Initializing lostBookieRecoveryDelay zNode to the conif value: {}", conf.getLostBookieRecoveryDelay());
        } else {
            LOG.info("Valid lostBookieRecoveryDelay zNode is available, so not creating " + "lostBookieRecoveryDelay zNode as part of Auditor initialization ");
        }
        lostBookieRecoveryDelayBeforeChange = this.ledgerUnderreplicationManager.getLostBookieRecoveryDelay();
    } catch (CompatibilityException ce) {
        throw new UnavailableException("CompatibilityException while initializing Auditor", ce);
    } catch (IOException | BKException | KeeperException ioe) {
        throw new UnavailableException("Exception while initializing Auditor", ioe);
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        throw new UnavailableException("Interrupted while initializing Auditor", ie);
    }
}

Example 24 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class DeleteLedgerService method handle.

@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    // only handle DELETE method
    if (HttpServer.Method.DELETE == request.getMethod()) {
        Map<String, String> params = request.getParams();
        if (params != null && params.containsKey("ledger_id")) {
            ClientConfiguration clientConf = new ClientConfiguration();
            clientConf.addConfiguration(conf);
            BookKeeper bk = new BookKeeper(clientConf);
            Long ledgerId = Long.parseLong(params.get("ledger_id"));
            bk.deleteLedger(ledgerId);
            String output = "Deleted ledger: " + ledgerId;
            String jsonResponse = JsonUtil.toJson(output);
            LOG.debug("output body:" + jsonResponse);
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } else {
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Not ledger found. Should provide ledger_id=<id>");
            return response;
        }
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be DELETE method");
        return response;
    }
}

Example 25 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class ListBookieInfoService method handle.

@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    if (HttpServer.Method.GET == request.getMethod()) {
        ClientConfiguration clientConf = new ClientConfiguration(conf);
        clientConf.setZkServers(conf.getZkServers());
        clientConf.setDiskWeightBasedPlacementEnabled(true);
        BookKeeper bk = new BookKeeper(clientConf);
        Map<BookieSocketAddress, BookieInfoReader.BookieInfo> map = bk.getBookieInfo();
        if (map.size() == 0) {
            bk.close();
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Not found any Bookie info.");
            return response;
        }
        /**
         * output:
         *  {
         *    "bookieAddress" : {free: xxx, total: xxx}",
         *    "bookieAddress" : {free: xxx, total: xxx},
         *    ...
         *    "clusterInfo" : {total_free: xxx, total: xxx}"
         *  }
         */
        LinkedHashMap<String, String> output = Maps.newLinkedHashMapWithExpectedSize(map.size());
        Long totalFree = 0L, total = 0L;
        for (Map.Entry<BookieSocketAddress, BookieInfoReader.BookieInfo> infoEntry : map.entrySet()) {
            BookieInfoReader.BookieInfo bInfo = infoEntry.getValue();
            output.put(infoEntry.getKey().toString(), ": {Free: " + bInfo.getFreeDiskSpace() + getReadable(bInfo.getFreeDiskSpace()) + ", Total: " + bInfo.getTotalDiskSpace() + getReadable(bInfo.getTotalDiskSpace()) + "},");
            totalFree += bInfo.getFreeDiskSpace();
            total += bInfo.getTotalDiskSpace();
        }
        output.put("ClusterInfo: ", "{Free: " + totalFree + getReadable(totalFree) + ", Total: " + total + getReadable(total) + "}");
        bk.close();
        String jsonResponse = JsonUtil.toJson(output);
        LOG.debug("output body:" + jsonResponse);
        response.setBody(jsonResponse);
        response.setCode(HttpServer.StatusCode.OK);
        return response;
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be GET method");
        return response;
    }
}