Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 51 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class MD5DigestBookKeeperTest method connectAndWriteToBookie.

// we pass in ledgerId because the method may throw exceptions
private void connectAndWriteToBookie(ClientConfiguration conf, AtomicLong ledgerWritten) throws Exception {
    LOG.info("Connecting to bookie");
    BookKeeper bkc = new BookKeeper(conf, zkc);
    LedgerHandle l = bkc.createLedger(1, 1, DigestType.CRC32, PASSWD);
    ledgerWritten.set(l.getId());
    l.addEntry(ENTRY);
    l.close();
    bkc.close();
}

Example 52 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class BookKeeperClient method deleteLedger.

public CompletableFuture<Void> deleteLedger(long lid, final boolean ignoreNonExistentLedger) {
    BookKeeper bk;
    try {
        bk = get();
    } catch (IOException ioe) {
        return FutureUtils.exception(ioe);
    }
    final CompletableFuture<Void> promise = new CompletableFuture<Void>();
    bk.asyncDeleteLedger(lid, new AsyncCallback.DeleteCallback() {

        @Override
        public void deleteComplete(int rc, Object ctx) {
            if (BKException.Code.OK == rc) {
                promise.complete(null);
            } else if (BKException.Code.NoSuchLedgerExistsException == rc) {
                if (ignoreNonExistentLedger) {
                    promise.complete(null);
                } else {
                    promise.completeExceptionally(BKException.create(rc));
                }
            } else {
                promise.completeExceptionally(BKException.create(rc));
            }
        }
    }, null);
    return promise;
}

Example 53 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class BookKeeperClient method close.

public void close() {
    BookKeeper bkcToClose;
    ZooKeeperClient zkcToClose;
    synchronized (this) {
        if (closed) {
            return;
        }
        closed = true;
        bkcToClose = bkc;
        zkcToClose = zkc;
    }
    LOG.info("BookKeeper Client closed {}", name);
    if (null != bkcToClose) {
        try {
            bkcToClose.close();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            LOG.warn("Interrupted on closing bookkeeper client {} : ", name, e);
            Thread.currentThread().interrupt();
        } catch (BKException e) {
            LOG.warn("Error on closing bookkeeper client {} : ", name, e);
        }
    }
    if (null != zkcToClose) {
        if (ownZK) {
            zkcToClose.close();
        }
    }
}

Example 54 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class BookKeeperClient method createLedger.

// Util functions
public CompletableFuture<LedgerHandle> createLedger(int ensembleSize, int writeQuorumSize, int ackQuorumSize) {
    BookKeeper bk;
    try {
        bk = get();
    } catch (IOException ioe) {
        return FutureUtils.exception(ioe);
    }
    final CompletableFuture<LedgerHandle> promise = new CompletableFuture<LedgerHandle>();
    bk.asyncCreateLedger(ensembleSize, writeQuorumSize, ackQuorumSize, BookKeeper.DigestType.CRC32, passwd, new AsyncCallback.CreateCallback() {

        @Override
        public void createComplete(int rc, LedgerHandle lh, Object ctx) {
            if (BKException.Code.OK == rc) {
                promise.complete(lh);
            } else {
                promise.completeExceptionally(BKException.create(rc));
            }
        }
    }, null, Collections.emptyMap());
    return promise;
}

Example 55 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class TestClient method main.

/**
 * First says if entries should be written to BookKeeper (0) or to the local
 * disk (1). Second parameter is an integer defining the length of a ledger entry.
 * Third parameter is the number of writes.
 *
 * @param args
 */
public static void main(String[] args) throws ParseException {
    Options options = new Options();
    options.addOption("length", true, "Length of packets being written. Default 1024");
    options.addOption("target", true, "Target medium to write to. Options are bk, fs & hdfs. Default fs");
    options.addOption("runfor", true, "Number of seconds to run for. Default 60");
    options.addOption("path", true, "Path to write to. fs & hdfs only. Default /foobar");
    options.addOption("zkservers", true, "ZooKeeper servers, comma separated. bk only. Default localhost:2181.");
    options.addOption("bkensemble", true, "BookKeeper ledger ensemble size. bk only. Default 3");
    options.addOption("bkquorum", true, "BookKeeper ledger quorum size. bk only. Default 2");
    options.addOption("bkthrottle", true, "BookKeeper throttle size. bk only. Default 10000");
    options.addOption("sync", false, "Use synchronous writes with BookKeeper. bk only.");
    options.addOption("numconcurrent", true, "Number of concurrently clients. Default 1");
    options.addOption("timeout", true, "Number of seconds after which to give up");
    options.addOption("help", false, "This message");
    CommandLineParser parser = new PosixParser();
    CommandLine cmd = parser.parse(options, args);
    if (cmd.hasOption("help")) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("TestClient <options>", options);
        System.exit(-1);
    }
    int length = Integer.parseInt(cmd.getOptionValue("length", "1024"));
    String target = cmd.getOptionValue("target", "fs");
    long runfor = Long.parseLong(cmd.getOptionValue("runfor", "60")) * 1000;
    StringBuilder sb = new StringBuilder();
    while (length-- > 0) {
        sb.append('a');
    }
    Timer timeouter = new Timer();
    if (cmd.hasOption("timeout")) {
        final long timeout = Long.parseLong(cmd.getOptionValue("timeout", "360")) * 1000;
        timeouter.schedule(new TimerTask() {

            public void run() {
                System.err.println("Timing out benchmark after " + timeout + "ms");
                System.exit(-1);
            }
        }, timeout);
    }
    BookKeeper bkc = null;
    try {
        int numFiles = Integer.parseInt(cmd.getOptionValue("numconcurrent", "1"));
        int numThreads = Math.min(numFiles, 1000);
        byte[] data = sb.toString().getBytes(UTF_8);
        long runid = System.currentTimeMillis();
        List<Callable<Long>> clients = new ArrayList<Callable<Long>>();
        if (target.equals("bk")) {
            String zkservers = cmd.getOptionValue("zkservers", "localhost:2181");
            int bkensemble = Integer.parseInt(cmd.getOptionValue("bkensemble", "3"));
            int bkquorum = Integer.parseInt(cmd.getOptionValue("bkquorum", "2"));
            int bkthrottle = Integer.parseInt(cmd.getOptionValue("bkthrottle", "10000"));
            ClientConfiguration conf = new ClientConfiguration();
            conf.setThrottleValue(bkthrottle);
            conf.setZkServers(zkservers);
            bkc = new BookKeeper(conf);
            List<LedgerHandle> handles = new ArrayList<LedgerHandle>();
            for (int i = 0; i < numFiles; i++) {
                handles.add(bkc.createLedger(bkensemble, bkquorum, DigestType.CRC32, new byte[] { 'a', 'b' }));
            }
            for (int i = 0; i < numFiles; i++) {
                clients.add(new BKClient(handles, data, runfor, cmd.hasOption("sync")));
            }
        } else if (target.equals("fs")) {
            List<FileOutputStream> streams = new ArrayList<FileOutputStream>();
            for (int i = 0; i < numFiles; i++) {
                String path = cmd.getOptionValue("path", "/foobar " + i);
                streams.add(new FileOutputStream(path + runid + "_" + i));
            }
            for (int i = 0; i < numThreads; i++) {
                clients.add(new FileClient(streams, data, runfor));
            }
        } else {
            LOG.error("Unknown option: " + target);
            throw new IllegalArgumentException("Unknown target " + target);
        }
        ExecutorService executor = Executors.newFixedThreadPool(numThreads);
        long start = System.currentTimeMillis();
        List<Future<Long>> results = executor.invokeAll(clients, 10, TimeUnit.MINUTES);
        long end = System.currentTimeMillis();
        long count = 0;
        for (Future<Long> r : results) {
            if (!r.isDone()) {
                LOG.warn("Job didn't complete");
                System.exit(2);
            }
            long c = r.get();
            if (c == 0) {
                LOG.warn("Task didn't complete");
            }
            count += c;
        }
        long time = end - start;
        LOG.info("Finished processing writes (ms): {} TPT: {} op/s", time, count / ((double) time / 1000));
        executor.shutdown();
    } catch (ExecutionException ee) {
        LOG.error("Exception in worker", ee);
    } catch (BKException e) {
        LOG.error("Error accessing bookkeeper", e);
    } catch (IOException ioe) {
        LOG.error("I/O exception during benchmark", ioe);
    } catch (InterruptedException ie) {
        LOG.error("Benchmark interrupted", ie);
        Thread.currentThread().interrupt();
    } finally {
        if (bkc != null) {
            try {
                bkc.close();
            } catch (BKException bke) {
                LOG.error("Error closing bookkeeper client", bke);
            } catch (InterruptedException ie) {
                LOG.warn("Interrupted closing bookkeeper client", ie);
                Thread.currentThread().interrupt();
            }
        }
    }
    timeouter.cancel();
}