Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 26 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class NetworkLessBookieTest method testUseLocalBookie.

@Test
public void testUseLocalBookie() throws Exception {
    ClientConfiguration conf = new ClientConfiguration();
    conf.setZkServers(zkUtil.getZooKeeperConnectString());
    conf.setZkTimeout(20000);
    try (BookKeeper bkc = new BookKeeper(conf)) {
        try (LedgerHandle h = bkc.createLedger(1, 1, DigestType.CRC32, "testPasswd".getBytes())) {
            h.addEntry("test".getBytes());
        }
    }
    for (BookieServer bk : bs) {
        for (Channel channel : bk.nettyServer.allChannels) {
            if (!(channel instanceof LocalChannel)) {
                fail();
            }
        }
    }
}

Example 27 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class BenchReadThroughputLatency method readLedger.

private static void readLedger(ClientConfiguration conf, long ledgerId, byte[] passwd) {
    LOG.info("Reading ledger {}", ledgerId);
    BookKeeper bk = null;
    long time = 0;
    long entriesRead = 0;
    long lastRead = 0;
    int nochange = 0;
    long absoluteLimit = 5000000;
    LedgerHandle lh = null;
    try {
        bk = new BookKeeper(conf);
        while (true) {
            lh = bk.openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, passwd);
            long lastConfirmed = Math.min(lh.getLastAddConfirmed(), absoluteLimit);
            if (lastConfirmed == lastRead) {
                nochange++;
                if (nochange == 10) {
                    break;
                } else {
                    Thread.sleep(1000);
                    continue;
                }
            } else {
                nochange = 0;
            }
            long starttime = System.nanoTime();
            while (lastRead < lastConfirmed) {
                long nextLimit = lastRead + 100000;
                long readTo = Math.min(nextLimit, lastConfirmed);
                Enumeration<LedgerEntry> entries = lh.readEntries(lastRead + 1, readTo);
                lastRead = readTo;
                while (entries.hasMoreElements()) {
                    LedgerEntry e = entries.nextElement();
                    entriesRead++;
                    if ((entriesRead % 10000) == 0) {
                        LOG.info("{} entries read", entriesRead);
                    }
                }
            }
            long endtime = System.nanoTime();
            time += endtime - starttime;
            lh.close();
            lh = null;
            Thread.sleep(1000);
        }
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
    } catch (Exception e) {
        LOG.error("Exception in reader", e);
    } finally {
        LOG.info("Read {} in {}ms", entriesRead, time / 1000 / 1000);
        try {
            if (lh != null) {
                lh.close();
            }
            if (bk != null) {
                bk.close();
            }
        } catch (Exception e) {
            LOG.error("Exception closing stuff", e);
        }
    }
}

Example 28 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class TestBenchmark method testReadThroughputLatency.

@Test
public void testReadThroughputLatency() throws Exception {
    final AtomicBoolean threwException = new AtomicBoolean(false);
    Thread t = new Thread() {

        public void run() {
            try {
                BenchReadThroughputLatency.main(new String[] { "--zookeeper", zkUtil.getZooKeeperConnectString(), "--listen", "10" });
            } catch (Throwable t) {
                LOG.error("Error reading", t);
                threwException.set(true);
            }
        }
    };
    t.start();
    Thread.sleep(10000);
    byte[] data = new byte[1024];
    Arrays.fill(data, (byte) 'x');
    long lastLedgerId = 0;
    Assert.assertTrue("Thread should be running", t.isAlive());
    for (int i = 0; i < 10; i++) {
        BookKeeper bk = new BookKeeper(zkUtil.getZooKeeperConnectString());
        LedgerHandle lh = bk.createLedger(BookKeeper.DigestType.CRC32, "benchPasswd".getBytes());
        lastLedgerId = lh.getId();
        try {
            for (int j = 0; j < 100; j++) {
                lh.addEntry(data);
            }
        } finally {
            lh.close();
            bk.close();
        }
    }
    for (int i = 0; i < 60; i++) {
        if (!t.isAlive()) {
            break;
        }
        Thread.sleep(100);
    }
    Assert.assertFalse("Thread should be finished", t.isAlive());
    BenchReadThroughputLatency.main(new String[] { "--zookeeper", zkUtil.getZooKeeperConnectString(), "--ledger", String.valueOf(lastLedgerId) });
}

Example 29 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class BookieInitializationTest method testWithDiskFullAndAbilityToCreateNewIndexFile.

@Test
public void testWithDiskFullAndAbilityToCreateNewIndexFile() throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setDiskCheckInterval(1000).setLedgerStorageClass(SortedLedgerStorage.class.getName()).setAutoRecoveryDaemonEnabled(false);
    conf.setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000);
    BookieServer server = new MockBookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkClient = new BookKeeper(clientConf);
    LedgerHandle lh = bkClient.createLedger(1, 1, 1, DigestType.CRC32, "passwd".getBytes());
    long entryId = -1;
    long numOfEntries = 5;
    for (int i = 0; i < numOfEntries; i++) {
        entryId = lh.addEntry("data".getBytes());
    }
    assertTrue("EntryId of the recently added entry should be 0", entryId == (numOfEntries - 1));
    // We want to simulate the scenario where Bookie is killed abruptly, so
    // SortedLedgerStorage's EntryMemTable and IndexInMemoryPageManager are
    // not flushed and hence when bookie is restarted it will replay the
    // journal. Since there is no easy way to kill the Bookie abruptly, we
    // are injecting no-op shutdown.
    server.shutdown();
    long usableSpace = tmpDir.getUsableSpace();
    long totalSpace = tmpDir.getTotalSpace();
    conf.setDiskUsageThreshold(0.001f).setDiskUsageWarnThreshold(0.0f).setReadOnlyModeEnabled(true).setIsForceGCAllowWhenNoSpace(true).setMinUsableSizeForIndexFileCreation(Long.MAX_VALUE);
    server = new BookieServer(conf);
    // Now we are trying to start the Bookie, which tries to replay the
    // Journal. While replaying the Journal it tries to create the IndexFile
    // for the ledger (whose entries are not flushed). but since we set
    // minUsableSizeForIndexFileCreation to very high value, it wouldn't. be
    // able to find any index dir when all discs are full
    server.start();
    assertFalse("Bookie should be Shutdown", server.getBookie().isRunning());
    server.shutdown();
    // Here we are setting MinUsableSizeForIndexFileCreation to very low
    // value. So if index dirs are full then it will consider the dirs which
    // have atleast MinUsableSizeForIndexFileCreation usable space for the
    // creation of new Index file.
    conf.setMinUsableSizeForIndexFileCreation(5 * 1024);
    server = new BookieServer(conf);
    server.start();
    Thread.sleep((conf.getDiskCheckInterval() * 2) + 100);
    assertTrue("Bookie should be up and running", server.getBookie().isRunning());
    assertTrue(server.getBookie().isReadOnly());
    server.shutdown();
    bkClient.close();
}

Example 30 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class TestAuth method entryCount.

/**
 * check if the entry exists. Restart the bookie to allow
 * access
 */
private int entryCount(long ledgerId, ServerConfiguration bookieConf, ClientConfiguration clientConf) throws Exception {
    LOG.info("Counting entries in {}", ledgerId);
    for (ServerConfiguration conf : bsConfs) {
        conf.setBookieAuthProviderFactoryClass(AlwaysSucceedBookieAuthProviderFactory.class.getName());
    }
    clientConf.setClientAuthProviderFactoryClass(SendUntilCompleteClientAuthProviderFactory.class.getName());
    restartBookies();
    BookKeeper bkc = new BookKeeper(clientConf, zkc);
    LedgerHandle lh = bkc.openLedger(ledgerId, DigestType.CRC32, PASSWD);
    if (lh.getLastAddConfirmed() < 0) {
        return 0;
    }
    Enumeration<LedgerEntry> e = lh.readEntries(0, lh.getLastAddConfirmed());
    int count = 0;
    while (e.hasMoreElements()) {
        count++;
        assertTrue("Should match what we wrote", Arrays.equals(e.nextElement().getEntry(), ENTRY));
    }
    return count;
}