Examples with LedgerHandle org.apache.bookkeeper.client.LedgerHandle used on opensource projects

Example 81 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class BookKeeperBuildersTest method testCreateLedgerWriteFlagsVarargs.

@Test
public void testCreateLedgerWriteFlagsVarargs() throws Exception {
    setNewGeneratedLedgerId(ledgerId);
    WriteHandle writer = newCreateLedgerOp().withAckQuorumSize(ackQuorumSize).withEnsembleSize(ensembleSize).withPassword(password).withWriteQuorumSize(writeQuorumSize).withCustomMetadata(customMetadata).withWriteFlags(DEFERRED_SYNC).execute().get();
    assertEquals(ledgerId, writer.getId());
    LedgerMetadata metadata = getLedgerMetadata(ledgerId);
    assertEquals(ensembleSize, metadata.getEnsembleSize());
    assertEquals(ackQuorumSize, metadata.getAckQuorumSize());
    assertEquals(writeQuorumSize, metadata.getWriteQuorumSize());
    assertArrayEquals(password, metadata.getPassword());
    LedgerHandle lh = (LedgerHandle) writer;
    assertEquals(writeFlagsDeferredSync, lh.getWriteFlags());
}

Example 82 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class BenchThroughputLatency method run.

public void run() {
    LOG.info("Running...");
    long start = previous = System.currentTimeMillis();
    int sent = 0;
    Thread reporter = new Thread() {

        public void run() {
            try {
                while (true) {
                    Thread.sleep(1000);
                    LOG.info("ms: {} req: {}", System.currentTimeMillis(), completedRequests.getAndSet(0));
                }
            } catch (InterruptedException ie) {
                LOG.info("Caught interrupted exception, going away");
                Thread.currentThread().interrupt();
            }
        }
    };
    reporter.start();
    long beforeSend = System.nanoTime();
    while (!Thread.currentThread().isInterrupted() && sent < sendLimit) {
        try {
            sem.acquire();
            if (sent == 10000) {
                long afterSend = System.nanoTime();
                long time = afterSend - beforeSend;
                LOG.info("Time to send first batch: {}s {}ns ", time / 1000 / 1000 / 1000, time);
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
        }
        final int index = getRandomLedger();
        LedgerHandle h = lh[index];
        if (h == null) {
            LOG.error("Handle " + index + " is null!");
        } else {
            long nanoTime = System.nanoTime();
            lh[index].asyncAddEntry(bytes, this, new Context(sent, nanoTime));
            counter.incrementAndGet();
        }
        sent++;
    }
    LOG.info("Sent: " + sent);
    try {
        int i = 0;
        while (this.counter.get() > 0) {
            Thread.sleep(1000);
            i++;
            if (i > 30) {
                break;
            }
        }
    } catch (InterruptedException e) {
        LOG.error("Interrupted while waiting", e);
        Thread.currentThread().interrupt();
    }
    synchronized (this) {
        duration = System.currentTimeMillis() - start;
    }
    throughput = sent * 1000 / getDuration();
    reporter.interrupt();
    try {
        reporter.join();
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
    }
    LOG.info("Finished processing in ms: " + getDuration() + " tp = " + throughput);
}

Example 83 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class BenchReadThroughputLatency method readLedger.

private static void readLedger(ClientConfiguration conf, long ledgerId, byte[] passwd) {
    LOG.info("Reading ledger {}", ledgerId);
    BookKeeper bk = null;
    long time = 0;
    long entriesRead = 0;
    long lastRead = 0;
    int nochange = 0;
    long absoluteLimit = 5000000;
    LedgerHandle lh = null;
    try {
        bk = new BookKeeper(conf);
        while (true) {
            lh = bk.openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, passwd);
            long lastConfirmed = Math.min(lh.getLastAddConfirmed(), absoluteLimit);
            if (lastConfirmed == lastRead) {
                nochange++;
                if (nochange == 10) {
                    break;
                } else {
                    Thread.sleep(1000);
                    continue;
                }
            } else {
                nochange = 0;
            }
            long starttime = System.nanoTime();
            while (lastRead < lastConfirmed) {
                long nextLimit = lastRead + 100000;
                long readTo = Math.min(nextLimit, lastConfirmed);
                Enumeration<LedgerEntry> entries = lh.readEntries(lastRead + 1, readTo);
                lastRead = readTo;
                while (entries.hasMoreElements()) {
                    LedgerEntry e = entries.nextElement();
                    entriesRead++;
                    if ((entriesRead % 10000) == 0) {
                        LOG.info("{} entries read", entriesRead);
                    }
                }
            }
            long endtime = System.nanoTime();
            time += endtime - starttime;
            lh.close();
            lh = null;
            Thread.sleep(1000);
        }
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
    } catch (Exception e) {
        LOG.error("Exception in reader", e);
    } finally {
        LOG.info("Read {} in {}ms", entriesRead, time / 1000 / 1000);
        try {
            if (lh != null) {
                lh.close();
            }
            if (bk != null) {
                bk.close();
            }
        } catch (Exception e) {
            LOG.error("Exception closing stuff", e);
        }
    }
}

Example 84 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class TestBenchmark method testReadThroughputLatency.

@Test
public void testReadThroughputLatency() throws Exception {
    final AtomicBoolean threwException = new AtomicBoolean(false);
    Thread t = new Thread() {

        public void run() {
            try {
                BenchReadThroughputLatency.main(new String[] { "--zookeeper", zkUtil.getZooKeeperConnectString(), "--listen", "10" });
            } catch (Throwable t) {
                LOG.error("Error reading", t);
                threwException.set(true);
            }
        }
    };
    t.start();
    Thread.sleep(10000);
    byte[] data = new byte[1024];
    Arrays.fill(data, (byte) 'x');
    long lastLedgerId = 0;
    Assert.assertTrue("Thread should be running", t.isAlive());
    for (int i = 0; i < 10; i++) {
        BookKeeper bk = new BookKeeper(zkUtil.getZooKeeperConnectString());
        LedgerHandle lh = bk.createLedger(BookKeeper.DigestType.CRC32, "benchPasswd".getBytes());
        lastLedgerId = lh.getId();
        try {
            for (int j = 0; j < 100; j++) {
                lh.addEntry(data);
            }
        } finally {
            lh.close();
            bk.close();
        }
    }
    for (int i = 0; i < 60; i++) {
        if (!t.isAlive()) {
            break;
        }
        Thread.sleep(100);
    }
    Assert.assertFalse("Thread should be finished", t.isAlive());
    BenchReadThroughputLatency.main(new String[] { "--zookeeper", zkUtil.getZooKeeperConnectString(), "--ledger", String.valueOf(lastLedgerId) });
}

Example 85 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class BookieInitializationTest method testWithDiskFullAndAbilityToCreateNewIndexFile.

@Test
public void testWithDiskFullAndAbilityToCreateNewIndexFile() throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setDiskCheckInterval(1000).setLedgerStorageClass(SortedLedgerStorage.class.getName()).setAutoRecoveryDaemonEnabled(false);
    conf.setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000);
    BookieServer server = new MockBookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkClient = new BookKeeper(clientConf);
    LedgerHandle lh = bkClient.createLedger(1, 1, 1, DigestType.CRC32, "passwd".getBytes());
    long entryId = -1;
    long numOfEntries = 5;
    for (int i = 0; i < numOfEntries; i++) {
        entryId = lh.addEntry("data".getBytes());
    }
    assertTrue("EntryId of the recently added entry should be 0", entryId == (numOfEntries - 1));
    // We want to simulate the scenario where Bookie is killed abruptly, so
    // SortedLedgerStorage's EntryMemTable and IndexInMemoryPageManager are
    // not flushed and hence when bookie is restarted it will replay the
    // journal. Since there is no easy way to kill the Bookie abruptly, we
    // are injecting no-op shutdown.
    server.shutdown();
    long usableSpace = tmpDir.getUsableSpace();
    long totalSpace = tmpDir.getTotalSpace();
    conf.setDiskUsageThreshold(0.001f).setDiskUsageWarnThreshold(0.0f).setReadOnlyModeEnabled(true).setIsForceGCAllowWhenNoSpace(true).setMinUsableSizeForIndexFileCreation(Long.MAX_VALUE);
    server = new BookieServer(conf);
    // Now we are trying to start the Bookie, which tries to replay the
    // Journal. While replaying the Journal it tries to create the IndexFile
    // for the ledger (whose entries are not flushed). but since we set
    // minUsableSizeForIndexFileCreation to very high value, it wouldn't. be
    // able to find any index dir when all discs are full
    server.start();
    assertFalse("Bookie should be Shutdown", server.getBookie().isRunning());
    server.shutdown();
    // Here we are setting MinUsableSizeForIndexFileCreation to very low
    // value. So if index dirs are full then it will consider the dirs which
    // have atleast MinUsableSizeForIndexFileCreation usable space for the
    // creation of new Index file.
    conf.setMinUsableSizeForIndexFileCreation(5 * 1024);
    server = new BookieServer(conf);
    server.start();
    Thread.sleep((conf.getDiskCheckInterval() * 2) + 100);
    assertTrue("Bookie should be up and running", server.getBookie().isRunning());
    assertTrue(server.getBookie().isReadOnly());
    server.shutdown();
    bkClient.close();
}