Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 36 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class LedgerStorageCheckpointTest method testCheckpointofILSWhenEntryLogIsRotated.

public void testCheckpointofILSWhenEntryLogIsRotated(boolean entryLogPerLedgerEnabled) throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setAutoRecoveryDaemonEnabled(false).setFlushInterval(30000).setBookiePort(PortManager.nextFreePort()).setEntryLogPerLedgerEnabled(entryLogPerLedgerEnabled).setLedgerStorageClass(InterleavedLedgerStorage.class.getName());
    Assert.assertEquals("Number of JournalDirs", 1, conf.getJournalDirs().length);
    // we know there is only one ledgerDir
    File ledgerDir = Bookie.getCurrentDirectories(conf.getLedgerDirs())[0];
    BookieServer server = new BookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkClient = new BookKeeper(clientConf);
    InterleavedLedgerStorage ledgerStorage = (InterleavedLedgerStorage) server.getBookie().ledgerStorage;
    int numOfEntries = 5;
    byte[] dataBytes = "data".getBytes();
    long ledgerId = 10;
    LedgerHandle handle = bkClient.createLedgerAdv(ledgerId, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
    for (int j = 0; j < numOfEntries; j++) {
        handle.addEntry(j, dataBytes);
    }
    handle.close();
    // simulate rolling entrylog
    ledgerStorage.entryLogger.rollLog();
    // sleep for a bit for checkpoint to do its task
    executorController.advance(Duration.ofMillis(500));
    File lastMarkFile = new File(ledgerDir, "lastMark");
    LogMark rolledLogMark = readLastMarkFile(lastMarkFile);
    if (entryLogPerLedgerEnabled) {
        Assert.assertEquals("rolledLogMark should be zero, since checkpoint" + "shouldn't have happened when entryLog is rotated", 0, rolledLogMark.compare(new LogMark()));
    } else {
        Assert.assertNotEquals("rolledLogMark shouldn't be zero, since checkpoint" + "should have happened when entryLog is rotated", 0, rolledLogMark.compare(new LogMark()));
    }
    bkClient.close();
    server.shutdown();
}

Example 37 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class LedgerStorageCheckpointTest method testCheckPointForEntryLoggerWithMultipleActiveEntryLogs.

/*
     * This is complete end-to-end scenario.
     *
     * 1) This testcase uses MockInterleavedLedgerStorage, which extends
     * InterleavedLedgerStorage but doesn't do anything when Bookie is shutdown.
     * This is needed to simulate Bookie crash.
     * 2) entryLogPerLedger is enabled
     * 3) ledgers are created and entries are added.
     * 4) wait for flushInterval period for checkpoint to complete
     * 5) simulate bookie crash
     * 6) delete the journal files and lastmark file
     * 7) Now restart the Bookie
     * 8) validate that the entries which were written can be read successfully.
     */
@Test
public void testCheckPointForEntryLoggerWithMultipleActiveEntryLogs() throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setAutoRecoveryDaemonEnabled(false).setFlushInterval(3000).setBookiePort(PortManager.nextFreePort()).setEntryLogPerLedgerEnabled(true).setLedgerStorageClass(MockInterleavedLedgerStorage.class.getName());
    Assert.assertEquals("Number of JournalDirs", 1, conf.getJournalDirs().length);
    // we know there is only one ledgerDir
    File ledgerDir = Bookie.getCurrentDirectories(conf.getLedgerDirs())[0];
    BookieServer server = new BookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    final BookKeeper bkClient = new BookKeeper(clientConf);
    int numOfLedgers = 12;
    int numOfEntries = 100;
    byte[] dataBytes = "data".getBytes();
    AtomicBoolean receivedExceptionForAdd = new AtomicBoolean(false);
    LongStream.range(0, numOfLedgers).parallel().mapToObj((ledgerId) -> {
        LedgerHandle handle = null;
        try {
            handle = bkClient.createLedgerAdv(ledgerId, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
        } catch (BKException | InterruptedException exc) {
            receivedExceptionForAdd.compareAndSet(false, true);
            LOG.error("Got Exception while trying to create LedgerHandle for ledgerId: " + ledgerId, exc);
        }
        return handle;
    }).forEach((writeHandle) -> {
        IntStream.range(0, numOfEntries).forEach((entryId) -> {
            try {
                writeHandle.addEntry(entryId, dataBytes);
            } catch (BKException | InterruptedException exc) {
                receivedExceptionForAdd.compareAndSet(false, true);
                LOG.error("Got Exception while trying to AddEntry of ledgerId: " + writeHandle.getId() + " entryId: " + entryId, exc);
            }
        });
        try {
            writeHandle.close();
        } catch (BKException | InterruptedException e) {
            receivedExceptionForAdd.compareAndSet(false, true);
            LOG.error("Got Exception while trying to close writeHandle of ledgerId: " + writeHandle.getId(), e);
        }
    });
    Assert.assertFalse("There shouldn't be any exceptions while creating writeHandle and adding entries to writeHandle", receivedExceptionForAdd.get());
    executorController.advance(Duration.ofMillis(conf.getFlushInterval()));
    // since we have waited for more than flushInterval SyncThread should have checkpointed.
    // if entrylogperledger is not enabled, then we checkpoint only when currentLog in EntryLogger
    // is rotated. but if entrylogperledger is enabled, then we checkpoint for every flushInterval period
    File lastMarkFile = new File(ledgerDir, "lastMark");
    Assert.assertTrue("lastMark file must be existing, because checkpoint should have happened", lastMarkFile.exists());
    LogMark rolledLogMark = readLastMarkFile(lastMarkFile);
    Assert.assertNotEquals("rolledLogMark should not be zero, since checkpoint has happenend", 0, rolledLogMark.compare(new LogMark()));
    bkClient.close();
    // here we are calling shutdown, but MockInterleavedLedgerStorage shudown/flush
    // methods are noop, so entrylogger is not flushed as part of this shutdown
    // here we are trying to simulate Bookie crash, but there is no way to
    // simulate bookie abrupt crash
    server.shutdown();
    // delete journal files and lastMark, to make sure that we are not reading from
    // Journal file
    File[] journalDirs = conf.getJournalDirs();
    for (File journalDir : journalDirs) {
        File journalDirectory = Bookie.getCurrentDirectory(journalDir);
        List<Long> journalLogsId = Journal.listJournalIds(journalDirectory, null);
        for (long journalId : journalLogsId) {
            File journalFile = new File(journalDirectory, Long.toHexString(journalId) + ".txn");
            journalFile.delete();
        }
    }
    // we know there is only one ledgerDir
    lastMarkFile = new File(ledgerDir, "lastMark");
    lastMarkFile.delete();
    // now we are restarting BookieServer
    conf.setLedgerStorageClass(InterleavedLedgerStorage.class.getName());
    server = new BookieServer(conf);
    server.start();
    BookKeeper newBKClient = new BookKeeper(clientConf);
    // since Bookie checkpointed successfully before shutdown/crash,
    // we should be able to read from entryLogs though journal is deleted
    AtomicBoolean receivedExceptionForRead = new AtomicBoolean(false);
    LongStream.range(0, numOfLedgers).parallel().forEach((ledgerId) -> {
        try {
            LedgerHandle lh = newBKClient.openLedger(ledgerId, DigestType.CRC32, "passwd".getBytes());
            Enumeration<LedgerEntry> entries = lh.readEntries(0, numOfEntries - 1);
            while (entries.hasMoreElements()) {
                LedgerEntry entry = entries.nextElement();
                byte[] readData = entry.getEntry();
                Assert.assertEquals("Ledger Entry Data should match", new String("data".getBytes()), new String(readData));
            }
            lh.close();
        } catch (BKException | InterruptedException e) {
            receivedExceptionForRead.compareAndSet(false, true);
            LOG.error("Got Exception while trying to read entries of ledger, ledgerId: " + ledgerId, e);
        }
    });
    Assert.assertFalse("There shouldn't be any exceptions while creating readHandle and while reading" + "entries using readHandle", receivedExceptionForRead.get());
    newBKClient.close();
    server.shutdown();
}

Example 38 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class DbLedgerStorageBookieTest method testV2ReadWrite.

@Test
public void testV2ReadWrite() throws Exception {
    ClientConfiguration conf = new ClientConfiguration();
    conf.setUseV2WireProtocol(true);
    conf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkc = new BookKeeper(conf);
    LedgerHandle lh1 = bkc.createLedger(1, 1, DigestType.CRC32, new byte[0]);
    lh1.addEntry("Foobar".getBytes());
    lh1.close();
    LedgerHandle lh2 = bkc.openLedger(lh1.getId(), DigestType.CRC32, new byte[0]);
    assertEquals(0, lh2.getLastAddConfirmed());
    assertEquals(new String(lh2.readEntries(0, 0).nextElement().getEntry()), "Foobar");
}

Example 39 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class BookkeeperVerifierMain method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    options.addOption("ledger_path", true, "Hostname or IP of bookie to benchmark");
    options.addOption("zookeeper", true, "Zookeeper ensemble, (default \"localhost:2181\")");
    options.addOption("ensemble_size", true, "Bookkeeper client ensemble size");
    options.addOption("write_quorum", true, "Bookkeeper client write quorum size");
    options.addOption("ack_quorum", true, "Bookkeeper client ack quorum size");
    options.addOption("duration", true, "Run duration in seconds");
    options.addOption("drainTimeout", true, "Seconds to wait for in progress ops to end");
    options.addOption("target_concurrent_ledgers", true, "target number ledgers to write to concurrently");
    options.addOption("target_concurrent_writes", true, "target number of concurrent writes per ledger");
    options.addOption("target_write_group", true, "target number of entries to write at a time");
    options.addOption("target_read_group", true, "target number of entries to read at a time");
    options.addOption("target_ledgers", true, "Target number of ledgers");
    options.addOption("target_ledger_size", true, "Target size per ledger");
    options.addOption("target_entry_size", true, "Target size per entry");
    options.addOption("target_concurrent_reads", true, "Number of reads to maintain");
    options.addOption("cold_to_hot_ratio", true, "Ratio of reads on open ledgers");
    options.addOption("help", false, "Print this help message");
    CommandLineParser parser = new PosixParser();
    CommandLine cmd = null;
    try {
        cmd = parser.parse(options, args);
    } catch (ParseException e) {
        printHelpAndExit(options, "Unable to parse command line", 1);
    }
    if (cmd.hasOption("help")) {
        printHelpAndExit(options, "Help:", 0);
    }
    String ledgerPath = cmd.getOptionValue("ledger_path", "/ledgers");
    String zkString = cmd.getOptionValue("zookeeper", "localhost:2181");
    int ensembleSize = 0;
    int writeQuorum = 0;
    int ackQuorum = 0;
    int duration = 0;
    int drainTimeout = 0;
    int targetConcurrentLedgers = 0;
    int targetConcurrentWrites = 0;
    int targetWriteGroup = 0;
    int targetReadGroup = 0;
    int targetLedgers = 0;
    long targetLedgerSize = 0;
    int targetEntrySize = 0;
    int targetConcurrentReads = 0;
    double coldToHotRatio = 0;
    try {
        ensembleSize = Integer.parseInt(cmd.getOptionValue("ensemble_size", "3"));
        writeQuorum = Integer.parseInt(cmd.getOptionValue("write_quorum", "3"));
        ackQuorum = Integer.parseInt(cmd.getOptionValue("ack_quorum", "2"));
        duration = Integer.parseInt(cmd.getOptionValue("duration", "600"));
        drainTimeout = Integer.parseInt(cmd.getOptionValue("drain_timeout", "10"));
        targetConcurrentLedgers = Integer.parseInt(cmd.getOptionValue("target_concurrent_ledgers", "4"));
        targetConcurrentWrites = Integer.parseInt(cmd.getOptionValue("target_concurrent_writes", "12"));
        targetWriteGroup = Integer.parseInt(cmd.getOptionValue("target_write_group", "4"));
        targetReadGroup = Integer.parseInt(cmd.getOptionValue("target_read_group", "4"));
        targetLedgers = Integer.parseInt(cmd.getOptionValue("target_ledgers", "32"));
        targetLedgerSize = Long.parseLong(cmd.getOptionValue("target_ledger_size", "33554432"));
        targetEntrySize = Integer.parseInt(cmd.getOptionValue("target_entry_size", "16384"));
        targetConcurrentReads = Integer.parseInt(cmd.getOptionValue("target_concurrent_reads", "16"));
        coldToHotRatio = Double.parseDouble(cmd.getOptionValue("cold_to_hot_ratio", "0.5"));
    } catch (NumberFormatException e) {
        printHelpAndExit(options, "Invalid argument", 0);
    }
    ClientConfiguration conf = new ClientConfiguration();
    conf.setZkServers(zkString);
    conf.setZkLedgersRootPath(ledgerPath);
    BookKeeper bkclient = new BookKeeper(conf);
    BookkeeperVerifier verifier = new BookkeeperVerifier(new DirectBookkeeperDriver(bkclient), ensembleSize, writeQuorum, ackQuorum, duration, drainTimeout, targetConcurrentLedgers, targetConcurrentWrites, targetWriteGroup, targetReadGroup, targetLedgers, targetLedgerSize, targetEntrySize, targetConcurrentReads, coldToHotRatio);
    try {
        verifier.run();
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    } finally {
        bkclient.close();
    }
}

Example 40 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class CompactedTopicTest method buildCompactedLedger.

/**
 * Build a compacted ledger, and return the id of the ledger, the position of the different
 * entries in the ledger, and a list of gaps, and the entry which should be returned after the gap.
 */
private Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> buildCompactedLedger(BookKeeper bk, int count) throws Exception {
    LedgerHandle lh = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    List<Pair<MessageIdData, Long>> positions = new ArrayList<>();
    List<Pair<MessageIdData, Long>> idsInGaps = new ArrayList<>();
    AtomicLong ledgerIds = new AtomicLong(10L);
    AtomicLong entryIds = new AtomicLong(0L);
    CompletableFuture.allOf(IntStream.range(0, count).mapToObj((i) -> {
        List<MessageIdData> idsInGap = new ArrayList<MessageIdData>();
        if (r.nextInt(10) == 1) {
            long delta = r.nextInt(10) + 1;
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
            ledgerIds.addAndGet(delta);
            entryIds.set(0);
        }
        long delta = r.nextInt(5);
        if (delta != 0) {
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
        }
        MessageIdData id = MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.addAndGet(delta + 1)).build();
        @Cleanup RawMessage m = new RawMessageImpl(id, Unpooled.EMPTY_BUFFER);
        CompletableFuture<Void> f = new CompletableFuture<>();
        ByteBuf buffer = m.serialize();
        lh.asyncAddEntry(buffer, (rc, ledger, eid, ctx) -> {
            if (rc != BKException.Code.OK) {
                f.completeExceptionally(BKException.create(rc));
            } else {
                positions.add(Pair.of(id, eid));
                idsInGap.forEach((gid) -> idsInGaps.add(Pair.of(gid, eid)));
                f.complete(null);
            }
        }, null);
        buffer.release();
        return f;
    }).toArray(CompletableFuture[]::new)).get();
    lh.close();
    return Triple.of(lh.getId(), positions, idsInGaps);
}