Examples with BookieShell org.apache.bookkeeper.bookie.BookieShell used on opensource projects

Example 1 with BookieShell

use of org.apache.bookkeeper.bookie.BookieShell in project bookkeeper by apache.

the class LedgerCmdTest method testLedgerDbStorageCmd.

/**
 * list of entry logger files that contains given ledgerId.
 */
@Test
public void testLedgerDbStorageCmd() throws Exception {
    BookKeeper bk = new BookKeeper(baseClientConf, zkc);
    LOG.info("Create ledger and add entries to it");
    LedgerHandle lh1 = createLedgerWithEntries(bk, 10);
    bs.forEach(bookieServer -> {
        try {
            BookieAccessor.forceFlush(bookieServer.getBookie());
        } catch (IOException e) {
            LOG.error("Error forceFlush:", e);
        }
    });
    String[] argv = { "ledger", Long.toString(lh1.getId()) };
    final ServerConfiguration conf = bsConfs.get(0);
    conf.setUseHostNameAsBookieID(true);
    BookieShell bkShell = new BookieShell(LedgerIdFormatter.LONG_LEDGERID_FORMATTER, EntryFormatter.STRING_FORMATTER);
    bkShell.setConf(conf);
    assertEquals("Failed to return exit code!", 0, bkShell.run(argv));
}

Example 2 with BookieShell

use of org.apache.bookkeeper.bookie.BookieShell in project bookkeeper by apache.

the class UpdateLedgerCmdTest method updateLedgerCmd.

private void updateLedgerCmd(String[] argv, int exitCode, ServerConfiguration conf) throws KeeperException, InterruptedException, IOException, UnknownHostException, Exception {
    LOG.info("Perform updateledgers command");
    BookieShell bkShell = new BookieShell();
    bkShell.setConf(conf);
    assertEquals("Failed to return exit code!", exitCode, bkShell.run(argv));
}

Example 3 with BookieShell

use of org.apache.bookkeeper.bookie.BookieShell in project bookkeeper by apache.

the class ConversionTest method test.

@Test
public void test() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    System.out.println(tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the interleaved storage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        interleavedStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        interleavedStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            interleavedStorage.addEntry(entry);
        }
    }
    interleavedStorage.flush();
    interleavedStorage.shutdown();
    // Run conversion tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "convert-to-db-storage" });
    Assert.assertEquals(0, res);
    // Verify that db index has the same entries
    DbLedgerStorage dbStorage = new DbLedgerStorage();
    dbStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(dbStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    ledgers = Sets.newTreeSet(interleavedStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, dbStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(dbStorage.readMasterKey(ledgerId)));
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = dbStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
            result.release();
            try {
                interleavedStorage.getEntry(ledgerId, entryId);
                Assert.fail("entry should not exist");
            } catch (NoLedgerException e) {
            // Ok
            }
        }
    }
    interleavedStorage.shutdown();
    dbStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}

Example 4 with BookieShell

use of org.apache.bookkeeper.bookie.BookieShell in project bookkeeper by apache.

the class LocationsIndexRebuildTest method test.

@Test
public void test() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    System.out.println(tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    conf.setLedgerStorageClass(DbLedgerStorage.class.getName());
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    DbLedgerStorage ledgerStorage = new DbLedgerStorage();
    ledgerStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the storage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        ledgerStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        ledgerStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 100; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ledgerStorage.addEntry(entry);
        }
    }
    ledgerStorage.flush();
    ledgerStorage.shutdown();
    // Rebuild index through the tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "rebuild-db-ledger-locations-index" });
    Assert.assertEquals(0, res);
    // Verify that db index has the same entries
    ledgerStorage = new DbLedgerStorage();
    ledgerStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(ledgerStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, ledgerStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(ledgerStorage.readMasterKey(ledgerId)));
        ByteBuf lastEntry = ledgerStorage.getLastEntry(ledgerId);
        assertEquals(ledgerId, lastEntry.readLong());
        long lastEntryId = lastEntry.readLong();
        assertEquals(99, lastEntryId);
        for (long entryId = 0; entryId < 100; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = ledgerStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
        }
    }
    ledgerStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}

Example 5 with BookieShell

use of org.apache.bookkeeper.bookie.BookieShell in project bookkeeper by apache.

the class ConversionRollbackTest method convertFromDbStorageToInterleaved.

@Test
public void convertFromDbStorageToInterleaved() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    log.info("Using temp directory: {}", tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    DbLedgerStorage dbStorage = new DbLedgerStorage();
    dbStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the dbStorage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        dbStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        dbStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            dbStorage.addEntry(entry);
        }
    }
    dbStorage.flush();
    dbStorage.shutdown();
    // Run conversion tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "convert-to-interleaved-storage" });
    Assert.assertEquals(0, res);
    // Verify that interleaved storage index has the same entries
    InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(interleavedStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, interleavedStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(interleavedStorage.readMasterKey(ledgerId)));
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = interleavedStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
        }
    }
    interleavedStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}