Examples with InterleavedLedgerStorage org.apache.bookkeeper.bookie.InterleavedLedgerStorage used on opensource projects

Example 1 with InterleavedLedgerStorage

use of org.apache.bookkeeper.bookie.InterleavedLedgerStorage in project bookkeeper by apache.

the class ConversionTest method test.

@Test
public void test() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    System.out.println(tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the interleaved storage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        interleavedStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        interleavedStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            interleavedStorage.addEntry(entry);
        }
    }
    interleavedStorage.flush();
    interleavedStorage.shutdown();
    // Run conversion tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "convert-to-db-storage" });
    Assert.assertEquals(0, res);
    // Verify that db index has the same entries
    DbLedgerStorage dbStorage = new DbLedgerStorage();
    dbStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(dbStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    ledgers = Sets.newTreeSet(interleavedStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, dbStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(dbStorage.readMasterKey(ledgerId)));
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = dbStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
            result.release();
            try {
                interleavedStorage.getEntry(ledgerId, entryId);
                Assert.fail("entry should not exist");
            } catch (NoLedgerException e) {
            // Ok
            }
        }
    }
    interleavedStorage.shutdown();
    dbStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}

Example 2 with InterleavedLedgerStorage

use of org.apache.bookkeeper.bookie.InterleavedLedgerStorage in project bookkeeper by apache.

the class ConversionRollbackTest method convertFromDbStorageToInterleaved.

@Test
public void convertFromDbStorageToInterleaved() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    log.info("Using temp directory: {}", tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    DbLedgerStorage dbStorage = new DbLedgerStorage();
    dbStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the dbStorage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        dbStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        dbStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            dbStorage.addEntry(entry);
        }
    }
    dbStorage.flush();
    dbStorage.shutdown();
    // Run conversion tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "convert-to-interleaved-storage" });
    Assert.assertEquals(0, res);
    // Verify that interleaved storage index has the same entries
    InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(interleavedStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, interleavedStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(interleavedStorage.readMasterKey(ledgerId)));
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = interleavedStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
        }
    }
    interleavedStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}