Examples with LedgerDirsManager org.apache.bookkeeper.bookie.LedgerDirsManager used on opensource projects

Example 6 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class LastMarkCommand method run.

@Override
public void run(ServerConfiguration conf) throws Exception {
    LedgerDirsManager dirsManager = new LedgerDirsManager(conf, conf.getJournalDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    File[] journalDirs = conf.getJournalDirs();
    for (int idx = 0; idx < journalDirs.length; idx++) {
        Journal journal = new Journal(idx, journalDirs[idx], conf, dirsManager);
        LogMark lastLogMark = journal.getLastLogMark().getCurMark();
        System.out.println("LastLogMark : Journal Id - " + lastLogMark.getLogFileId() + "(" + Long.toHexString(lastLogMark.getLogFileId()) + ".txn), Pos - " + lastLogMark.getLogFileOffset());
    }
}

Example 7 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class GetLastLogMarkService method handle.

@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    if (HttpServer.Method.GET == request.getMethod()) {
        try {
            /**
             * output:
             *  {
             *    "<Journal_id>" : "<Pos>",
             *    ...
             *  }
             */
            Map<String, String> output = Maps.newHashMap();
            List<Journal> journals = Lists.newArrayListWithCapacity(conf.getJournalDirs().length);
            int idx = 0;
            for (File journalDir : conf.getJournalDirs()) {
                journals.add(new Journal(idx++, journalDir, conf, new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()))));
            }
            for (Journal journal : journals) {
                LogMark lastLogMark = journal.getLastLogMark().getCurMark();
                LOG.debug("LastLogMark: Journal Id - " + lastLogMark.getLogFileId() + "(" + Long.toHexString(lastLogMark.getLogFileId()) + ".txn), Pos - " + lastLogMark.getLogFileOffset());
                output.put("LastLogMark: Journal Id - " + lastLogMark.getLogFileId() + "(" + Long.toHexString(lastLogMark.getLogFileId()) + ".txn)", "Pos - " + lastLogMark.getLogFileOffset());
            }
            String jsonResponse = JsonUtil.toJson(output);
            LOG.debug("output body:" + jsonResponse);
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } catch (Exception e) {
            LOG.error("Exception occurred while getting last log mark", e);
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("ERROR handling request: " + e.getMessage());
            return response;
        }
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be GET method");
        return response;
    }
}

Example 8 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class ConversionTest method test.

@Test
public void test() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    System.out.println(tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the interleaved storage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        interleavedStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        interleavedStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            interleavedStorage.addEntry(entry);
        }
    }
    interleavedStorage.flush();
    interleavedStorage.shutdown();
    // Run conversion tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "convert-to-db-storage" });
    Assert.assertEquals(0, res);
    // Verify that db index has the same entries
    DbLedgerStorage dbStorage = new DbLedgerStorage();
    dbStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(dbStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    ledgers = Sets.newTreeSet(interleavedStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, dbStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(dbStorage.readMasterKey(ledgerId)));
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = dbStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
            result.release();
            try {
                interleavedStorage.getEntry(ledgerId, entryId);
                Assert.fail("entry should not exist");
            } catch (NoLedgerException e) {
            // Ok
            }
        }
    }
    interleavedStorage.shutdown();
    dbStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}

Example 9 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class LocationsIndexRebuildTest method test.

@Test
public void test() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    System.out.println(tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    conf.setLedgerStorageClass(DbLedgerStorage.class.getName());
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    DbLedgerStorage ledgerStorage = new DbLedgerStorage();
    ledgerStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the storage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        ledgerStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        ledgerStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 100; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ledgerStorage.addEntry(entry);
        }
    }
    ledgerStorage.flush();
    ledgerStorage.shutdown();
    // Rebuild index through the tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "rebuild-db-ledger-locations-index" });
    Assert.assertEquals(0, res);
    // Verify that db index has the same entries
    ledgerStorage = new DbLedgerStorage();
    ledgerStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(ledgerStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, ledgerStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(ledgerStorage.readMasterKey(ledgerId)));
        ByteBuf lastEntry = ledgerStorage.getLastEntry(ledgerId);
        assertEquals(ledgerId, lastEntry.readLong());
        long lastEntryId = lastEntry.readLong();
        assertEquals(99, lastEntryId);
        for (long entryId = 0; entryId < 100; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = ledgerStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
        }
    }
    ledgerStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}

Example 10 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class ConversionRollbackTest method convertFromDbStorageToInterleaved.

@Test
public void convertFromDbStorageToInterleaved() throws Exception {
    File tmpDir = File.createTempFile("bkTest", ".dir");
    tmpDir.delete();
    tmpDir.mkdir();
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    log.info("Using temp directory: {}", tmpDir);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    DbLedgerStorage dbStorage = new DbLedgerStorage();
    dbStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    // Insert some ledger & entries in the dbStorage
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        dbStorage.setMasterKey(ledgerId, ("ledger-" + ledgerId).getBytes());
        dbStorage.setFenced(ledgerId);
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(128);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            dbStorage.addEntry(entry);
        }
    }
    dbStorage.flush();
    dbStorage.shutdown();
    // Run conversion tool
    BookieShell shell = new BookieShell();
    shell.setConf(conf);
    int res = shell.run(new String[] { "convert-to-interleaved-storage" });
    Assert.assertEquals(0, res);
    // Verify that interleaved storage index has the same entries
    InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage();
    interleavedStorage.initialize(conf, null, ledgerDirsManager, ledgerDirsManager, null, checkpointSource, checkpointer, NullStatsLogger.INSTANCE);
    Set<Long> ledgers = Sets.newTreeSet(interleavedStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
    Assert.assertEquals(Sets.newTreeSet(Lists.newArrayList(0L, 1L, 2L, 3L, 4L)), ledgers);
    for (long ledgerId = 0; ledgerId < 5; ledgerId++) {
        Assert.assertEquals(true, interleavedStorage.isFenced(ledgerId));
        Assert.assertEquals("ledger-" + ledgerId, new String(interleavedStorage.readMasterKey(ledgerId)));
        for (long entryId = 0; entryId < 10000; entryId++) {
            ByteBuf entry = Unpooled.buffer(1024);
            entry.writeLong(ledgerId);
            entry.writeLong(entryId);
            entry.writeBytes(("entry-" + entryId).getBytes());
            ByteBuf result = interleavedStorage.getEntry(ledgerId, entryId);
            Assert.assertEquals(entry, result);
        }
    }
    interleavedStorage.shutdown();
    FileUtils.forceDelete(tmpDir);
}