Examples with LedgerDirsManager org.apache.bookkeeper.bookie.LedgerDirsManager used on opensource projects

Example 1 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class ForceReadOnlyBookieTest method testBookieForceStartAsReadOnly.

/**
 * Check force start readonly bookie.
 */
@Test
public void testBookieForceStartAsReadOnly() throws Exception {
    // create ledger, add entries
    LedgerHandle ledger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    ledger.close();
    LOG.info("successed prepare");
    // start bookie 1 as readonly
    bsConfs.get(1).setReadOnlyModeEnabled(true);
    bsConfs.get(1).setForceReadOnlyBookie(true);
    restartBookies();
    Bookie bookie = bs.get(1).getBookie();
    assertTrue("Bookie should be running and in readonly mode", bookie.isRunning() && bookie.isReadOnly());
    LOG.info("successed force start ReadOnlyBookie");
    // Check new bookie with readonly mode enabled.
    File[] ledgerDirs = bsConfs.get(1).getLedgerDirs();
    assertEquals("Only one ledger dir should be present", 1, ledgerDirs.length);
    // kill the writable bookie
    killBookie(0);
    // read entry from read only bookie
    Enumeration<LedgerEntry> readEntries = ledger.readEntries(0, 9);
    while (readEntries.hasMoreElements()) {
        LedgerEntry entry = readEntries.nextElement();
        assertEquals("Entry should contain correct data", "data", new String(entry.getEntry()));
    }
    LOG.info("successed read entry from ReadOnlyBookie");
    // test will not transfer to Writable mode.
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    ledgerDirsManager.addToWritableDirs(new File(ledgerDirs[0], "current"), true);
    assertTrue("Bookie should be running and in readonly mode", bookie.isRunning() && bookie.isReadOnly());
    LOG.info("successed: bookie still readonly");
}

Example 2 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class ReadOnlyBookieTest method testBookieShouldServeAsReadOnly.

/**
 * Check readonly bookie.
 */
@Test
public void testBookieShouldServeAsReadOnly() throws Exception {
    killBookie(0);
    baseConf.setReadOnlyModeEnabled(true);
    startNewBookie();
    LedgerHandle ledger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    // Check new bookie with readonly mode enabled.
    File[] ledgerDirs = bsConfs.get(1).getLedgerDirs();
    assertEquals("Only one ledger dir should be present", 1, ledgerDirs.length);
    Bookie bookie = bs.get(1).getBookie();
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    // Now add the current ledger dir to filled dirs list
    ledgerDirsManager.addToFilledDirs(new File(ledgerDirs[0], "current"));
    try {
        ledger.addEntry("data".getBytes());
        fail("Should fail to add entry since there isn't enough bookies alive.");
    } catch (BKException.BKNotEnoughBookiesException e) {
    // Expected
    }
    assertTrue("Bookie should be running and converted to readonly mode", bookie.isRunning() && bookie.isReadOnly());
    // Now kill the other bookie and read entries from the readonly bookie
    killBookie(0);
    Enumeration<LedgerEntry> readEntries = ledger.readEntries(0, 9);
    while (readEntries.hasMoreElements()) {
        LedgerEntry entry = readEntries.nextElement();
        assertEquals("Entry should contain correct data", "data", new String(entry.getEntry()));
    }
}

Example 3 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class ReadOnlyBookieTest method testBookieShutdownIfReadOnlyModeNotEnabled.

/**
 * check readOnlyModeEnabled=false.
 */
@Test
public void testBookieShutdownIfReadOnlyModeNotEnabled() throws Exception {
    killBookie(1);
    baseConf.setReadOnlyModeEnabled(false);
    startNewBookie();
    File[] ledgerDirs = bsConfs.get(1).getLedgerDirs();
    assertEquals("Only one ledger dir should be present", 1, ledgerDirs.length);
    Bookie bookie = bs.get(1).getBookie();
    LedgerHandle ledger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    // Now add the current ledger dir to filled dirs list
    ledgerDirsManager.addToFilledDirs(new File(ledgerDirs[0], "current"));
    try {
        ledger.addEntry("data".getBytes());
        fail("Should fail to add entry since there isn't enough bookies alive.");
    } catch (BKException.BKNotEnoughBookiesException e) {
    // Expected
    }
    // wait for up to 10 seconds for bookie to shut down
    for (int i = 0; i < 10 && bookie.isAlive(); i++) {
        Thread.sleep(1000);
    }
    assertFalse("Bookie should shutdown if readOnlyMode not enabled", bookie.isAlive());
}

Example 4 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class ReadOnlyBookieTest method testBookieContinueWritingIfMultipleLedgersPresent.

/**
 * Check multiple ledger dirs.
 */
@Test
public void testBookieContinueWritingIfMultipleLedgersPresent() throws Exception {
    startNewBookieWithMultipleLedgerDirs(2);
    File[] ledgerDirs = bsConfs.get(1).getLedgerDirs();
    assertEquals("Only one ledger dir should be present", 2, ledgerDirs.length);
    Bookie bookie = bs.get(1).getBookie();
    LedgerHandle ledger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    // Now add the current ledger dir to filled dirs list
    ledgerDirsManager.addToFilledDirs(new File(ledgerDirs[0], "current"));
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    assertEquals("writable dirs should have one dir", 1, ledgerDirsManager.getWritableLedgerDirs().size());
    assertTrue("Bookie should shutdown if readOnlyMode not enabled", bookie.isAlive());
}

Example 5 with LedgerDirsManager

use of org.apache.bookkeeper.bookie.LedgerDirsManager in project bookkeeper by apache.

the class LocationsIndexRebuildOp method initiate.

public void initiate() throws IOException {
    LOG.info("Starting index rebuilding");
    // Move locations index to a backup directory
    String basePath = Bookie.getCurrentDirectory(conf.getLedgerDirs()[0]).toString();
    Path currentPath = FileSystems.getDefault().getPath(basePath, "locations");
    String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(new Date());
    Path backupPath = FileSystems.getDefault().getPath(basePath, "locations.BACKUP-" + timestamp);
    Files.move(currentPath, backupPath);
    LOG.info("Created locations index backup at {}", backupPath);
    long startTime = System.nanoTime();
    EntryLogger entryLogger = new EntryLogger(conf, new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold())));
    Set<Long> entryLogs = entryLogger.getEntryLogsSet();
    String locationsDbPath = FileSystems.getDefault().getPath(basePath, "locations").toFile().toString();
    Set<Long> activeLedgers = getActiveLedgers(conf, KeyValueStorageRocksDB.factory, basePath);
    LOG.info("Found {} active ledgers in ledger manager", activeLedgers.size());
    KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(locationsDbPath, DbConfigType.Huge, conf);
    int totalEntryLogs = entryLogs.size();
    int completedEntryLogs = 0;
    LOG.info("Scanning {} entry logs", totalEntryLogs);
    for (long entryLogId : entryLogs) {
        entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {

            @Override
            public void process(long ledgerId, long offset, ByteBuf entry) throws IOException {
                long entryId = entry.getLong(8);
                // Actual location indexed is pointing past the entry size
                long location = (entryLogId << 32L) | (offset + 4);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Rebuilding {}:{} at location {} / {}", ledgerId, entryId, location >> 32, location & (Integer.MAX_VALUE - 1));
                }
                // Update the ledger index page
                LongPairWrapper key = LongPairWrapper.get(ledgerId, entryId);
                LongWrapper value = LongWrapper.get(location);
                newIndex.put(key.array, value.array);
            }

            @Override
            public boolean accept(long ledgerId) {
                return activeLedgers.contains(ledgerId);
            }
        });
        ++completedEntryLogs;
        LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs, totalEntryLogs);
    }
    newIndex.sync();
    newIndex.close();
    LOG.info("Rebuilding index is done. Total time: {}", DurationFormatUtils.formatDurationHMS(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}