Examples with NoWritableLedgerDirException org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException used on opensource projects

Example 6 with NoWritableLedgerDirException

use of org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException in project bookkeeper by apache.

the class BookieStorageThresholdTest method testStorageThresholdCompaction.

@Test
public void testStorageThresholdCompaction() throws Exception {
    stopAllBookies();
    ServerConfiguration conf = newServerConfiguration();
    File ledgerDir1 = createTempDir("ledger", "test1");
    File ledgerDir2 = createTempDir("ledger", "test2");
    File journalDir = createTempDir("journal", "test");
    String[] ledgerDirNames = new String[] { ledgerDir1.getPath(), ledgerDir2.getPath() };
    conf.setLedgerDirNames(ledgerDirNames);
    conf.setJournalDirName(journalDir.getPath());
    BookieServer server = startBookie(conf);
    bs.add(server);
    bsConfs.add(conf);
    Bookie bookie = server.getBookie();
    // since we are going to set dependency injected ledgermonitor, so we need to shutdown
    // the ledgermonitor which was created as part of the initialization of Bookie
    bookie.ledgerMonitor.shutdown();
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    // flag latches
    final CountDownLatch diskWritable = new CountDownLatch(1);
    final CountDownLatch diskFull = new CountDownLatch(1);
    ledgerDirsManager.addLedgerDirsListener(new LedgerDirsListener() {

        @Override
        public void fatalError() {
        }

        @Override
        public void diskWritable(File disk) {
            diskWritable.countDown();
        }

        @Override
        public void diskJustWritable(File disk) {
        }

        @Override
        public void diskFull(File disk) {
            diskFull.countDown();
        }

        @Override
        public void diskFailed(File disk) {
        }

        @Override
        public void diskAlmostFull(File disk) {
        }

        @Override
        public void allDisksFull() {
        }
    });
    // Dependency Injected class
    ThresholdTestDiskChecker thresholdTestDiskChecker = new ThresholdTestDiskChecker(baseConf.getDiskUsageThreshold(), baseConf.getDiskUsageWarnThreshold());
    LedgerDirsMonitor ledgerDirsMonitor = new LedgerDirsMonitor(baseConf, thresholdTestDiskChecker, ledgerDirsManager);
    // set the ledgermonitor and idxmonitor and initiate/start it
    bookie.ledgerMonitor = ledgerDirsMonitor;
    bookie.idxMonitor = ledgerDirsMonitor;
    bookie.ledgerMonitor.init();
    bookie.ledgerMonitor.start();
    // create ledgers and add fragments
    LedgerHandle[] lhs = prepareData(3);
    for (LedgerHandle lh : lhs) {
        lh.close();
    }
    // delete ledger2 and ledger3
    bkc.deleteLedger(lhs[1].getId());
    bkc.deleteLedger(lhs[2].getId());
    // validating that LedgerDirsListener are not triggered yet
    assertTrue("Disk Full shouldn't have been triggered yet", diskFull.getCount() == 1);
    assertTrue("Disk writable shouldn't have been triggered yet", diskWritable.getCount() == 1);
    // set exception injection to true, so that next time when checkDir of DiskChecker (ThresholdTestDiskChecker) is
    // called it will throw DiskOutOfSpaceException
    thresholdTestDiskChecker.setInjectDiskOutOfSpaceException(true);
    // now we are waiting for diskFull latch count to get to 0.
    // we are waiting for diskCheckInterval period, so that next time when LedgerDirsMonitor monitors diskusage of
    // its directories, it would get DiskOutOfSpaceException and hence diskFull of all LedgerDirsListener would be
    // called.
    diskFull.await(baseConf.getDiskCheckInterval() + 500, TimeUnit.MILLISECONDS);
    // verifying that diskFull of all LedgerDirsListener are invoked, so countdown of diskFull should come down to 0
    assertTrue("Disk Full should have been triggered", diskFull.getCount() == 0);
    // making sure diskWritable of LedgerDirsListener are not invoked yet
    assertTrue("Disk writable shouldn't have been triggered yet", diskWritable.getCount() == 1);
    // waiting momentarily, because transition to Readonly mode happens asynchronously when there are no more
    // writableLedgerDirs
    Thread.sleep(500);
    assertTrue("Bookie should be transitioned to ReadOnly", bookie.isReadOnly());
    // there are no writableLedgerDirs
    for (File ledgerDir : bookie.getLedgerDirsManager().getAllLedgerDirs()) {
        assertFalse("Found entry log file ([0,1,2].log. They should have been compacted" + ledgerDir, TestUtils.hasLogFiles(ledgerDir.getParentFile(), true, 0, 1, 2));
    }
    try {
        ledgerDirsManager.getWritableLedgerDirs();
        fail("It is expected that there wont be any Writable LedgerDirs and getWritableLedgerDirs " + "is supposed to throw NoWritableLedgerDirException");
    } catch (NoWritableLedgerDirException nowritableDirsException) {
    }
    // disable exception injection
    thresholdTestDiskChecker.setInjectDiskOutOfSpaceException(false);
    // now we are waiting for diskWritable latch count to get to 0.
    // we are waiting for diskCheckInterval period, so that next time when LedgerDirsMonitor monitors diskusage of
    // its directories, it would find writableledgerdirectory and hence diskWritable of all LedgerDirsListener would
    // be called.
    diskWritable.await(baseConf.getDiskCheckInterval() + 500, TimeUnit.MILLISECONDS);
    // verifying that diskWritable of all LedgerDirsListener are invoked, so countdown of diskWritable should come
    // down to 0
    assertTrue("Disk writable should have been triggered", diskWritable.getCount() == 0);
    // waiting momentarily, because transition to ReadWrite mode happens asynchronously when there is new
    // writableLedgerDirectory
    Thread.sleep(500);
    assertFalse("Bookie should be transitioned to ReadWrite", bookie.isReadOnly());
}

Example 7 with NoWritableLedgerDirException

use of org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException in project bookkeeper by apache.

the class CompactionTest method testMinorCompactionWithNoWritableLedgerDirsButIsForceGCAllowWhenNoSpaceIsSet.

@Test
public void testMinorCompactionWithNoWritableLedgerDirsButIsForceGCAllowWhenNoSpaceIsSet() throws Exception {
    stopAllBookies();
    ServerConfiguration conf = newServerConfiguration();
    // disable major compaction
    conf.setMajorCompactionThreshold(0.0f);
    // here we are setting isForceGCAllowWhenNoSpace to true, so Major and Minor compaction wont be disabled in case
    // when discs are full
    conf.setIsForceGCAllowWhenNoSpace(true);
    conf.setGcWaitTime(600000);
    conf.setMinorCompactionInterval(120000);
    conf.setMajorCompactionInterval(240000);
    // We need at least 2 ledger dirs because compaction will flush ledger cache, and will
    // trigger relocateIndexFileAndFlushHeader. If we only have one ledger dir, compaction will always fail
    // when there's no writeable ledger dir.
    File ledgerDir1 = createTempDir("ledger", "test1");
    File ledgerDir2 = createTempDir("ledger", "test2");
    File journalDir = createTempDir("journal", "test");
    String[] ledgerDirNames = new String[] { ledgerDir1.getPath(), ledgerDir2.getPath() };
    conf.setLedgerDirNames(ledgerDirNames);
    conf.setJournalDirName(journalDir.getPath());
    BookieServer server = startBookie(conf);
    bs.add(server);
    bsConfs.add(conf);
    // prepare data
    LedgerHandle[] lhs = prepareData(3, false);
    for (LedgerHandle lh : lhs) {
        lh.close();
    }
    long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime;
    long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime;
    assertFalse(getGCThread().enableMajorCompaction);
    assertTrue(getGCThread().enableMinorCompaction);
    for (BookieServer bookieServer : bs) {
        Bookie bookie = bookieServer.getBookie();
        bookie.ledgerStorage.flush();
        bookie.ledgerMonitor.shutdown();
        LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
        List<File> ledgerDirs = ledgerDirsManager.getAllLedgerDirs();
        // interleavedLedgerStorage.
        for (File ledgerDir : ledgerDirs) {
            ledgerDirsManager.addToFilledDirs(ledgerDir);
        }
    }
    // remove ledger2 and ledger3
    bkc.deleteLedger(lhs[1].getId());
    bkc.deleteLedger(lhs[2].getId());
    LOG.info("Finished deleting the ledgers contains most entries.");
    getGCThread().triggerGC(true, false, false).get();
    // after garbage collection, major compaction should not be executed
    assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime);
    assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime);
    // entry logs ([0,1,2].log) should be compacted.
    for (File ledgerDirectory : server.getBookie().getLedgerDirsManager().getAllLedgerDirs()) {
        assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: " + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory.getParentFile(), true, 0, 1, 2));
    }
    // even entry log files are removed, we still can access entries for ledger1
    // since those entries has been compacted to new entry log
    verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
    // for the sake of validity of test lets make sure that there is no writableLedgerDir in the bookies
    for (BookieServer bookieServer : bs) {
        Bookie bookie = bookieServer.getBookie();
        LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
        try {
            List<File> ledgerDirs = ledgerDirsManager.getWritableLedgerDirs();
            // it is expected not to have any writableLedgerDirs since we added all of them to FilledDirs
            fail("It is expected not to have any writableLedgerDirs");
        } catch (NoWritableLedgerDirException nwe) {
        }
    }
}

Example 8 with NoWritableLedgerDirException

use of org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException in project bookkeeper by apache.

the class BookieInitializationTest method testWithDiskFullReadOnlyDisabledOrForceGCAllowDisabled.

/**
 * Check disk full. Expected to fail on start.
 */
@Test
public void testWithDiskFullReadOnlyDisabledOrForceGCAllowDisabled() throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    long usableSpace = tmpDir.getUsableSpace();
    long totalSpace = tmpDir.getTotalSpace();
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setDiskCheckInterval(1000).setDiskUsageThreshold((1.0f - ((float) usableSpace / (float) totalSpace)) * 0.999f).setDiskUsageWarnThreshold(0.0f);
    conf.setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000);
    // if isForceGCAllowWhenNoSpace or readOnlyModeEnabled is not set and Bookie is
    // started when Disk is full, then it will fail to start with NoWritableLedgerDirException
    conf.setIsForceGCAllowWhenNoSpace(false).setReadOnlyModeEnabled(false);
    try {
        new Bookie(conf);
        fail("NoWritableLedgerDirException expected");
    } catch (NoWritableLedgerDirException e) {
    // expected
    }
    conf.setIsForceGCAllowWhenNoSpace(true).setReadOnlyModeEnabled(false);
    try {
        new Bookie(conf);
        fail("NoWritableLedgerDirException expected");
    } catch (NoWritableLedgerDirException e) {
    // expected
    }
    conf.setIsForceGCAllowWhenNoSpace(false).setReadOnlyModeEnabled(true);
    try {
        new Bookie(conf);
        fail("NoWritableLedgerDirException expected");
    } catch (NoWritableLedgerDirException e) {
    // expected
    }
}

Example 9 with NoWritableLedgerDirException

use of org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException in project bookkeeper by apache.

the class LedgerDirsMonitor method check.

private void check() {
    try {
        List<File> writableDirs = ldm.getWritableLedgerDirs();
        // Check all writable dirs disk space usage.
        for (File dir : writableDirs) {
            try {
                diskUsages.put(dir, diskChecker.checkDir(dir));
            } catch (DiskErrorException e) {
                LOG.error("Ledger directory {} failed on disk checking : ", dir, e);
                // Notify disk failure to all listeners
                for (LedgerDirsListener listener : ldm.getListeners()) {
                    listener.diskFailed(dir);
                }
            } catch (DiskWarnThresholdException e) {
                LOG.warn("Ledger directory {} is almost full.", dir);
                diskUsages.put(dir, e.getUsage());
                for (LedgerDirsListener listener : ldm.getListeners()) {
                    listener.diskAlmostFull(dir);
                }
            } catch (DiskOutOfSpaceException e) {
                LOG.error("Ledger directory {} is out-of-space.", dir);
                diskUsages.put(dir, e.getUsage());
                // Notify disk full to all listeners
                ldm.addToFilledDirs(dir);
            }
        }
        // Let's get NoWritableLedgerDirException without waiting for the next iteration
        // in case we are out of writable dirs
        // otherwise for the duration of {interval} we end up in the state where
        // bookie cannot get writable dir but considered to be writable
        ldm.getWritableLedgerDirs();
    } catch (NoWritableLedgerDirException e) {
        for (LedgerDirsListener listener : ldm.getListeners()) {
            listener.allDisksFull();
        }
    }
    List<File> fullfilledDirs = new ArrayList<File>(ldm.getFullFilledLedgerDirs());
    boolean hasWritableLedgerDirs = ldm.hasWritableLedgerDirs();
    float totalDiskUsage = 0;
    // their usage is < conf.getDiskUsageThreshold.
    try {
        if (hasWritableLedgerDirs || (totalDiskUsage = diskChecker.getTotalDiskUsage(ldm.getAllLedgerDirs())) < conf.getDiskLowWaterMarkUsageThreshold()) {
            // Check all full-filled disk space usage
            for (File dir : fullfilledDirs) {
                try {
                    diskUsages.put(dir, diskChecker.checkDir(dir));
                    ldm.addToWritableDirs(dir, true);
                } catch (DiskErrorException e) {
                    // Notify disk failure to all the listeners
                    for (LedgerDirsListener listener : ldm.getListeners()) {
                        listener.diskFailed(dir);
                    }
                } catch (DiskWarnThresholdException e) {
                    diskUsages.put(dir, e.getUsage());
                    // the full-filled dir become writable but still
                    // above
                    // warn threshold
                    ldm.addToWritableDirs(dir, false);
                } catch (DiskOutOfSpaceException e) {
                    // the full-filled dir is still full-filled
                    diskUsages.put(dir, e.getUsage());
                }
            }
        } else {
            LOG.debug("Current TotalDiskUsage: {} is greater than LWMThreshold: {}." + " So not adding any filledDir to WritableDirsList", totalDiskUsage, conf.getDiskLowWaterMarkUsageThreshold());
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException while monitoring Dirs", ioe);
        for (LedgerDirsListener listener : ldm.getListeners()) {
            listener.fatalError();
        }
    }
}