Examples with LedgerDirsListener org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener used on opensource projects

Example 1 with LedgerDirsListener

use of org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener in project bookkeeper by apache.

the class TestSyncThread method testSyncThreadDisksFull.

/**
 * Test that if the ledger storage throws
 * a disk full exception, the owner of the sync
 * thread will be notified.
 */
@Test
public void testSyncThreadDisksFull() throws Exception {
    int flushInterval = 100;
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setFlushInterval(flushInterval);
    CheckpointSource checkpointSource = new DummyCheckpointSource();
    final CountDownLatch diskFullLatch = new CountDownLatch(1);
    LedgerDirsListener listener = new DummyLedgerDirsListener() {

        @Override
        public void allDisksFull() {
            diskFullLatch.countDown();
        }
    };
    LedgerStorage storage = new DummyLedgerStorage() {

        @Override
        public void checkpoint(Checkpoint checkpoint) throws IOException {
            throw new NoWritableLedgerDirException("Disk full error in sync thread");
        }
    };
    final SyncThread t = new SyncThread(conf, listener, storage, checkpointSource);
    t.startCheckpoint(Checkpoint.MAX);
    assertTrue("Should have disk full error", diskFullLatch.await(10, TimeUnit.SECONDS));
    t.shutdown();
}

Example 2 with LedgerDirsListener

use of org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener in project bookkeeper by apache.

the class BookieStorageThresholdTest method testStorageThresholdCompaction.

@Test
public void testStorageThresholdCompaction() throws Exception {
    stopAllBookies();
    ServerConfiguration conf = newServerConfiguration();
    File ledgerDir1 = createTempDir("ledger", "test1");
    File ledgerDir2 = createTempDir("ledger", "test2");
    File journalDir = createTempDir("journal", "test");
    String[] ledgerDirNames = new String[] { ledgerDir1.getPath(), ledgerDir2.getPath() };
    conf.setLedgerDirNames(ledgerDirNames);
    conf.setJournalDirName(journalDir.getPath());
    BookieServer server = startBookie(conf);
    bs.add(server);
    bsConfs.add(conf);
    Bookie bookie = server.getBookie();
    // since we are going to set dependency injected ledgermonitor, so we need to shutdown
    // the ledgermonitor which was created as part of the initialization of Bookie
    bookie.ledgerMonitor.shutdown();
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    // flag latches
    final CountDownLatch diskWritable = new CountDownLatch(1);
    final CountDownLatch diskFull = new CountDownLatch(1);
    ledgerDirsManager.addLedgerDirsListener(new LedgerDirsListener() {

        @Override
        public void fatalError() {
        }

        @Override
        public void diskWritable(File disk) {
            diskWritable.countDown();
        }

        @Override
        public void diskJustWritable(File disk) {
        }

        @Override
        public void diskFull(File disk) {
            diskFull.countDown();
        }

        @Override
        public void diskFailed(File disk) {
        }

        @Override
        public void diskAlmostFull(File disk) {
        }

        @Override
        public void allDisksFull() {
        }
    });
    // Dependency Injected class
    ThresholdTestDiskChecker thresholdTestDiskChecker = new ThresholdTestDiskChecker(baseConf.getDiskUsageThreshold(), baseConf.getDiskUsageWarnThreshold());
    LedgerDirsMonitor ledgerDirsMonitor = new LedgerDirsMonitor(baseConf, thresholdTestDiskChecker, ledgerDirsManager);
    // set the ledgermonitor and idxmonitor and initiate/start it
    bookie.ledgerMonitor = ledgerDirsMonitor;
    bookie.idxMonitor = ledgerDirsMonitor;
    bookie.ledgerMonitor.init();
    bookie.ledgerMonitor.start();
    // create ledgers and add fragments
    LedgerHandle[] lhs = prepareData(3);
    for (LedgerHandle lh : lhs) {
        lh.close();
    }
    // delete ledger2 and ledger3
    bkc.deleteLedger(lhs[1].getId());
    bkc.deleteLedger(lhs[2].getId());
    // validating that LedgerDirsListener are not triggered yet
    assertTrue("Disk Full shouldn't have been triggered yet", diskFull.getCount() == 1);
    assertTrue("Disk writable shouldn't have been triggered yet", diskWritable.getCount() == 1);
    // set exception injection to true, so that next time when checkDir of DiskChecker (ThresholdTestDiskChecker) is
    // called it will throw DiskOutOfSpaceException
    thresholdTestDiskChecker.setInjectDiskOutOfSpaceException(true);
    // now we are waiting for diskFull latch count to get to 0.
    // we are waiting for diskCheckInterval period, so that next time when LedgerDirsMonitor monitors diskusage of
    // its directories, it would get DiskOutOfSpaceException and hence diskFull of all LedgerDirsListener would be
    // called.
    diskFull.await(baseConf.getDiskCheckInterval() + 500, TimeUnit.MILLISECONDS);
    // verifying that diskFull of all LedgerDirsListener are invoked, so countdown of diskFull should come down to 0
    assertTrue("Disk Full should have been triggered", diskFull.getCount() == 0);
    // making sure diskWritable of LedgerDirsListener are not invoked yet
    assertTrue("Disk writable shouldn't have been triggered yet", diskWritable.getCount() == 1);
    // waiting momentarily, because transition to Readonly mode happens asynchronously when there are no more
    // writableLedgerDirs
    Thread.sleep(500);
    assertTrue("Bookie should be transitioned to ReadOnly", bookie.isReadOnly());
    // there are no writableLedgerDirs
    for (File ledgerDir : bookie.getLedgerDirsManager().getAllLedgerDirs()) {
        assertFalse("Found entry log file ([0,1,2].log. They should have been compacted" + ledgerDir, TestUtils.hasLogFiles(ledgerDir.getParentFile(), true, 0, 1, 2));
    }
    try {
        ledgerDirsManager.getWritableLedgerDirs();
        fail("It is expected that there wont be any Writable LedgerDirs and getWritableLedgerDirs " + "is supposed to throw NoWritableLedgerDirException");
    } catch (NoWritableLedgerDirException nowritableDirsException) {
    }
    // disable exception injection
    thresholdTestDiskChecker.setInjectDiskOutOfSpaceException(false);
    // now we are waiting for diskWritable latch count to get to 0.
    // we are waiting for diskCheckInterval period, so that next time when LedgerDirsMonitor monitors diskusage of
    // its directories, it would find writableledgerdirectory and hence diskWritable of all LedgerDirsListener would
    // be called.
    diskWritable.await(baseConf.getDiskCheckInterval() + 500, TimeUnit.MILLISECONDS);
    // verifying that diskWritable of all LedgerDirsListener are invoked, so countdown of diskWritable should come
    // down to 0
    assertTrue("Disk writable should have been triggered", diskWritable.getCount() == 0);
    // waiting momentarily, because transition to ReadWrite mode happens asynchronously when there is new
    // writableLedgerDirectory
    Thread.sleep(500);
    assertFalse("Bookie should be transitioned to ReadWrite", bookie.isReadOnly());
}

Example 3 with LedgerDirsListener

use of org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener in project bookkeeper by apache.

the class TestSyncThread method testSyncThreadShutdownOnError.

/**
 * Test that if the ledger storage throws a
 * runtime exception, the bookie will be told
 * to shutdown.
 */
@Test
public void testSyncThreadShutdownOnError() throws Exception {
    int flushInterval = 100;
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setFlushInterval(flushInterval);
    CheckpointSource checkpointSource = new DummyCheckpointSource();
    final CountDownLatch fatalLatch = new CountDownLatch(1);
    LedgerDirsListener listener = new DummyLedgerDirsListener() {

        @Override
        public void fatalError() {
            fatalLatch.countDown();
        }
    };
    LedgerStorage storage = new DummyLedgerStorage() {

        @Override
        public void checkpoint(Checkpoint checkpoint) throws IOException {
            throw new RuntimeException("Fatal error in sync thread");
        }
    };
    final SyncThread t = new SyncThread(conf, listener, storage, checkpointSource);
    t.startCheckpoint(Checkpoint.MAX);
    assertTrue("Should have called fatal error", fatalLatch.await(10, TimeUnit.SECONDS));
    t.shutdown();
}

Example 4 with LedgerDirsListener

use of org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener in project bookkeeper by apache.

the class TestSyncThread method testSyncThreadLongShutdown.

/**
 * Test that if a flush is taking a long time,
 * the sync thread will not shutdown until it
 * has finished.
 */
@Test
public void testSyncThreadLongShutdown() throws Exception {
    int flushInterval = 100;
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setFlushInterval(flushInterval);
    CheckpointSource checkpointSource = new DummyCheckpointSource();
    LedgerDirsListener listener = new DummyLedgerDirsListener();
    final CountDownLatch checkpointCalledLatch = new CountDownLatch(1);
    final CountDownLatch checkpointLatch = new CountDownLatch(1);
    final CountDownLatch flushCalledLatch = new CountDownLatch(1);
    final CountDownLatch flushLatch = new CountDownLatch(1);
    final AtomicBoolean failedSomewhere = new AtomicBoolean(false);
    LedgerStorage storage = new DummyLedgerStorage() {

        @Override
        public void flush() throws IOException {
            flushCalledLatch.countDown();
            try {
                flushLatch.await();
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
                LOG.error("Interrupted in flush thread", ie);
                failedSomewhere.set(true);
            }
        }

        @Override
        public void checkpoint(Checkpoint checkpoint) throws IOException {
            checkpointCalledLatch.countDown();
            try {
                checkpointLatch.await();
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
                LOG.error("Interrupted in checkpoint thread", ie);
                failedSomewhere.set(true);
            }
        }
    };
    final SyncThread t = new SyncThread(conf, listener, storage, checkpointSource);
    t.startCheckpoint(Checkpoint.MAX);
    assertTrue("Checkpoint should have been called", checkpointCalledLatch.await(10, TimeUnit.SECONDS));
    Future<Boolean> done = executor.submit(() -> {
        try {
            t.shutdown();
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            LOG.error("Interrupted shutting down sync thread", ie);
            failedSomewhere.set(true);
            return false;
        }
        return true;
    });
    checkpointLatch.countDown();
    assertFalse("Shutdown shouldn't have finished", done.isDone());
    assertTrue("Flush should have been called", flushCalledLatch.await(10, TimeUnit.SECONDS));
    assertFalse("Shutdown shouldn't have finished", done.isDone());
    flushLatch.countDown();
    assertTrue("Shutdown should have finished successfully", done.get(10, TimeUnit.SECONDS));
    assertFalse("Shouldn't have failed anywhere", failedSomewhere.get());
}

Example 5 with LedgerDirsListener

use of org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener in project bookkeeper by apache.

the class TestSyncThread method testSyncThreadSuspension.

/**
 * Test that sync thread suspension works.
 * i.e. when we suspend the syncthread, nothing
 * will be synced.
 */
@Test
public void testSyncThreadSuspension() throws Exception {
    int flushInterval = 100;
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setFlushInterval(flushInterval);
    CheckpointSource checkpointSource = new DummyCheckpointSource();
    LedgerDirsListener listener = new DummyLedgerDirsListener();
    final AtomicInteger checkpointCount = new AtomicInteger(0);
    LedgerStorage storage = new DummyLedgerStorage() {

        @Override
        public void checkpoint(Checkpoint checkpoint) throws IOException {
            checkpointCount.incrementAndGet();
        }
    };
    final SyncThread t = new SyncThread(conf, listener, storage, checkpointSource);
    t.startCheckpoint(Checkpoint.MAX);
    while (checkpointCount.get() == 0) {
        Thread.sleep(flushInterval);
    }
    t.suspendSync();
    Thread.sleep(flushInterval);
    int count = checkpointCount.get();
    for (int i = 0; i < 10; i++) {
        t.startCheckpoint(Checkpoint.MAX);
        assertEquals("Checkpoint count shouldn't change", count, checkpointCount.get());
    }
    t.resumeSync();
    int i = 0;
    while (checkpointCount.get() == count) {
        Thread.sleep(flushInterval);
        i++;
        if (i > 100) {
            fail("Checkpointing never resumed");
        }
    }
    t.shutdown();
}