Examples with LedgerUnderreplicationManager org.apache.bookkeeper.meta.LedgerUnderreplicationManager used on opensource projects

Example 11 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestReplicationWorker method testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR.

/**
 * Tests that ReplicationWorker should fence the ledger and release ledger
 * lock after timeout. Then replication should happen normally.
 */
@Test
public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);
    BookieSocketAddress newBkAddr = startNewBookieAndReturnAddress();
    LOG.info("New Bookie addr : {}", newBkAddr);
    // set to 3s instead of default 30s
    baseConf.setOpenLedgerRereplicationGracePeriod("3000");
    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf);
    @Cleanup MetadataClientDriver clientDriver = MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
    clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
    LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    rw.start();
    try {
        underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
        while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
            Thread.sleep(100);
        }
        killAllBookies(lh, newBkAddr);
        // Should be able to read the entries from 0-9
        verifyRecoveredLedgers(lh, 0, 9);
        lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
        assertFalse("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
    } finally {
        rw.shutdown();
        underReplicationManager.close();
    }
}

Example 12 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestReplicationWorker method testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR.

/**
 * Tests that ReplicationWorker should not have identified for postponing
 * the replication if ledger is in open state and lastFragment is not in
 * underReplication state. Note that RW should not fence such ledgers.
 */
@Test
public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);
    BookieSocketAddress newBkAddr = startNewBookieAndReturnAddress();
    LOG.info("New Bookie addr : {}", newBkAddr);
    // under-replication
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf);
    baseClientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
    driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
    LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    rw.start();
    try {
        underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
        while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
            Thread.sleep(100);
        }
        killAllBookies(lh, newBkAddr);
        // Should be able to read the entries from 0-9
        verifyRecoveredLedgers(lh, 0, 9);
        lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
        // Ledger should be still in open state
        assertTrue("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
    } finally {
        rw.shutdown();
        underReplicationManager.close();
    }
}

Example 13 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class BookKeeperAdmin method setLostBookieRecoveryDelay.

/**
 * Setter for LostBookieRecoveryDelay value (in seconds) in Zookeeper.
 *
 * @param lostBookieRecoveryDelay
 *                              lostBookieRecoveryDelay value (in seconds) to set
 * @throws CompatibilityException
 * @throws KeeperException
 * @throws InterruptedException
 * @throws UnavailableException
 */
public void setLostBookieRecoveryDelay(int lostBookieRecoveryDelay) throws CompatibilityException, KeeperException, InterruptedException, UnavailableException {
    LedgerUnderreplicationManager urlManager = getUnderreplicationManager();
    urlManager.setLostBookieRecoveryDelay(lostBookieRecoveryDelay);
}

Example 14 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class BookKeeperAdmin method triggerAudit.

/**
 * Trigger AuditTask by resetting lostBookieRecoveryDelay to its current
 * value. If Autorecovery is not enabled or if there is no Auditor then this
 * method will throw UnavailableException.
 *
 * @throws CompatibilityException
 * @throws KeeperException
 * @throws InterruptedException
 * @throws UnavailableException
 * @throws IOException
 */
public void triggerAudit() throws CompatibilityException, KeeperException, InterruptedException, UnavailableException, IOException {
    LedgerUnderreplicationManager urlManager = getUnderreplicationManager();
    if (!urlManager.isLedgerReplicationEnabled()) {
        LOG.error("Autorecovery is disabled. So giving up!");
        throw new UnavailableException("Autorecovery is disabled. So giving up!");
    }
    BookieSocketAddress auditorId = AuditorElector.getCurrentAuditor(new ServerConfiguration(bkc.conf), bkc.getZkHandle());
    if (auditorId == null) {
        LOG.error("No auditor elected, though Autorecovery is enabled. So giving up.");
        throw new UnavailableException("No auditor elected, though Autorecovery is enabled. So giving up.");
    }
    int previousLostBookieRecoveryDelayValue = urlManager.getLostBookieRecoveryDelay();
    LOG.info("Resetting LostBookieRecoveryDelay value: {}, to kickstart audit task", previousLostBookieRecoveryDelayValue);
    urlManager.setLostBookieRecoveryDelay(previousLostBookieRecoveryDelayValue);
}

Example 15 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class AuditorPeriodicCheckTest method testIndexCorruption.

/**
 * test that the period checker will detect corruptions in
 * the bookie index files.
 */
@Test
public void testIndexCorruption() throws Exception {
    LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
    long ledgerToCorrupt = lh.getId();
    for (int i = 0; i < 100; i++) {
        lh.addEntry("testdata".getBytes());
    }
    lh.close();
    // push ledgerToCorrupt out of page cache (bookie is configured to only use 1 page)
    lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
    for (int i = 0; i < 100; i++) {
        lh.addEntry("testdata".getBytes());
    }
    lh.close();
    BookieAccessor.forceFlush(bs.get(0).getBookie());
    File ledgerDir = bsConfs.get(0).getLedgerDirs()[0];
    ledgerDir = Bookie.getCurrentDirectory(ledgerDir);
    // corrupt of entryLogs
    File index = new File(ledgerDir, IndexPersistenceMgr.getLedgerName(ledgerToCorrupt));
    LOG.info("file to corrupt{}", index);
    ByteBuffer junk = ByteBuffer.allocate(1024 * 1024);
    FileOutputStream out = new FileOutputStream(index);
    out.getChannel().write(junk);
    out.close();
    long underReplicatedLedger = -1;
    for (int i = 0; i < 10; i++) {
        underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
        if (underReplicatedLedger != -1) {
            break;
        }
        Thread.sleep(CHECK_INTERVAL * 1000);
    }
    assertEquals("Ledger should be under replicated", ledgerToCorrupt, underReplicatedLedger);
    underReplicationManager.close();
}