Examples with MetadataClientDriver org.apache.bookkeeper.meta.MetadataClientDriver used on opensource projects

Example 1 with MetadataClientDriver

use of org.apache.bookkeeper.meta.MetadataClientDriver in project bookkeeper by apache.

the class TestReplicationWorker method testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR.

/**
 * Tests that ReplicationWorker should fence the ledger and release ledger
 * lock after timeout. Then replication should happen normally.
 */
@Test
public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);
    BookieSocketAddress newBkAddr = startNewBookieAndReturnAddress();
    LOG.info("New Bookie addr : {}", newBkAddr);
    // set to 3s instead of default 30s
    baseConf.setOpenLedgerRereplicationGracePeriod("3000");
    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf);
    @Cleanup MetadataClientDriver clientDriver = MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
    clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
    LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    rw.start();
    try {
        underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
        while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
            Thread.sleep(100);
        }
        killAllBookies(lh, newBkAddr);
        // Should be able to read the entries from 0-9
        verifyRecoveredLedgers(lh, 0, 9);
        lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
        assertFalse("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
    } finally {
        rw.shutdown();
        underReplicationManager.close();
    }
}

Example 2 with MetadataClientDriver

use of org.apache.bookkeeper.meta.MetadataClientDriver in project bookkeeper by apache.

the class TestReplicationWorker method testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR.

/**
 * Tests that ReplicationWorker should not have identified for postponing
 * the replication if ledger is in open state and lastFragment is not in
 * underReplication state. Note that RW should not fence such ledgers.
 */
@Test
public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);
    BookieSocketAddress newBkAddr = startNewBookieAndReturnAddress();
    LOG.info("New Bookie addr : {}", newBkAddr);
    // under-replication
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf);
    baseClientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
    driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty());
    LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    rw.start();
    try {
        underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
        while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) {
            Thread.sleep(100);
        }
        killAllBookies(lh, newBkAddr);
        // Should be able to read the entries from 0-9
        verifyRecoveredLedgers(lh, 0, 9);
        lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
        // Ledger should be still in open state
        assertTrue("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
    } finally {
        rw.shutdown();
        underReplicationManager.close();
    }
}

Example 3 with MetadataClientDriver

use of org.apache.bookkeeper.meta.MetadataClientDriver in project bookkeeper by apache.

the class AuditorLedgerCheckerTest method testTriggerAuditorWithNoPendingAuditTask.

@Test
public void testTriggerAuditorWithNoPendingAuditTask() throws Exception {
    // wait for a second so that the initial periodic check finishes
    Thread.sleep(1000);
    int lostBookieRecoveryDelayConfValue = baseConf.getLostBookieRecoveryDelay();
    Auditor auditorBookiesAuditor = getAuditorBookiesAuditor();
    Future<?> auditTask = auditorBookiesAuditor.getAuditTask();
    int lostBookieRecoveryDelayBeforeChange = auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange();
    Assert.assertEquals("auditTask is supposed to be null", null, auditTask);
    Assert.assertEquals("lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", lostBookieRecoveryDelayConfValue, lostBookieRecoveryDelayBeforeChange);
    @Cleanup("shutdown") OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("test-scheduler").numThreads(1).build();
    @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
    driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.of(zkc));
    // there is no easy way to validate if the Auditor has executed Audit process (Auditor.startAudit),
    // without shuttingdown Bookie. To test if by resetting LostBookieRecoveryDelay it does Auditing
    // even when there is no pending AuditTask, following approach is needed.
    // Here we are creating few ledgers ledgermetadata with non-existing bookies as its ensemble.
    // When Auditor does audit it recognizes these ledgers as underreplicated and mark them as
    // under-replicated, since these bookies are not available.
    int numofledgers = 5;
    Random rand = new Random();
    for (int i = 0; i < numofledgers; i++) {
        LedgerMetadata metadata = new LedgerMetadata(3, 2, 2, DigestType.CRC32, "passwd".getBytes());
        ArrayList<BookieSocketAddress> ensemble = new ArrayList<BookieSocketAddress>();
        ensemble.add(new BookieSocketAddress("99.99.99.99:9999"));
        ensemble.add(new BookieSocketAddress("11.11.11.11:1111"));
        ensemble.add(new BookieSocketAddress("88.88.88.88:8888"));
        metadata.addEnsemble(0, ensemble);
        MutableInt ledgerCreateRC = new MutableInt(-1);
        CountDownLatch latch = new CountDownLatch(1);
        long ledgerId = (Math.abs(rand.nextLong())) % 100000000;
        try (LedgerManager lm = driver.getLedgerManagerFactory().newLedgerManager()) {
            lm.createLedgerMetadata(ledgerId, metadata, (rc, result) -> {
                ledgerCreateRC.setValue(rc);
                latch.countDown();
            });
        }
        Assert.assertTrue("Ledger creation should complete within 2 secs", latch.await(2000, TimeUnit.MILLISECONDS));
        Assert.assertEquals("LedgerCreate should succeed and return OK rc value", BKException.Code.OK, ledgerCreateRC.getValue());
        ledgerList.add(ledgerId);
    }
    final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size());
    urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelayBeforeChange);
    assertTrue("Audit should be triggered and created ledgers should be marked as underreplicated", underReplicaLatch.await(2, TimeUnit.SECONDS));
    assertEquals("All the ledgers should be marked as underreplicated", ledgerList.size(), urLedgerList.size());
    auditTask = auditorBookiesAuditor.getAuditTask();
    Assert.assertEquals("auditTask is supposed to be null", null, auditTask);
    Assert.assertEquals("lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", lostBookieRecoveryDelayBeforeChange, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
}

Example 4 with MetadataClientDriver

use of org.apache.bookkeeper.meta.MetadataClientDriver in project bookkeeper by apache.

the class DiscoveryCommand method run.

@Override
public void run(ServerConfiguration conf) throws Exception {
    URI metadataServiceUri = URI.create(conf.getMetadataServiceUri());
    @Cleanup("shutdown") ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    try (MetadataClientDriver driver = MetadataDrivers.getClientDriver(metadataServiceUri)) {
        ClientConfiguration clientConf = new ClientConfiguration(conf);
        driver.initialize(clientConf, executor, NullStatsLogger.INSTANCE, Optional.empty());
        run(driver.getRegistrationClient());
    }
}