Examples with LedgerManagerFactory org.apache.bookkeeper.meta.LedgerManagerFactory used on opensource projects

Example 1 with LedgerManagerFactory

use of org.apache.bookkeeper.meta.LedgerManagerFactory in project bookkeeper by apache.

the class AuditorPeriodicCheckTest method testEntryLogCorruption.

/**
 * test that the periodic checking will detect corruptions in
 * the bookie entry log.
 */
@Test
public void testEntryLogCorruption() throws Exception {
    LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    underReplicationManager.disableLedgerReplication();
    LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
    long ledgerId = lh.getId();
    for (int i = 0; i < 100; i++) {
        lh.addEntry("testdata".getBytes());
    }
    lh.close();
    BookieAccessor.forceFlush(bs.get(0).getBookie());
    File ledgerDir = bsConfs.get(0).getLedgerDirs()[0];
    ledgerDir = Bookie.getCurrentDirectory(ledgerDir);
    // corrupt of entryLogs
    File[] entryLogs = ledgerDir.listFiles(new FilenameFilter() {

        public boolean accept(File dir, String name) {
            return name.endsWith(".log");
        }
    });
    ByteBuffer junk = ByteBuffer.allocate(1024 * 1024);
    for (File f : entryLogs) {
        FileOutputStream out = new FileOutputStream(f);
        out.getChannel().write(junk);
        out.close();
    }
    // restart to clear read buffers
    restartBookies();
    underReplicationManager.enableLedgerReplication();
    long underReplicatedLedger = -1;
    for (int i = 0; i < 10; i++) {
        underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
        if (underReplicatedLedger != -1) {
            break;
        }
        Thread.sleep(CHECK_INTERVAL * 1000);
    }
    assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger);
    underReplicationManager.close();
}

Example 2 with LedgerManagerFactory

use of org.apache.bookkeeper.meta.LedgerManagerFactory in project bookkeeper by apache.

the class AuditorPeriodicCheckTest method testPeriodicCheckWhenDisabled.

/**
 * Test that the period checker will not run when auto replication has been disabled.
 */
@Test
public void testPeriodicCheckWhenDisabled() throws Exception {
    LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
    final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    final int numLedgers = 10;
    final int numMsgs = 2;
    final CountDownLatch completeLatch = new CountDownLatch(numMsgs * numLedgers);
    final AtomicInteger rc = new AtomicInteger(BKException.Code.OK);
    List<LedgerHandle> lhs = new ArrayList<LedgerHandle>();
    for (int i = 0; i < numLedgers; i++) {
        LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
        lhs.add(lh);
        for (int j = 0; j < 2; j++) {
            lh.asyncAddEntry("testdata".getBytes(), new AddCallback() {

                public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) {
                    if (rc.compareAndSet(BKException.Code.OK, rc2)) {
                        LOG.info("Failed to add entry : {}", BKException.getMessage(rc2));
                    }
                    completeLatch.countDown();
                }
            }, null);
        }
    }
    completeLatch.await();
    if (rc.get() != BKException.Code.OK) {
        throw BKException.create(rc.get());
    }
    for (LedgerHandle lh : lhs) {
        lh.close();
    }
    underReplicationManager.disableLedgerReplication();
    final AtomicInteger numReads = new AtomicInteger(0);
    ServerConfiguration conf = killBookie(0);
    Bookie deadBookie = new Bookie(conf) {

        @Override
        public ByteBuf readEntry(long ledgerId, long entryId) throws IOException, NoLedgerException {
            // we want to disable during checking
            numReads.incrementAndGet();
            throw new IOException("Fake I/O exception");
        }
    };
    bsConfs.add(conf);
    bs.add(startBookie(conf, deadBookie));
    Thread.sleep(CHECK_INTERVAL * 2000);
    assertEquals("Nothing should have tried to read", 0, numReads.get());
    underReplicationManager.enableLedgerReplication();
    // give it time to run
    Thread.sleep(CHECK_INTERVAL * 2000);
    underReplicationManager.disableLedgerReplication();
    // give it time to stop, from this point nothing new should be marked
    Thread.sleep(CHECK_INTERVAL * 2000);
    int numUnderreplicated = 0;
    long underReplicatedLedger = -1;
    do {
        underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
        if (underReplicatedLedger == -1) {
            break;
        }
        numUnderreplicated++;
        underReplicationManager.markLedgerReplicated(underReplicatedLedger);
    } while (underReplicatedLedger != -1);
    // give a chance to run again (it shouldn't, it's disabled)
    Thread.sleep(CHECK_INTERVAL * 2000);
    // ensure that nothing is marked as underreplicated
    underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
    assertEquals("There should be no underreplicated ledgers", -1, underReplicatedLedger);
    LOG.info("{} of {} ledgers underreplicated", numUnderreplicated, numUnderreplicated);
    assertTrue("All should be underreplicated", numUnderreplicated <= numLedgers && numUnderreplicated > 0);
}

Example 3 with LedgerManagerFactory

use of org.apache.bookkeeper.meta.LedgerManagerFactory in project bookkeeper by apache.

the class AuditorPeriodicCheckTest method testFailedWriteRecovery.

/*
     * Validates that the periodic ledger check will fix entries with a failed write.
     */
@Test
public void testFailedWriteRecovery() throws Exception {
    LedgerManagerFactory mFactory = driver.getLedgerManagerFactory();
    LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    underReplicationManager.disableLedgerReplication();
    LedgerHandle lh = bkc.createLedger(2, 2, 1, DigestType.CRC32, "passwd".getBytes());
    // kill one of the bookies and replace it with one that rejects write;
    // This way we get into the under replication state
    BookieSocketAddress replacedBookie = replaceBookieWithWriteFailingBookie(lh);
    // Write a few entries; this should cause under replication
    byte[] data = "foobar".getBytes();
    data = "foobar".getBytes();
    lh.addEntry(data);
    lh.addEntry(data);
    lh.addEntry(data);
    lh.close();
    // enable under replication detection and wait for it to report
    // under replicated ledger
    underReplicationManager.enableLedgerReplication();
    long underReplicatedLedger = -1;
    for (int i = 0; i < 5; i++) {
        underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
        if (underReplicatedLedger != -1) {
            break;
        }
        Thread.sleep(CHECK_INTERVAL * 1000);
    }
    assertEquals("Ledger should be under replicated", lh.getId(), underReplicatedLedger);
    // now start the replication workers
    List<ReplicationWorker> l = new ArrayList<ReplicationWorker>();
    for (int i = 0; i < numBookies; i++) {
        ReplicationWorker rw = new ReplicationWorker(zkc, bsConfs.get(i), NullStatsLogger.INSTANCE);
        rw.start();
        l.add(rw);
    }
    underReplicationManager.close();
    // Wait for ensemble to change after replication
    Thread.sleep(3000);
    for (ReplicationWorker rw : l) {
        rw.shutdown();
    }
    // check that ensemble has changed and the bookie that rejected writes has
    // been replaced in the ensemble
    LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, "passwd".getBytes());
    for (Map.Entry<Long, ArrayList<BookieSocketAddress>> e : LedgerHandleAdapter.getLedgerMetadata(newLh).getEnsembles().entrySet()) {
        ArrayList<BookieSocketAddress> ensemble = e.getValue();
        assertFalse("Ensemble hasn't been updated", ensemble.contains(replacedBookie));
    }
    newLh.close();
}

Example 4 with LedgerManagerFactory

use of org.apache.bookkeeper.meta.LedgerManagerFactory in project bookkeeper by apache.

the class Auditor method initialize.

private void initialize(ServerConfiguration conf, ZooKeeper zkc) throws UnavailableException {
    try {
        ClientConfiguration clientConfiguration = new ClientConfiguration(conf);
        clientConfiguration.setClientRole(ClientConfiguration.CLIENT_ROLE_SYSTEM);
        LOG.info("AuthProvider used by the Auditor is {}", clientConfiguration.getClientAuthProviderFactoryClass());
        this.bkc = new BookKeeper(clientConfiguration, zkc);
        LedgerManagerFactory ledgerManagerFactory = AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, bkc.getMetadataClientDriver().getLayoutManager());
        ledgerManager = ledgerManagerFactory.newLedgerManager();
        this.bookieLedgerIndexer = new BookieLedgerIndexer(ledgerManager);
        this.ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager();
        this.admin = new BookKeeperAdmin(bkc, statsLogger);
        if (this.ledgerUnderreplicationManager.initializeLostBookieRecoveryDelay(conf.getLostBookieRecoveryDelay())) {
            LOG.info("Initializing lostBookieRecoveryDelay zNode to the conif value: {}", conf.getLostBookieRecoveryDelay());
        } else {
            LOG.info("Valid lostBookieRecoveryDelay zNode is available, so not creating " + "lostBookieRecoveryDelay zNode as part of Auditor initialization ");
        }
        lostBookieRecoveryDelayBeforeChange = this.ledgerUnderreplicationManager.getLostBookieRecoveryDelay();
    } catch (CompatibilityException ce) {
        throw new UnavailableException("CompatibilityException while initializing Auditor", ce);
    } catch (IOException | BKException | KeeperException ioe) {
        throw new UnavailableException("Exception while initializing Auditor", ioe);
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        throw new UnavailableException("Interrupted while initializing Auditor", ie);
    }
}

Example 5 with LedgerManagerFactory

use of org.apache.bookkeeper.meta.LedgerManagerFactory in project bookkeeper by apache.

the class ListLedgerService method handle.

@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    // parameter could be like: print_metadata=true&page=PageIndex
    if (HttpServer.Method.GET == request.getMethod()) {
        Map<String, String> params = request.getParams();
        // default not print metadata
        boolean printMeta = (params != null) && params.containsKey("print_metadata") && params.get("print_metadata").equals("true");
        // Page index should start from 1;
        int pageIndex = (printMeta && params.containsKey("page")) ? Integer.parseInt(params.get("page")) : -1;
        LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
        LedgerManager manager = mFactory.newLedgerManager();
        LedgerManager.LedgerRangeIterator iter = manager.getLedgerRanges();
        // output <ledgerId: ledgerMetadata>
        LinkedHashMap<String, String> output = Maps.newLinkedHashMap();
        // futures for readLedgerMetadata for each page.
        List<ReadLedgerMetadataCallback> futures = Lists.newArrayListWithExpectedSize(LIST_LEDGER_BATCH_SIZE);
        if (printMeta) {
            int ledgerIndex = 0;
            // start and end ledger index for wanted page.
            int startLedgerIndex = 0;
            int endLedgerIndex = 0;
            if (pageIndex > 0) {
                startLedgerIndex = (pageIndex - 1) * LIST_LEDGER_BATCH_SIZE;
                endLedgerIndex = startLedgerIndex + LIST_LEDGER_BATCH_SIZE - 1;
            }
            // get metadata
            while (iter.hasNext()) {
                LedgerManager.LedgerRange r = iter.next();
                for (Long lid : r.getLedgers()) {
                    ledgerIndex++;
                    if (// no actual page parameter provided
                    endLedgerIndex == 0 || (ledgerIndex >= startLedgerIndex && ledgerIndex <= endLedgerIndex)) {
                        ReadLedgerMetadataCallback cb = new ReadLedgerMetadataCallback(lid);
                        manager.readLedgerMetadata(lid, cb);
                        futures.add(cb);
                    }
                }
                if (futures.size() >= LIST_LEDGER_BATCH_SIZE) {
                    while (futures.size() > 0) {
                        ReadLedgerMetadataCallback cb = futures.remove(0);
                        keepLedgerMetadata(cb, output);
                    }
                }
            }
            while (futures.size() > 0) {
                ReadLedgerMetadataCallback cb = futures.remove(0);
                keepLedgerMetadata(cb, output);
            }
        } else {
            while (iter.hasNext()) {
                LedgerManager.LedgerRange r = iter.next();
                for (Long lid : r.getLedgers()) {
                    output.put(lid.toString(), null);
                }
            }
        }
        manager.close();
        String jsonResponse = JsonUtil.toJson(output);
        LOG.debug("output body:" + jsonResponse);
        response.setBody(jsonResponse);
        response.setCode(HttpServer.StatusCode.OK);
        return response;
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be GET method");
        return response;
    }
}