Examples with LedgerUnderreplicationManager org.apache.bookkeeper.meta.LedgerUnderreplicationManager used on opensource projects

Example 6 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method testMarkingAsReplicated.

/**
 * Test that when a ledger has been marked as replicated, it
 * will not be offered to anther client.
 * This test checked that by marking two ledgers, and acquiring
 * them on a single client. It marks one as replicated and then
 * the client is killed. We then check that another client can
 * acquire a ledger, and that it's not the one that was previously
 * marked as replicated.
 */
@Test
public void testMarkingAsReplicated() throws Exception {
    String missingReplica = "localhost:3181";
    LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
    LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
    Long ledgerA = 0xfeadeefdacL;
    Long ledgerB = 0xdefadebL;
    m1.markLedgerUnderreplicated(ledgerA, missingReplica);
    m1.markLedgerUnderreplicated(ledgerB, missingReplica);
    Future<Long> fA = getLedgerToReplicate(m1);
    Future<Long> fB = getLedgerToReplicate(m1);
    Long lA = fA.get(5, TimeUnit.SECONDS);
    Long lB = fB.get(5, TimeUnit.SECONDS);
    assertTrue("Should be the ledgers I just marked", (lA.equals(ledgerA) && lB.equals(ledgerB)) || (lA.equals(ledgerB) && lB.equals(ledgerA)));
    Future<Long> f = getLedgerToReplicate(m2);
    try {
        f.get(1, TimeUnit.SECONDS);
        fail("Shouldn't be able to find a ledger to replicate");
    } catch (TimeoutException te) {
    // correct behaviour
    }
    m1.markLedgerReplicated(lA);
    // should kill the lock
    zkc1.close();
    zkc1 = null;
    Long l = f.get(5, TimeUnit.SECONDS);
    assertEquals("Should be the ledger I marked", lB, l);
}

Example 7 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method test2reportSame.

/**
 * Test that when a ledger is marked as underreplicated with
 * the same missing replica twice, only marking as replicated
 * will be enough to remove it from the list.
 */
@Test
public void test2reportSame() throws Exception {
    String missingReplica1 = "localhost:3181";
    LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
    LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
    Long ledgerA = 0xfeadeefdacL;
    m1.markLedgerUnderreplicated(ledgerA, missingReplica1);
    m2.markLedgerUnderreplicated(ledgerA, missingReplica1);
    // verify duplicate missing replica
    UnderreplicatedLedgerFormat.Builder builderA = UnderreplicatedLedgerFormat.newBuilder();
    String znode = getUrLedgerZnode(ledgerA);
    byte[] data = zkc1.getData(znode, false, null);
    TextFormat.merge(new String(data, Charset.forName("UTF-8")), builderA);
    List<String> replicaList = builderA.getReplicaList();
    assertEquals("Published duplicate missing replica : " + replicaList, 1, replicaList.size());
    assertTrue("Published duplicate missing replica : " + replicaList, replicaList.contains(missingReplica1));
    Future<Long> fA = getLedgerToReplicate(m1);
    Long lA = fA.get(5, TimeUnit.SECONDS);
    assertEquals("Should be the ledger I just marked", lA, ledgerA);
    m1.markLedgerReplicated(lA);
    Future<Long> f = getLedgerToReplicate(m2);
    try {
        f.get(1, TimeUnit.SECONDS);
        fail("Shouldn't be able to find a ledger to replicate");
    } catch (TimeoutException te) {
    // correct behaviour
    }
}

Example 8 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class AuditorPeriodicBookieCheckTest method testPeriodicBookieCheckInterval.

/**
 * Test that the periodic bookie checker works.
 */
@Test
public void testPeriodicBookieCheckInterval() throws Exception {
    bsConfs.get(0).setZkServers(zkUtil.getZooKeeperConnectString());
    runFunctionWithLedgerManagerFactory(bsConfs.get(0), mFactory -> {
        try (LedgerManager ledgerManager = mFactory.newLedgerManager()) {
            @Cleanup final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
            LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
            LedgerMetadata md = LedgerHandleAdapter.getLedgerMetadata(lh);
            List<BookieSocketAddress> ensemble = md.getEnsembles().get(0L);
            ensemble.set(0, new BookieSocketAddress("1.1.1.1", 1000));
            TestCallbacks.GenericCallbackFuture<Void> cb = new TestCallbacks.GenericCallbackFuture<Void>();
            ledgerManager.writeLedgerMetadata(lh.getId(), md, cb);
            cb.get();
            long underReplicatedLedger = -1;
            for (int i = 0; i < 10; i++) {
                underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
                if (underReplicatedLedger != -1) {
                    break;
                }
                Thread.sleep(CHECK_INTERVAL * 1000);
            }
            assertEquals("Ledger should be under replicated", lh.getId(), underReplicatedLedger);
        } catch (Exception e) {
            throw new UncheckedExecutionException(e.getMessage(), e);
        }
        return null;
    });
}

Example 9 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class AuditorRollingRestartTest method testAuditingDuringRollingRestart.

private void testAuditingDuringRollingRestart(LedgerManagerFactory mFactory) throws Exception {
    final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
    for (int i = 0; i < 10; i++) {
        lh.asyncAddEntry("foobar".getBytes(), new TestCallbacks.AddCallbackFuture(i), null);
    }
    lh.addEntry("foobar".getBytes());
    lh.close();
    assertEquals("shouldn't be anything under replicated", underReplicationManager.pollLedgerToRereplicate(), -1);
    underReplicationManager.disableLedgerReplication();
    BookieSocketAddress auditor = AuditorElector.getCurrentAuditor(baseConf, zkc);
    ServerConfiguration conf = killBookie(auditor);
    Thread.sleep(2000);
    startBookie(conf);
    // give it time to run
    Thread.sleep(2000);
    assertEquals("shouldn't be anything under replicated", -1, underReplicationManager.pollLedgerToRereplicate());
}

Example 10 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class ListUnderReplicatedLedgerService method handle.

/*
     * Print the node which holds the auditor lock.
     */
@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    // parameter as this: ?missingreplica=<bookie_address>&excludingmissingreplica=<bookid_address>
    Map<String, String> params = request.getParams();
    if (HttpServer.Method.GET == request.getMethod()) {
        final String includingBookieId;
        final String excludingBookieId;
        if (params != null && params.containsKey("missingreplica")) {
            includingBookieId = params.get("missingreplica");
        } else {
            includingBookieId = null;
        }
        if (params != null && params.containsKey("excludingmissingreplica")) {
            excludingBookieId = params.get("excludingmissingreplica");
        } else {
            excludingBookieId = null;
        }
        Predicate<List<String>> predicate = null;
        if (!StringUtils.isBlank(includingBookieId) && !StringUtils.isBlank(excludingBookieId)) {
            predicate = replicasList -> (replicasList.contains(includingBookieId) && !replicasList.contains(excludingBookieId));
        } else if (!StringUtils.isBlank(includingBookieId)) {
            predicate = replicasList -> replicasList.contains(includingBookieId);
        } else if (!StringUtils.isBlank(excludingBookieId)) {
            predicate = replicasList -> !replicasList.contains(excludingBookieId);
        }
        try {
            List<Long> outputLedgers = Lists.newArrayList();
            LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
            LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager();
            Iterator<Long> iter = underreplicationManager.listLedgersToRereplicate(predicate);
            while (iter.hasNext()) {
                outputLedgers.add(iter.next());
            }
            if (outputLedgers.isEmpty()) {
                response.setCode(HttpServer.StatusCode.NOT_FOUND);
                response.setBody("No under replicated ledgers found");
                return response;
            } else {
                response.setCode(HttpServer.StatusCode.OK);
                String jsonResponse = JsonUtil.toJson(outputLedgers);
                LOG.debug("output body: " + jsonResponse);
                response.setBody(jsonResponse);
                return response;
            }
        } catch (Exception e) {
            LOG.error("Exception occurred while listing under replicated ledgers", e);
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Exception when get." + e.getMessage());
            return response;
        }
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be GET method");
        return response;
    }
}