Examples with LedgerUnderreplicationManager org.apache.bookkeeper.meta.LedgerUnderreplicationManager used on opensource projects

Example 16 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method verifyMarkLedgerUnderreplicated.

private void verifyMarkLedgerUnderreplicated(Collection<String> missingReplica) throws KeeperException, InterruptedException, CompatibilityException, UnavailableException {
    Long ledgerA = 0xfeadeefdacL;
    String znodeA = getUrLedgerZnode(ledgerA);
    LedgerUnderreplicationManager replicaMgr = lmf1.newLedgerUnderreplicationManager();
    for (String replica : missingReplica) {
        replicaMgr.markLedgerUnderreplicated(ledgerA, replica);
    }
    String urLedgerA = getData(znodeA);
    UnderreplicatedLedgerFormat.Builder builderA = UnderreplicatedLedgerFormat.newBuilder();
    for (String replica : missingReplica) {
        builderA.addReplica(replica);
    }
    List<String> replicaList = builderA.getReplicaList();
    for (String replica : missingReplica) {
        assertTrue("UrLedger:" + urLedgerA + " doesn't contain failed bookie :" + replica, replicaList.contains(replica));
    }
}

Example 17 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method testHierarchyCleanupInterference.

/**
 * Test that as the hierarchy gets cleaned up, it doesn't interfere
 * with the marking of other ledgers as underreplicated.
 */
@Test
public void testHierarchyCleanupInterference() throws Exception {
    final LedgerUnderreplicationManager replicaMgr1 = lmf1.newLedgerUnderreplicationManager();
    final LedgerUnderreplicationManager replicaMgr2 = lmf2.newLedgerUnderreplicationManager();
    final int iterations = 100;
    final AtomicBoolean threadFailed = new AtomicBoolean(false);
    Thread markUnder = new Thread() {

        public void run() {
            long l = 1;
            try {
                for (int i = 0; i < iterations; i++) {
                    replicaMgr1.markLedgerUnderreplicated(l, "localhost:3181");
                    l += 10000;
                }
            } catch (Exception e) {
                LOG.error("markUnder Thread failed with exception", e);
                threadFailed.set(true);
                return;
            }
        }
    };
    final AtomicInteger processed = new AtomicInteger(0);
    Thread markRepl = new Thread() {

        public void run() {
            try {
                for (int i = 0; i < iterations; i++) {
                    long l = replicaMgr2.getLedgerToRereplicate();
                    replicaMgr2.markLedgerReplicated(l);
                    processed.incrementAndGet();
                }
            } catch (Exception e) {
                LOG.error("markRepl Thread failed with exception", e);
                threadFailed.set(true);
                return;
            }
        }
    };
    markRepl.setDaemon(true);
    markUnder.setDaemon(true);
    markRepl.start();
    markUnder.start();
    markUnder.join();
    assertFalse("Thread failed to complete", threadFailed.get());
    int lastProcessed = 0;
    while (true) {
        markRepl.join(10000);
        if (!markRepl.isAlive()) {
            break;
        }
        assertFalse("markRepl thread not progressing", lastProcessed == processed.get());
    }
    assertFalse("Thread failed to complete", threadFailed.get());
    List<String> children = zkc1.getChildren(urLedgerPath, false);
    for (String s : children) {
        LOG.info("s: {}", s);
    }
    assertEquals("All hierarchies should be cleaned up", 0, children.size());
}

Example 18 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method testHierarchyCleanup.

/**
 * Test that the hierarchy gets cleaned up as ledgers
 * are marked as fully replicated.
 */
@Test
public void testHierarchyCleanup() throws Exception {
    final LedgerUnderreplicationManager replicaMgr = lmf1.newLedgerUnderreplicationManager();
    // 4 ledgers, 2 in the same hierarchy
    long[] ledgers = { 0x00000000deadbeefL, 0x00000000deadbeeeL, 0x00000000beefcafeL, 0x00000000cafed00dL };
    for (long l : ledgers) {
        replicaMgr.markLedgerUnderreplicated(l, "localhost:3181");
    }
    // can't simply test top level as we are limited to ledger
    // ids no larger than an int
    String testPath = urLedgerPath + "/0000/0000";
    List<String> children = zkc1.getChildren(testPath, false);
    assertEquals("Wrong number of hierarchies", 3, children.size());
    int marked = 0;
    while (marked < 3) {
        long l = replicaMgr.getLedgerToRereplicate();
        if (l != ledgers[0]) {
            replicaMgr.markLedgerReplicated(l);
            marked++;
        } else {
            replicaMgr.releaseUnderreplicatedLedger(l);
        }
    }
    children = zkc1.getChildren(testPath, false);
    assertEquals("Wrong number of hierarchies", 1, children.size());
    long l = replicaMgr.getLedgerToRereplicate();
    assertEquals("Got wrong ledger", ledgers[0], l);
    replicaMgr.markLedgerReplicated(l);
    children = zkc1.getChildren(urLedgerPath, false);
    assertEquals("All hierarchies should be cleaned up", 0, children.size());
}

Example 19 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method testRelease.

/**
 * Test releasing of a ledger
 * A ledger is released when a client decides it does not want
 * to replicate it (or cannot at the moment).
 * When a client releases a previously acquired ledger, another
 * client should then be able to acquire it.
 */
@Test
public void testRelease() throws Exception {
    String missingReplica = "localhost:3181";
    LedgerUnderreplicationManager m1 = lmf1.newLedgerUnderreplicationManager();
    LedgerUnderreplicationManager m2 = lmf2.newLedgerUnderreplicationManager();
    Long ledgerA = 0xfeadeefdacL;
    Long ledgerB = 0xdefadebL;
    m1.markLedgerUnderreplicated(ledgerA, missingReplica);
    m1.markLedgerUnderreplicated(ledgerB, missingReplica);
    Future<Long> fA = getLedgerToReplicate(m1);
    Future<Long> fB = getLedgerToReplicate(m1);
    Long lA = fA.get(5, TimeUnit.SECONDS);
    Long lB = fB.get(5, TimeUnit.SECONDS);
    assertTrue("Should be the ledgers I just marked", (lA.equals(ledgerA) && lB.equals(ledgerB)) || (lA.equals(ledgerB) && lB.equals(ledgerA)));
    Future<Long> f = getLedgerToReplicate(m2);
    try {
        f.get(1, TimeUnit.SECONDS);
        fail("Shouldn't be able to find a ledger to replicate");
    } catch (TimeoutException te) {
    // correct behaviour
    }
    m1.markLedgerReplicated(lA);
    m1.releaseUnderreplicatedLedger(lB);
    Long l = f.get(5, TimeUnit.SECONDS);
    assertEquals("Should be the ledger I marked", lB, l);
}

Example 20 with LedgerUnderreplicationManager

use of org.apache.bookkeeper.meta.LedgerUnderreplicationManager in project bookkeeper by apache.

the class TestLedgerUnderreplicationManager method testBasicInteraction.

/**
 * Test basic interactions with the ledger underreplication
 * manager.
 * Mark some ledgers as underreplicated.
 * Ensure that getLedgerToReplicate will block until it a ledger
 * becomes available.
 */
@Test
public void testBasicInteraction() throws Exception {
    Set<Long> ledgers = new HashSet<Long>();
    ledgers.add(0xdeadbeefL);
    ledgers.add(0xbeefcafeL);
    ledgers.add(0xffffbeefL);
    ledgers.add(0xfacebeefL);
    String missingReplica = "localhost:3181";
    int count = 0;
    LedgerUnderreplicationManager m = lmf1.newLedgerUnderreplicationManager();
    Iterator<Long> iter = ledgers.iterator();
    while (iter.hasNext()) {
        m.markLedgerUnderreplicated(iter.next(), missingReplica);
        count++;
    }
    List<Future<Long>> futures = new ArrayList<Future<Long>>();
    for (int i = 0; i < count; i++) {
        futures.add(getLedgerToReplicate(m));
    }
    for (Future<Long> f : futures) {
        Long l = f.get(5, TimeUnit.SECONDS);
        assertTrue(ledgers.remove(l));
    }
    Future<Long> f = getLedgerToReplicate(m);
    try {
        f.get(1, TimeUnit.SECONDS);
        fail("Shouldn't be able to find a ledger to replicate");
    } catch (TimeoutException te) {
    // correct behaviour
    }
    Long newl = 0xfefefefefefeL;
    m.markLedgerUnderreplicated(newl, missingReplica);
    assertEquals("Should have got the one just added", newl, f.get(5, TimeUnit.SECONDS));
}