Examples with BookieSocketAddress org.apache.bookkeeper.net.BookieSocketAddress used on opensource projects

Example 46 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class BookKeeperCloseTest method testBookKeeperAdmin.

/**
 * Test that BookKeeperAdmin operationg using a closed BK client will
 * throw a ClientClosedException.
 */
@Test
public void testBookKeeperAdmin() throws Exception {
    BookKeeper bk = new BookKeeper(baseClientConf, zkc);
    try (BookKeeperAdmin bkadmin = new BookKeeperAdmin(bk)) {
        LOG.info("Create ledger and add entries to it");
        LedgerHandle lh1 = createLedgerWithEntries(bk, 100);
        LedgerHandle lh2 = createLedgerWithEntries(bk, 100);
        LedgerHandle lh3 = createLedgerWithEntries(bk, 100);
        lh3.close();
        BookieSocketAddress bookieToKill = getBookie(0);
        killBookie(bookieToKill);
        startNewBookie();
        CheckerCb checkercb = new CheckerCb();
        LedgerChecker lc = new LedgerChecker(bk);
        lc.checkLedger(lh3, checkercb);
        assertEquals("Should have completed", checkercb.getRc(30, TimeUnit.SECONDS), BKException.Code.OK);
        assertEquals("Should have a missing fragment", 1, checkercb.getResult(30, TimeUnit.SECONDS).size());
        // make sure a bookie in each quorum is slow
        restartBookieSlow();
        restartBookieSlow();
        bk.close();
        try {
            bkadmin.openLedger(lh1.getId());
            fail("Shouldn't be able to open with a closed client");
        } catch (BKException.BKClientClosedException cce) {
        // correct behaviour
        }
        try {
            bkadmin.openLedgerNoRecovery(lh1.getId());
            fail("Shouldn't be able to open with a closed client");
        } catch (BKException.BKClientClosedException cce) {
        // correct behaviour
        }
        try {
            bkadmin.recoverBookieData(bookieToKill);
            fail("Shouldn't be able to recover with a closed client");
        } catch (BKException.BKClientClosedException cce) {
        // correct behaviour
        }
        try {
            bkadmin.replicateLedgerFragment(lh3, checkercb.getResult(10, TimeUnit.SECONDS).iterator().next());
            fail("Shouldn't be able to replicate with a closed client");
        } catch (BKException.BKClientClosedException cce) {
        // correct behaviour
        }
    }
}

Example 47 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class BookKeeperDiskSpaceWeightedLedgerPlacementTest method testDiskSpaceWeightedBookieSelectionWithBookiesDying.

/**
 * Test to show that weight based selection honors the disk weight of bookies and also adapts
 * when bookies go away permanently.
 */
@FlakyTest("https://github.com/apache/bookkeeper/issues/503")
public void testDiskSpaceWeightedBookieSelectionWithBookiesDying() throws Exception {
    long freeDiskSpace = 1000000L;
    int multiple = 3;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setDiskWeightBasedPlacementEnabled(true).setGetBookieInfoRetryIntervalSeconds(1, TimeUnit.SECONDS).setBookieMaxWeightMultipleForWeightBasedPlacement(multiple).setZkServers(zkUtil.getZooKeeperConnectString());
    final BookKeeperCheckInfoReader client = new BookKeeperCheckInfoReader(conf);
    for (int i = 0; i < numBookies; i++) {
        // the first 8 bookies have freeDiskSpace of 1MB; While the remaining 2 have 1GB
        if (i < numBookies - 2) {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace);
        } else {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, multiple * freeDiskSpace);
        }
    }
    Map<BookieSocketAddress, Integer> m = new HashMap<BookieSocketAddress, Integer>();
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPasswd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsemble(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // make sure that bookies with higher weight are chosen 3X as often as the median;
    // since the number of ledgers is small (2000), there may be variation
    double ratio1 = (double) m.get(bs.get(numBookies - 2).getLocalAddress()) / (double) m.get(bs.get(0).getLocalAddress());
    assertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
    double ratio2 = (double) m.get(bs.get(numBookies - 1).getLocalAddress()) / (double) m.get(bs.get(1).getLocalAddress());
    assertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    // the remaining bookies should be uniform
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    BookieServer server1 = bs.get(numBookies - 2);
    BookieServer server2 = bs.get(numBookies - 1);
    killBookieAndWaitForZK(numBookies - 1);
    killBookieAndWaitForZK(numBookies - 2);
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPasswd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsemble(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // make sure that bookies with higher weight are chosen 3X as often as the median;
    for (int i = 0; i < numBookies - 3; i++) {
        double delta = Math.abs((double) m.get(bs.get(i).getLocalAddress()) - (double) m.get(bs.get(i + 1).getLocalAddress()));
        delta = (delta * 100) / (double) m.get(bs.get(i + 1).getLocalAddress());
        // the deviation should be less than 30%
        assertTrue("Weigheted placement is not honored: " + delta, delta <= 30);
    }
    // since the following 2 bookies were down, they shouldn't ever be selected
    assertTrue("Weigheted placement is not honored" + m.get(server1.getLocalAddress()), m.get(server1.getLocalAddress()) == 0);
    assertTrue("Weigheted placement is not honored" + m.get(server2.getLocalAddress()), m.get(server2.getLocalAddress()) == 0);
    client.close();
}

Example 48 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class BookKeeperDiskSpaceWeightedLedgerPlacementTest method testDiskSpaceWeightedBookieSelectionWithBookiesBeingAdded.

/**
 * Test to show that weight based selection honors the disk weight of bookies and also adapts
 * when bookies are added.
 */
@FlakyTest("https://github.com/apache/bookkeeper/issues/503")
public void testDiskSpaceWeightedBookieSelectionWithBookiesBeingAdded() throws Exception {
    long freeDiskSpace = 1000000L;
    int multiple = 3;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setDiskWeightBasedPlacementEnabled(true).setGetBookieInfoRetryIntervalSeconds(1, TimeUnit.SECONDS).setBookieMaxWeightMultipleForWeightBasedPlacement(multiple).setZkServers(zkUtil.getZooKeeperConnectString());
    final BookKeeperCheckInfoReader client = new BookKeeperCheckInfoReader(conf);
    for (int i = 0; i < numBookies; i++) {
        // all the bookies have freeDiskSpace of 1MB
        replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace);
    }
    // let the last two bookies be down initially
    ServerConfiguration conf1 = killBookieAndWaitForZK(numBookies - 1);
    ServerConfiguration conf2 = killBookieAndWaitForZK(numBookies - 2);
    Map<BookieSocketAddress, Integer> m = new HashMap<BookieSocketAddress, Integer>();
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPasswd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsemble(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // since the number of ledgers is small (2000), there may be variation
    for (int i = 0; i < numBookies - 3; i++) {
        double delta = Math.abs((double) m.get(bs.get(i).getLocalAddress()) - (double) m.get(bs.get(i + 1).getLocalAddress()));
        delta = (delta * 100) / (double) m.get(bs.get(i + 1).getLocalAddress());
        // the deviation should be less than 30%
        assertTrue("Weigheted placement is not honored: " + delta, delta <= 30);
    }
    // bring up the two dead bookies; they'll also have 3X more free space than the rest of the bookies
    restartBookie(client, conf1, multiple * freeDiskSpace, multiple * freeDiskSpace, null);
    restartBookie(client, conf2, multiple * freeDiskSpace, multiple * freeDiskSpace, null);
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPasswd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsemble(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // since the number of ledgers created is small (2000), we allow a range of 2X to 4X instead of the exact 3X
    for (int i = 0; i < numBookies - 2; i++) {
        double ratio1 = (double) m.get(bs.get(numBookies - 2).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        assertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
        double ratio2 = (double) m.get(bs.get(numBookies - 1).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        assertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    }
    client.close();
}

Example 49 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class BookKeeperDiskSpaceWeightedLedgerPlacementTest method testDiskSpaceWeightedBookieSelectionWithPeriodicBookieInfoUpdate.

/**
 * Tests that the bookie selection is based on the amount of free disk space a bookie has. Also make sure that
 * the periodic bookieInfo read is working and causes the new weights to be taken into account.
 */
@FlakyTest("https://github.com/apache/bookkeeper/issues/503")
public void testDiskSpaceWeightedBookieSelectionWithPeriodicBookieInfoUpdate() throws Exception {
    long freeDiskSpace = 1000000L;
    int multiple = 3;
    int updateIntervalSecs = 6;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setZkServers(zkUtil.getZooKeeperConnectString());
    conf.setDiskWeightBasedPlacementEnabled(true).setGetBookieInfoRetryIntervalSeconds(1, TimeUnit.SECONDS).setBookieMaxWeightMultipleForWeightBasedPlacement(multiple).setGetBookieInfoIntervalSeconds(updateIntervalSecs, TimeUnit.SECONDS);
    final BookKeeperCheckInfoReader client = new BookKeeperCheckInfoReader(conf);
    AtomicBoolean useHigherValue = new AtomicBoolean(false);
    for (int i = 0; i < numBookies; i++) {
        // the start of the test, and 3MB once useHigherValue is set
        if (i < numBookies - 2) {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace);
        } else {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace, multiple * freeDiskSpace, useHigherValue);
        }
    }
    Map<BookieSocketAddress, Integer> m = new HashMap<BookieSocketAddress, Integer>();
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPasswd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsemble(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    for (int i = 0; i < numBookies - 1; i++) {
        double delta = Math.abs((double) m.get(bs.get(i).getLocalAddress()) - (double) m.get(bs.get(i + 1).getLocalAddress()));
        delta = (delta * 100) / (double) m.get(bs.get(i + 1).getLocalAddress());
        // the deviation should be <30%
        assertTrue("Weigheted placement is not honored: " + delta, delta <= 30);
    }
    // Sleep for double the time required to update the bookie infos, and then check each one
    useHigherValue.set(true);
    Thread.sleep(updateIntervalSecs * 1000);
    for (int i = 0; i < numBookies; i++) {
        if (i < numBookies - 2) {
            client.blockUntilBookieWeightIs(bs.get(i).getLocalAddress(), Optional.of(freeDiskSpace));
        } else {
            client.blockUntilBookieWeightIs(bs.get(i).getLocalAddress(), Optional.of(freeDiskSpace * multiple));
        }
    }
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPasswd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsemble(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // since the number of ledgers created is small (2000), we allow a range of 2X to 4X instead of the exact 3X
    for (int i = 0; i < numBookies - 2; i++) {
        double ratio1 = (double) m.get(bs.get(numBookies - 2).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        assertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
        double ratio2 = (double) m.get(bs.get(numBookies - 1).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        assertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    }
    client.close();
}

Example 50 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class BookieRecoveryTest method testBookieRecoveryOnClosedLedgers.

/**
 * Test recoverying the closed ledgers when the failed bookie server is in the last ensemble.
 */
@Test
public void testBookieRecoveryOnClosedLedgers() throws Exception {
    // Create the ledgers
    int numLedgers = 3;
    List<LedgerHandle> lhs = createLedgers(numLedgers, numBookies, 2);
    // Write the entries for the ledgers with dummy values
    int numMsgs = 10;
    writeEntriestoLedgers(numMsgs, 0, lhs);
    closeLedgers(lhs);
    // Shutdown last bookie server in last ensemble
    ArrayList<BookieSocketAddress> lastEnsemble = lhs.get(0).getLedgerMetadata().getEnsembles().entrySet().iterator().next().getValue();
    BookieSocketAddress bookieToKill = lastEnsemble.get(lastEnsemble.size() - 1);
    killBookie(bookieToKill);
    // start a new bookie
    startNewBookie();
    LOG.info("Now recover the data on the killed bookie (" + bookieToKill + ") and replicate it to a random available one");
    bkAdmin.recoverBookieData(bookieToKill);
    for (LedgerHandle lh : lhs) {
        assertTrue("Not fully replicated", verifyFullyReplicated(lh, numMsgs));
        lh.close();
    }
}