Examples with BookieSocketAddress org.apache.bookkeeper.net.BookieSocketAddress used on opensource projects

Example 81 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testWeightedPlacementAndNewEnsembleWithEnoughBookiesInSameRack.

@Test
public void testWeightedPlacementAndNewEnsembleWithEnoughBookiesInSameRack() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.7", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.8", 3181);
    BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.9", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr7.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr8.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr9.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    addrs.add(addr9);
    int maxMultiple = 4;
    conf.setDiskWeightBasedPlacementEnabled(true);
    conf.setBookieMaxWeightMultipleForWeightBasedPlacement(maxMultiple);
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE);
    repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    Map<BookieSocketAddress, BookieInfo> bookieInfoMap = new HashMap<BookieSocketAddress, BookieInfo>();
    bookieInfoMap.put(addr1, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr2, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr3, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr4, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr5, new BookieInfo(1000L, 1000L));
    bookieInfoMap.put(addr6, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr7, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr8, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr9, new BookieInfo(1000L, 1000L));
    repp.updateBookieInfo(bookieInfoMap);
    Map<BookieSocketAddress, Long> selectionCounts = new HashMap<BookieSocketAddress, Long>();
    for (BookieSocketAddress b : addrs) {
        selectionCounts.put(b, 0L);
    }
    int numTries = 10000;
    Set<BookieSocketAddress> excludeList = new HashSet<BookieSocketAddress>();
    ArrayList<BookieSocketAddress> ensemble;
    for (int i = 0; i < numTries; i++) {
        // addr2 is on /r2 and this is the only one on this rack. So the replacement
        // will come from other racks. However, the weight should be honored in such
        // selections as well
        ensemble = repp.newEnsemble(3, 2, 2, null, excludeList);
        assertTrue("Rackaware selection not happening " + getNumCoveredWriteQuorums(ensemble, 2), getNumCoveredWriteQuorums(ensemble, 2) >= 2);
        for (BookieSocketAddress b : ensemble) {
            selectionCounts.put(b, selectionCounts.get(b) + 1);
        }
    }
    // the median weight used is 100 since addr2 and addr6 have the same weight, we use their
    // selection counts as the same as median
    double observedMultiple1 = ((double) selectionCounts.get(addr5) / (double) selectionCounts.get(addr2));
    double observedMultiple2 = ((double) selectionCounts.get(addr9) / (double) selectionCounts.get(addr6));
    assertTrue("Weights not being honored expected 2 observed " + observedMultiple1, Math.abs(observedMultiple1 - maxMultiple) < 0.5);
    assertTrue("Weights not being honored expected 4 observed " + observedMultiple2, Math.abs(observedMultiple2 - maxMultiple) < 0.5);
}

Example 82 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testWeightedPlacementAndReplaceBookieWithEnoughBookiesInSameRack.

@Test
public void testWeightedPlacementAndReplaceBookieWithEnoughBookiesInSameRack() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    int multiple = 10;
    conf.setDiskWeightBasedPlacementEnabled(true);
    // no max cap on weight
    conf.setBookieMaxWeightMultipleForWeightBasedPlacement(-1);
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE);
    repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    Map<BookieSocketAddress, BookieInfo> bookieInfoMap = new HashMap<BookieSocketAddress, BookieInfo>();
    bookieInfoMap.put(addr1, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr2, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr3, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr4, new BookieInfo(multiple * 100L, multiple * 100L));
    repp.updateBookieInfo(bookieInfoMap);
    Map<BookieSocketAddress, Long> selectionCounts = new HashMap<BookieSocketAddress, Long>();
    selectionCounts.put(addr3, 0L);
    selectionCounts.put(addr4, 0L);
    int numTries = 50000;
    BookieSocketAddress replacedBookie;
    for (int i = 0; i < numTries; i++) {
        // replace node under r2
        replacedBookie = repp.replaceBookie(1, 1, 1, null, new HashSet<>(), addr2, new HashSet<>());
        assertTrue("replaced : " + replacedBookie, addr3.equals(replacedBookie) || addr4.equals(replacedBookie));
        selectionCounts.put(replacedBookie, selectionCounts.get(replacedBookie) + 1);
    }
    double observedMultiple = ((double) selectionCounts.get(addr4) / (double) selectionCounts.get(addr3));
    assertTrue("Weights not being honored " + observedMultiple, Math.abs(observedMultiple - multiple) < 1);
}

Example 83 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method getNumCoveredWriteQuorums.

static int getNumCoveredWriteQuorums(ArrayList<BookieSocketAddress> ensemble, int writeQuorumSize) throws Exception {
    int ensembleSize = ensemble.size();
    int numCoveredWriteQuorums = 0;
    for (int i = 0; i < ensembleSize; i++) {
        Set<String> racks = new HashSet<String>();
        for (int j = 0; j < writeQuorumSize; j++) {
            int bookieIdx = (i + j) % ensembleSize;
            BookieSocketAddress addr = ensemble.get(bookieIdx);
            racks.add(StaticDNSResolver.getRack(addr.getHostName()));
        }
        numCoveredWriteQuorums += (racks.size() > 1 ? 1 : 0);
    }
    return numCoveredWriteQuorums;
}

Example 84 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testNewEnsembleWithEnoughRacks.

@Test
public void testNewEnsembleWithEnoughRacks() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.4", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.5", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.7", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.8", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.9", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getHostName(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/default-region/r2");
    StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/default-region/r3");
    StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/default-region/r4");
    StaticDNSResolver.addNodeToRack(addr5.getHostName(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/default-region/r2");
    StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/default-region/r3");
    StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/default-region/r4");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    try {
        ArrayList<BookieSocketAddress> ensemble1 = repp.newEnsemble(3, 2, 2, null, new HashSet<>());
        assertEquals(3, getNumCoveredWriteQuorums(ensemble1, 2));
        ArrayList<BookieSocketAddress> ensemble2 = repp.newEnsemble(4, 2, 2, null, new HashSet<>());
        assertEquals(4, getNumCoveredWriteQuorums(ensemble2, 2));
    } catch (BKNotEnoughBookiesException bnebe) {
        fail("Should not get not enough bookies exception even there is only one rack.");
    }
}

Example 85 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestBookieHealthCheck method testNoQuarantineOnBkRestart.

@Test
public void testNoQuarantineOnBkRestart() throws Exception {
    final LedgerHandle lh = bkc.createLedger(2, 2, 2, BookKeeper.DigestType.CRC32, new byte[] {});
    final int numEntries = 20;
    BookieSocketAddress bookieToRestart = lh.getLedgerMetadata().getEnsemble(0).get(0);
    // we add entries on a separate thread so that we can restart a bookie on the current thread
    Thread addEntryThread = new Thread() {

        public void run() {
            for (int i = 0; i < numEntries; i++) {
                byte[] msg = ("msg-" + i).getBytes();
                try {
                    lh.addEntry(msg);
                    // we add sufficient sleep to make sure all entries are not added before we restart the bookie
                    Thread.sleep(100);
                } catch (Exception e) {
                    LOG.error("Error sending msg");
                }
            }
        }
    };
    addEntryThread.start();
    restartBookie(bookieToRestart);
    // make sure the health check runs once
    Thread.sleep(baseClientConf.getBookieHealthCheckIntervalSeconds() * 2 * 1000);
    // the bookie watcher should not contain the bookieToRestart in the quarantine set
    Assert.assertFalse(bkc.bookieWatcher.quarantinedBookies.asMap().containsKey(bookieToRestart));
}