Examples with BookieSocketAddress org.apache.bookkeeper.net.BookieSocketAddress used on opensource projects

Example 71 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRegionAwareEnsemblePlacementPolicy method testEnsembleDurabilityDisabledInternal.

public void testEnsembleDurabilityDisabledInternal(int minDurability, boolean disableDurability) throws Exception {
    repp.uninitalize();
    repp = new RegionAwareEnsemblePlacementPolicy();
    conf.setProperty(REPP_REGIONS_TO_WRITE, "region1;region2;region3");
    conf.setProperty(REPP_MINIMUM_REGIONS_FOR_DURABILITY, minDurability);
    FeatureProvider featureProvider = new SettableFeatureProvider("", 0);
    if (minDurability <= 1) {
        conf.setProperty(REPP_ENABLE_DURABILITY_ENFORCEMENT_IN_REPLACE, false);
    } else {
        conf.setProperty(REPP_ENABLE_DURABILITY_ENFORCEMENT_IN_REPLACE, true);
    }
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, featureProvider, NullStatsLogger.INSTANCE);
    BookieSocketAddress addr1 = new BookieSocketAddress("127.1.0.2", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.1.0.3", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.1.0.4", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.1.0.5", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.1.0.6", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.1.0.7", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.1.0.8", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.1.0.9", 3181);
    BookieSocketAddress addr9 = new BookieSocketAddress("127.1.0.10", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/r1");
    StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/r2");
    StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region1/r3");
    StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region1/r4");
    StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/region1/r11");
    StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/region1/r12");
    StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/region1/r13");
    StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/region1/r14");
    StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/region1/r23");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    addrs.add(addr9);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    if (disableDurability) {
        ((SettableFeature) featureProvider.getFeature(BookKeeperConstants.FEATURE_REPP_DISABLE_DURABILITY_ENFORCEMENT)).set(true);
    }
    ArrayList<BookieSocketAddress> ensemble;
    try {
        ensemble = repp.newEnsemble(6, 6, 4, null, new HashSet<BookieSocketAddress>());
        assert (ensemble.size() == 6);
    } catch (BKNotEnoughBookiesException bnebe) {
        LOG.error("BKNotEnoughBookiesException", bnebe);
        fail("Should not get not enough bookies exception even there is only one rack.");
        throw bnebe;
    }
    Set<BookieSocketAddress> excludedAddrs = new HashSet<BookieSocketAddress>();
    try {
        repp.replaceBookie(6, 6, 4, null, new HashSet<>(ensemble), addr4, excludedAddrs);
    } catch (BKNotEnoughBookiesException bnebe) {
        fail("Should not get not enough bookies exception even there is only one rack.");
    }
}

Example 72 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRegionAwareEnsemblePlacementPolicy method getNumCoveredRegionsInWriteQuorum.

private int getNumCoveredRegionsInWriteQuorum(ArrayList<BookieSocketAddress> ensemble, int writeQuorumSize) throws Exception {
    int ensembleSize = ensemble.size();
    int numCoveredWriteQuorums = 0;
    for (int i = 0; i < ensembleSize; i++) {
        Set<String> regions = new HashSet<String>();
        for (int j = 0; j < writeQuorumSize; j++) {
            int bookieIdx = (i + j) % ensembleSize;
            BookieSocketAddress addr = ensemble.get(bookieIdx);
            regions.add(StaticDNSResolver.getRegion(addr.getHostName()));
        }
        numCoveredWriteQuorums += (regions.size() > 1 ? 1 : 0);
    }
    return numCoveredWriteQuorums;
}

Example 73 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestLedgerFragmentReplication method testReplicateLFFailsOnlyOnLastUnClosedFragments.

/**
 * Tests that fragment re-replication fails on last unclosed ledger
 * fragments.
 */
@Test
public void testReplicateLFFailsOnlyOnLastUnClosedFragments() throws Exception {
    byte[] data = "TestLedgerFragmentReplication".getBytes();
    LedgerHandle lh = bkc.createLedger(3, 3, TEST_DIGEST_TYPE, TEST_PSSWD);
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill = lh.getLedgerMetadata().getEnsembles().get(0L).get(0);
    startNewBookie();
    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);
    // Lets reform ensemble
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill2 = lh.getLedgerMetadata().getEnsembles().get(0L).get(1);
    BookieSocketAddress newBkAddr = startNewBookieAndReturnAddress();
    LOG.info("New Bookie addr : {}", newBkAddr);
    LOG.info("Killing Bookie", replicaToKill2);
    killBookie(replicaToKill2);
    Set<LedgerFragment> result = getFragmentsToReplicate(lh);
    BookKeeperAdmin admin = new BookKeeperAdmin(baseClientConf);
    // 0-9 entries should be copy to new bookie
    int unclosedCount = 0;
    for (LedgerFragment lf : result) {
        if (lf.isClosed()) {
            admin.replicateLedgerFragment(lh, lf);
        } else {
            unclosedCount++;
            try {
                admin.replicateLedgerFragment(lh, lf);
                fail("Shouldn't be able to rereplicate unclosed ledger");
            } catch (BKException bke) {
            // correct behaviour
            }
        }
    }
    assertEquals("Should be only one unclosed fragment", 1, unclosedCount);
}

Example 74 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestLedgerFragmentReplication method testReplicateLFShouldReturnFalseIfTheReplicationFails.

/**
 * Tests that ReplicateLedgerFragment should return false if replication
 * fails.
 */
@Test
public void testReplicateLFShouldReturnFalseIfTheReplicationFails() throws Exception {
    byte[] data = "TestLedgerFragmentReplication".getBytes();
    LedgerHandle lh = bkc.createLedger(2, 1, TEST_DIGEST_TYPE, TEST_PSSWD);
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    // Kill the first Bookie
    BookieSocketAddress replicaToKill = lh.getLedgerMetadata().getEnsembles().get(0L).get(0);
    killBookie(replicaToKill);
    LOG.info("Killed Bookie =" + replicaToKill);
    // Write some more entries
    for (int i = 0; i < 10; i++) {
        lh.addEntry(data);
    }
    // Kill the second Bookie
    replicaToKill = lh.getLedgerMetadata().getEnsembles().get(0L).get(0);
    killBookie(replicaToKill);
    LOG.info("Killed Bookie =" + replicaToKill);
    Set<LedgerFragment> fragments = getFragmentsToReplicate(lh);
    BookKeeperAdmin admin = new BookKeeperAdmin(baseClientConf);
    for (LedgerFragment lf : fragments) {
        try {
            admin.replicateLedgerFragment(lh, lf);
        } catch (BKException.BKLedgerRecoveryException e) {
        // expected
        }
    }
}

Example 75 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testWeightedPlacementAndReplaceBookieWithoutEnoughBookiesInSameRack.

@Test
public void testWeightedPlacementAndReplaceBookieWithoutEnoughBookiesInSameRack() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181);
    // update dns mapping
    StaticDNSResolver.reset();
    StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r4");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    int multiple = 10, maxMultiple = 4;
    conf.setDiskWeightBasedPlacementEnabled(true);
    conf.setBookieMaxWeightMultipleForWeightBasedPlacement(maxMultiple);
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE);
    repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    Map<BookieSocketAddress, BookieInfo> bookieInfoMap = new HashMap<BookieSocketAddress, BookieInfo>();
    bookieInfoMap.put(addr1, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr2, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr3, new BookieInfo(200L, 200L));
    bookieInfoMap.put(addr4, new BookieInfo(multiple * 100L, multiple * 100L));
    repp.updateBookieInfo(bookieInfoMap);
    Map<BookieSocketAddress, Long> selectionCounts = new HashMap<BookieSocketAddress, Long>();
    selectionCounts.put(addr1, 0L);
    selectionCounts.put(addr2, 0L);
    selectionCounts.put(addr3, 0L);
    selectionCounts.put(addr4, 0L);
    int numTries = 50000;
    BookieSocketAddress replacedBookie;
    for (int i = 0; i < numTries; i++) {
        // addr2 is on /r2 and this is the only one on this rack. So the replacement
        // will come from other racks. However, the weight should be honored in such
        // selections as well
        replacedBookie = repp.replaceBookie(1, 1, 1, null, new HashSet<>(), addr2, new HashSet<>());
        assertTrue(addr1.equals(replacedBookie) || addr3.equals(replacedBookie) || addr4.equals(replacedBookie));
        selectionCounts.put(replacedBookie, selectionCounts.get(replacedBookie) + 1);
    }
    double medianWeight = 150;
    double medianSelectionCounts = (double) (medianWeight / bookieInfoMap.get(addr1).getWeight()) * selectionCounts.get(addr1);
    double observedMultiple1 = ((double) selectionCounts.get(addr4) / (double) medianSelectionCounts);
    double observedMultiple2 = ((double) selectionCounts.get(addr4) / (double) selectionCounts.get(addr3));
    LOG.info("oM1 " + observedMultiple1 + " oM2 " + observedMultiple2);
    assertTrue("Weights not being honored expected " + maxMultiple + " observed " + observedMultiple1, Math.abs(observedMultiple1 - maxMultiple) < 1);
    // expected multiple for addr3
    double expected = (medianWeight * maxMultiple) / bookieInfoMap.get(addr3).getWeight();
    assertTrue("Weights not being honored expected " + expected + " observed " + observedMultiple2, Math.abs(observedMultiple2 - expected) < 1);
}