Examples with BookieInfo org.apache.bookkeeper.client.BookieInfoReader.BookieInfo used on opensource projects

Example 1 with BookieInfo

use of org.apache.bookkeeper.client.BookieInfoReader.BookieInfo in project bookkeeper by apache.

the class BookieClient method getBookieInfo.

public void getBookieInfo(final BookieSocketAddress addr, final long requested, final GetBookieInfoCallback cb, final Object ctx) {
    final PerChannelBookieClientPool client = lookupClient(addr);
    if (client == null) {
        cb.getBookieInfoComplete(getRc(BKException.Code.BookieHandleNotAvailableException), new BookieInfo(), ctx);
        return;
    }
    client.obtain((rc, pcbc) -> {
        if (rc != BKException.Code.OK) {
            try {
                executor.submit(safeRun(() -> {
                    cb.getBookieInfoComplete(rc, new BookieInfo(), ctx);
                }));
            } catch (RejectedExecutionException re) {
                cb.getBookieInfoComplete(getRc(BKException.Code.InterruptedException), new BookieInfo(), ctx);
            }
        } else {
            pcbc.getBookieInfo(requested, cb, ctx);
        }
    }, requested);
}

Example 2 with BookieInfo

use of org.apache.bookkeeper.client.BookieInfoReader.BookieInfo in project bookkeeper by apache.

the class BookieClientTest method testGetBookieInfo.

@Test
public void testGetBookieInfo() throws IOException, InterruptedException {
    BookieSocketAddress addr = bs.getLocalAddress();
    BookieClient bc = new BookieClient(new ClientConfiguration(), new NioEventLoopGroup(), executor, scheduler, NullStatsLogger.INSTANCE);
    long flags = BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE | BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE;
    class CallbackObj {

        int rc;

        long requested;

        long freeDiskSpace, totalDiskCapacity;

        CountDownLatch latch = new CountDownLatch(1);

        CallbackObj(long requested) {
            this.requested = requested;
            this.rc = 0;
            this.freeDiskSpace = 0L;
            this.totalDiskCapacity = 0L;
        }
    }
    CallbackObj obj = new CallbackObj(flags);
    bc.getBookieInfo(addr, flags, new GetBookieInfoCallback() {

        @Override
        public void getBookieInfoComplete(int rc, BookieInfo bInfo, Object ctx) {
            CallbackObj obj = (CallbackObj) ctx;
            obj.rc = rc;
            if (rc == Code.OK) {
                if ((obj.requested & BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE) != 0) {
                    obj.freeDiskSpace = bInfo.getFreeDiskSpace();
                }
                if ((obj.requested & BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE) != 0) {
                    obj.totalDiskCapacity = bInfo.getTotalDiskSpace();
                }
            }
            obj.latch.countDown();
        }
    }, obj);
    obj.latch.await();
    System.out.println("Return code: " + obj.rc + "FreeDiskSpace: " + obj.freeDiskSpace + " TotalCapacity: " + obj.totalDiskCapacity);
    assertTrue("GetBookieInfo failed with error " + obj.rc, obj.rc == Code.OK);
    assertTrue("GetBookieInfo failed with error " + obj.rc, obj.freeDiskSpace <= obj.totalDiskCapacity);
    assertTrue("GetBookieInfo failed with error " + obj.rc, obj.totalDiskCapacity > 0);
}

Example 3 with BookieInfo

use of org.apache.bookkeeper.client.BookieInfoReader.BookieInfo in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testWeightedPlacementAndReplaceBookieWithoutEnoughBookiesInSameRack.

@Test
public void testWeightedPlacementAndReplaceBookieWithoutEnoughBookiesInSameRack() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181);
    // update dns mapping
    StaticDNSResolver.reset();
    StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r4");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    int multiple = 10, maxMultiple = 4;
    conf.setDiskWeightBasedPlacementEnabled(true);
    conf.setBookieMaxWeightMultipleForWeightBasedPlacement(maxMultiple);
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE);
    repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    Map<BookieSocketAddress, BookieInfo> bookieInfoMap = new HashMap<BookieSocketAddress, BookieInfo>();
    bookieInfoMap.put(addr1, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr2, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr3, new BookieInfo(200L, 200L));
    bookieInfoMap.put(addr4, new BookieInfo(multiple * 100L, multiple * 100L));
    repp.updateBookieInfo(bookieInfoMap);
    Map<BookieSocketAddress, Long> selectionCounts = new HashMap<BookieSocketAddress, Long>();
    selectionCounts.put(addr1, 0L);
    selectionCounts.put(addr2, 0L);
    selectionCounts.put(addr3, 0L);
    selectionCounts.put(addr4, 0L);
    int numTries = 50000;
    BookieSocketAddress replacedBookie;
    for (int i = 0; i < numTries; i++) {
        // addr2 is on /r2 and this is the only one on this rack. So the replacement
        // will come from other racks. However, the weight should be honored in such
        // selections as well
        replacedBookie = repp.replaceBookie(1, 1, 1, null, new HashSet<>(), addr2, new HashSet<>());
        assertTrue(addr1.equals(replacedBookie) || addr3.equals(replacedBookie) || addr4.equals(replacedBookie));
        selectionCounts.put(replacedBookie, selectionCounts.get(replacedBookie) + 1);
    }
    double medianWeight = 150;
    double medianSelectionCounts = (double) (medianWeight / bookieInfoMap.get(addr1).getWeight()) * selectionCounts.get(addr1);
    double observedMultiple1 = ((double) selectionCounts.get(addr4) / (double) medianSelectionCounts);
    double observedMultiple2 = ((double) selectionCounts.get(addr4) / (double) selectionCounts.get(addr3));
    LOG.info("oM1 " + observedMultiple1 + " oM2 " + observedMultiple2);
    assertTrue("Weights not being honored expected " + maxMultiple + " observed " + observedMultiple1, Math.abs(observedMultiple1 - maxMultiple) < 1);
    // expected multiple for addr3
    double expected = (medianWeight * maxMultiple) / bookieInfoMap.get(addr3).getWeight();
    assertTrue("Weights not being honored expected " + expected + " observed " + observedMultiple2, Math.abs(observedMultiple2 - expected) < 1);
}

Example 4 with BookieInfo

use of org.apache.bookkeeper.client.BookieInfoReader.BookieInfo in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testWeightedPlacementAndNewEnsembleWithEnoughBookiesInSameRack.

@Test
public void testWeightedPlacementAndNewEnsembleWithEnoughBookiesInSameRack() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.5", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.6", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.7", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.8", 3181);
    BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.9", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr5.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr6.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr7.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr8.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    StaticDNSResolver.addNodeToRack(addr9.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r3");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    addrs.add(addr9);
    int maxMultiple = 4;
    conf.setDiskWeightBasedPlacementEnabled(true);
    conf.setBookieMaxWeightMultipleForWeightBasedPlacement(maxMultiple);
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE);
    repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    Map<BookieSocketAddress, BookieInfo> bookieInfoMap = new HashMap<BookieSocketAddress, BookieInfo>();
    bookieInfoMap.put(addr1, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr2, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr3, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr4, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr5, new BookieInfo(1000L, 1000L));
    bookieInfoMap.put(addr6, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr7, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr8, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr9, new BookieInfo(1000L, 1000L));
    repp.updateBookieInfo(bookieInfoMap);
    Map<BookieSocketAddress, Long> selectionCounts = new HashMap<BookieSocketAddress, Long>();
    for (BookieSocketAddress b : addrs) {
        selectionCounts.put(b, 0L);
    }
    int numTries = 10000;
    Set<BookieSocketAddress> excludeList = new HashSet<BookieSocketAddress>();
    ArrayList<BookieSocketAddress> ensemble;
    for (int i = 0; i < numTries; i++) {
        // addr2 is on /r2 and this is the only one on this rack. So the replacement
        // will come from other racks. However, the weight should be honored in such
        // selections as well
        ensemble = repp.newEnsemble(3, 2, 2, null, excludeList);
        assertTrue("Rackaware selection not happening " + getNumCoveredWriteQuorums(ensemble, 2), getNumCoveredWriteQuorums(ensemble, 2) >= 2);
        for (BookieSocketAddress b : ensemble) {
            selectionCounts.put(b, selectionCounts.get(b) + 1);
        }
    }
    // the median weight used is 100 since addr2 and addr6 have the same weight, we use their
    // selection counts as the same as median
    double observedMultiple1 = ((double) selectionCounts.get(addr5) / (double) selectionCounts.get(addr2));
    double observedMultiple2 = ((double) selectionCounts.get(addr9) / (double) selectionCounts.get(addr6));
    assertTrue("Weights not being honored expected 2 observed " + observedMultiple1, Math.abs(observedMultiple1 - maxMultiple) < 0.5);
    assertTrue("Weights not being honored expected 4 observed " + observedMultiple2, Math.abs(observedMultiple2 - maxMultiple) < 0.5);
}

Example 5 with BookieInfo

use of org.apache.bookkeeper.client.BookieInfoReader.BookieInfo in project bookkeeper by apache.

the class TestRackawareEnsemblePlacementPolicy method testWeightedPlacementAndReplaceBookieWithEnoughBookiesInSameRack.

@Test
public void testWeightedPlacementAndReplaceBookieWithEnoughBookiesInSameRack() throws Exception {
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION_AND_RACK);
    StaticDNSResolver.addNodeToRack(addr2.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr3.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    StaticDNSResolver.addNodeToRack(addr4.getSocketAddress().getAddress().getHostAddress(), NetworkTopology.DEFAULT_REGION + "/r2");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    int multiple = 10;
    conf.setDiskWeightBasedPlacementEnabled(true);
    // no max cap on weight
    conf.setBookieMaxWeightMultipleForWeightBasedPlacement(-1);
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE);
    repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    Map<BookieSocketAddress, BookieInfo> bookieInfoMap = new HashMap<BookieSocketAddress, BookieInfo>();
    bookieInfoMap.put(addr1, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr2, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr3, new BookieInfo(100L, 100L));
    bookieInfoMap.put(addr4, new BookieInfo(multiple * 100L, multiple * 100L));
    repp.updateBookieInfo(bookieInfoMap);
    Map<BookieSocketAddress, Long> selectionCounts = new HashMap<BookieSocketAddress, Long>();
    selectionCounts.put(addr3, 0L);
    selectionCounts.put(addr4, 0L);
    int numTries = 50000;
    BookieSocketAddress replacedBookie;
    for (int i = 0; i < numTries; i++) {
        // replace node under r2
        replacedBookie = repp.replaceBookie(1, 1, 1, null, new HashSet<>(), addr2, new HashSet<>());
        assertTrue("replaced : " + replacedBookie, addr3.equals(replacedBookie) || addr4.equals(replacedBookie));
        selectionCounts.put(replacedBookie, selectionCounts.get(replacedBookie) + 1);
    }
    double observedMultiple = ((double) selectionCounts.get(addr4) / (double) selectionCounts.get(addr3));
    assertTrue("Weights not being honored " + observedMultiple, Math.abs(observedMultiple - multiple) < 1);
}