Examples with BookieClient org.apache.bookkeeper.proto.BookieClient used on opensource projects

Example 1 with BookieClient

use of org.apache.bookkeeper.proto.BookieClient in project bookkeeper by apache.

the class BookieClientTest method testWriteGaps.

@Test
public void testWriteGaps() throws Exception {
    final Object notifyObject = new Object();
    byte[] passwd = new byte[20];
    Arrays.fill(passwd, (byte) 'a');
    BookieSocketAddress addr = bs.getLocalAddress();
    ResultStruct arc = new ResultStruct();
    BookieClient bc = new BookieClient(new ClientConfiguration(), eventLoopGroup, executor, scheduler, NullStatsLogger.INSTANCE);
    ByteBufList bb = createByteBuffer(1, 1, 1);
    bc.addEntry(addr, 1, passwd, 1, bb, wrcb, arc, BookieProtocol.FLAG_NONE);
    synchronized (arc) {
        arc.wait(1000);
        assertEquals(0, arc.rc);
        bc.readEntry(addr, 1, 1, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(1, arc.entry.getInt());
    }
    bb = createByteBuffer(2, 1, 2);
    bc.addEntry(addr, 1, passwd, 2, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    bb = createByteBuffer(3, 1, 3);
    bc.addEntry(addr, 1, passwd, 3, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    bb = createByteBuffer(5, 1, 5);
    bc.addEntry(addr, 1, passwd, 5, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    bb = createByteBuffer(7, 1, 7);
    bc.addEntry(addr, 1, passwd, 7, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    synchronized (notifyObject) {
        bb = createByteBuffer(11, 1, 11);
        bc.addEntry(addr, 1, passwd, 11, bb, wrcb, notifyObject, BookieProtocol.FLAG_NONE);
        notifyObject.wait();
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 6, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 7, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(7, arc.entry.getInt(), BookieProtocol.FLAG_NONE);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 1, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(1, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 2, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(2, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 3, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(3, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 4, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 11, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(11, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 5, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(5, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 10, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 12, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 13, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
}

Example 2 with BookieClient

use of org.apache.bookkeeper.proto.BookieClient in project bookkeeper by apache.

the class BookieClientTest method testGetBookieInfo.

@Test
public void testGetBookieInfo() throws IOException, InterruptedException {
    BookieSocketAddress addr = bs.getLocalAddress();
    BookieClient bc = new BookieClient(new ClientConfiguration(), new NioEventLoopGroup(), executor, scheduler, NullStatsLogger.INSTANCE);
    long flags = BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE | BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE;
    class CallbackObj {

        int rc;

        long requested;

        long freeDiskSpace, totalDiskCapacity;

        CountDownLatch latch = new CountDownLatch(1);

        CallbackObj(long requested) {
            this.requested = requested;
            this.rc = 0;
            this.freeDiskSpace = 0L;
            this.totalDiskCapacity = 0L;
        }
    }
    CallbackObj obj = new CallbackObj(flags);
    bc.getBookieInfo(addr, flags, new GetBookieInfoCallback() {

        @Override
        public void getBookieInfoComplete(int rc, BookieInfo bInfo, Object ctx) {
            CallbackObj obj = (CallbackObj) ctx;
            obj.rc = rc;
            if (rc == Code.OK) {
                if ((obj.requested & BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE) != 0) {
                    obj.freeDiskSpace = bInfo.getFreeDiskSpace();
                }
                if ((obj.requested & BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE) != 0) {
                    obj.totalDiskCapacity = bInfo.getTotalDiskSpace();
                }
            }
            obj.latch.countDown();
        }
    }, obj);
    obj.latch.await();
    System.out.println("Return code: " + obj.rc + "FreeDiskSpace: " + obj.freeDiskSpace + " TotalCapacity: " + obj.totalDiskCapacity);
    assertTrue("GetBookieInfo failed with error " + obj.rc, obj.rc == Code.OK);
    assertTrue("GetBookieInfo failed with error " + obj.rc, obj.freeDiskSpace <= obj.totalDiskCapacity);
    assertTrue("GetBookieInfo failed with error " + obj.rc, obj.totalDiskCapacity > 0);
}

Example 3 with BookieClient

use of org.apache.bookkeeper.proto.BookieClient in project bookkeeper by apache.

the class MockBookKeeperTestCase method setup.

@Before
public void setup() throws Exception {
    mockLedgerMetadataRegistry = new ConcurrentHashMap<>();
    mockLedgerData = new ConcurrentHashMap<>();
    mockNextLedgerId = new AtomicLong(1);
    fencedLedgers = new ConcurrentSkipListSet<>();
    scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(4).name("bk-test").build();
    executor = OrderedExecutor.newBuilder().build();
    bookieWatcher = mock(BookieWatcher.class);
    bookieClient = mock(BookieClient.class);
    ledgerManager = mock(LedgerManager.class);
    ledgerIdGenerator = mock(LedgerIdGenerator.class);
    bk = mock(BookKeeper.class);
    NullStatsLogger nullStatsLogger = setupLoggers();
    failedBookies = new ArrayList<>();
    availableBookies = new HashSet<>();
    when(bk.getCloseLock()).thenReturn(new ReentrantReadWriteLock());
    when(bk.isClosed()).thenReturn(false);
    when(bk.getBookieWatcher()).thenReturn(bookieWatcher);
    when(bk.getDisableEnsembleChangeFeature()).thenReturn(mock(Feature.class));
    when(bk.getExplicitLacInterval()).thenReturn(0);
    when(bk.getMainWorkerPool()).thenReturn(executor);
    when(bk.getBookieClient()).thenReturn(bookieClient);
    when(bk.getScheduler()).thenReturn(scheduler);
    when(bk.getReadSpeculativeRequestPolicy()).thenReturn(Optional.absent());
    mockBookKeeperGetConf(new ClientConfiguration());
    when(bk.getStatsLogger()).thenReturn(nullStatsLogger);
    when(bk.getLedgerManager()).thenReturn(ledgerManager);
    when(bk.getLedgerIdGenerator()).thenReturn(ledgerIdGenerator);
    when(bk.getReturnRc(anyInt())).thenAnswer(invocationOnMock -> invocationOnMock.getArgument(0));
    setupLedgerIdGenerator();
    setupCreateLedgerMetadata();
    setupReadLedgerMetadata();
    setupWriteLedgerMetadata();
    setupRemoveLedgerMetadata();
    setupRegisterLedgerMetadataListener();
    setupBookieWatcherForNewEnsemble();
    setupBookieWatcherForEnsembleChange();
    setupBookieClientReadEntry();
    setupBookieClientAddEntry();
}

Example 4 with BookieClient

use of org.apache.bookkeeper.proto.BookieClient in project bookkeeper by apache.

the class BenchBookie method main.

/**
 * @param args
 * @throws InterruptedException
 */
public static void main(String[] args) throws InterruptedException, ParseException, IOException, BKException, KeeperException {
    Options options = new Options();
    options.addOption("host", true, "Hostname or IP of bookie to benchmark");
    options.addOption("port", true, "Port of bookie to benchmark (default 3181)");
    options.addOption("zookeeper", true, "Zookeeper ensemble, (default \"localhost:2181\")");
    options.addOption("size", true, "Size of message to send, in bytes (default 1024)");
    options.addOption("warmupCount", true, "Number of messages in warmup phase (default 999)");
    options.addOption("latencyCount", true, "Number of messages in latency phase (default 5000)");
    options.addOption("throughputCount", true, "Number of messages in throughput phase (default 50000)");
    options.addOption("help", false, "This message");
    CommandLineParser parser = new PosixParser();
    CommandLine cmd = parser.parse(options, args);
    if (cmd.hasOption("help") || !cmd.hasOption("host")) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("BenchBookie <options>", options);
        System.exit(-1);
    }
    String addr = cmd.getOptionValue("host");
    int port = Integer.parseInt(cmd.getOptionValue("port", "3181"));
    int size = Integer.parseInt(cmd.getOptionValue("size", "1024"));
    String servers = cmd.getOptionValue("zookeeper", "localhost:2181");
    int warmUpCount = Integer.parseInt(cmd.getOptionValue("warmupCount", "999"));
    int latencyCount = Integer.parseInt(cmd.getOptionValue("latencyCount", "5000"));
    int throughputCount = Integer.parseInt(cmd.getOptionValue("throughputCount", "50000"));
    EventLoopGroup eventLoop;
    if (SystemUtils.IS_OS_LINUX) {
        try {
            eventLoop = new EpollEventLoopGroup();
        } catch (Throwable t) {
            LOG.warn("Could not use Netty Epoll event loop for benchmark {}", t.getMessage());
            eventLoop = new NioEventLoopGroup();
        }
    } else {
        eventLoop = new NioEventLoopGroup();
    }
    OrderedExecutor executor = OrderedExecutor.newBuilder().name("BenchBookieClientScheduler").numThreads(1).build();
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("BookKeeperClientScheduler"));
    ClientConfiguration conf = new ClientConfiguration();
    BookieClient bc = new BookieClient(conf, eventLoop, executor, scheduler, NullStatsLogger.INSTANCE);
    LatencyCallback lc = new LatencyCallback();
    ThroughputCallback tc = new ThroughputCallback();
    long ledger = getValidLedgerId(servers);
    for (long entry = 0; entry < warmUpCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), tc, null, BookieProtocol.FLAG_NONE);
    }
    LOG.info("Waiting for warmup");
    tc.waitFor(warmUpCount);
    ledger = getValidLedgerId(servers);
    LOG.info("Benchmarking latency");
    long startTime = System.nanoTime();
    for (long entry = 0; entry < latencyCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        lc.resetComplete();
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), lc, null, BookieProtocol.FLAG_NONE);
        lc.waitForComplete();
    }
    long endTime = System.nanoTime();
    LOG.info("Latency: " + (((double) (endTime - startTime)) / ((double) latencyCount)) / 1000000.0);
    ledger = getValidLedgerId(servers);
    LOG.info("Benchmarking throughput");
    startTime = System.currentTimeMillis();
    tc = new ThroughputCallback();
    for (long entry = 0; entry < throughputCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), tc, null, BookieProtocol.FLAG_NONE);
    }
    tc.waitFor(throughputCount);
    endTime = System.currentTimeMillis();
    LOG.info("Throughput: " + ((long) throughputCount) * 1000 / (endTime - startTime));
    bc.close();
    scheduler.shutdown();
    eventLoop.shutdownGracefully();
    executor.shutdown();
}

Example 5 with BookieClient

use of org.apache.bookkeeper.proto.BookieClient in project bookkeeper by apache.

the class BookieInfoReader method getReadWriteBookieInfo.

/**
 * Performs scan described by instanceState using the cached bookie information
 * in bookieInfoMap.
 */
synchronized void getReadWriteBookieInfo() {
    State queuedType = instanceState.getAndClearQueuedType();
    Collection<BookieSocketAddress> toScan;
    if (queuedType == State.FULL) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Doing full scan");
        }
        toScan = bookieInfoMap.getFullScanTargets();
    } else if (queuedType == State.PARTIAL) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Doing partial scan");
        }
        toScan = bookieInfoMap.getPartialScanTargets();
    } else {
        if (LOG.isErrorEnabled()) {
            LOG.error("Invalid state, queuedType cannot be UNQUEUED in getReadWriteBookieInfo");
        }
        assert (queuedType != State.UNQUEUED);
        return;
    }
    BookieClient bkc = bk.getBookieClient();
    final long requested = BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE | BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE;
    totalSent = 0;
    completedCnt = 0;
    errorCnt = 0;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Getting bookie info for: {}", toScan);
    }
    for (BookieSocketAddress b : toScan) {
        bkc.getBookieInfo(b, requested, new GetBookieInfoCallback() {

            void processReadInfoComplete(int rc, BookieInfo bInfo, Object ctx) {
                synchronized (BookieInfoReader.this) {
                    BookieSocketAddress b = (BookieSocketAddress) ctx;
                    if (rc != BKException.Code.OK) {
                        if (LOG.isErrorEnabled()) {
                            LOG.error("Reading bookie info from bookie {} failed due to {}", b, BKException.codeLogger(rc));
                        }
                        // We reread bookies missing from the map each time, so remove to ensure
                        // we get to it on the next scan
                        bookieInfoMap.clearInfo(b);
                        errorCnt++;
                    } else {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Bookie Info for bookie {} is {}", b, bInfo);
                        }
                        bookieInfoMap.gotInfo(b, bInfo);
                    }
                    completedCnt++;
                    if (totalSent == completedCnt) {
                        onExit();
                    }
                }
            }

            @Override
            public void getBookieInfoComplete(final int rc, final BookieInfo bInfo, final Object ctx) {
                scheduler.submit(new Runnable() {

                    @Override
                    public void run() {
                        processReadInfoComplete(rc, bInfo, ctx);
                    }
                });
            }
        }, b);
        totalSent++;
    }
    if (totalSent == 0) {
        onExit();
    }
}