Examples with OrderedExecutor org.apache.bookkeeper.common.util.OrderedExecutor used on opensource projects

Example 6 with OrderedExecutor

use of org.apache.bookkeeper.common.util.OrderedExecutor in project bookkeeper by apache.

the class BenchBookie method main.

/**
 * @param args
 * @throws InterruptedException
 */
public static void main(String[] args) throws InterruptedException, ParseException, IOException, BKException, KeeperException {
    Options options = new Options();
    options.addOption("host", true, "Hostname or IP of bookie to benchmark");
    options.addOption("port", true, "Port of bookie to benchmark (default 3181)");
    options.addOption("zookeeper", true, "Zookeeper ensemble, (default \"localhost:2181\")");
    options.addOption("size", true, "Size of message to send, in bytes (default 1024)");
    options.addOption("warmupCount", true, "Number of messages in warmup phase (default 999)");
    options.addOption("latencyCount", true, "Number of messages in latency phase (default 5000)");
    options.addOption("throughputCount", true, "Number of messages in throughput phase (default 50000)");
    options.addOption("help", false, "This message");
    CommandLineParser parser = new PosixParser();
    CommandLine cmd = parser.parse(options, args);
    if (cmd.hasOption("help") || !cmd.hasOption("host")) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("BenchBookie <options>", options);
        System.exit(-1);
    }
    String addr = cmd.getOptionValue("host");
    int port = Integer.parseInt(cmd.getOptionValue("port", "3181"));
    int size = Integer.parseInt(cmd.getOptionValue("size", "1024"));
    String servers = cmd.getOptionValue("zookeeper", "localhost:2181");
    int warmUpCount = Integer.parseInt(cmd.getOptionValue("warmupCount", "999"));
    int latencyCount = Integer.parseInt(cmd.getOptionValue("latencyCount", "5000"));
    int throughputCount = Integer.parseInt(cmd.getOptionValue("throughputCount", "50000"));
    EventLoopGroup eventLoop;
    if (SystemUtils.IS_OS_LINUX) {
        try {
            eventLoop = new EpollEventLoopGroup();
        } catch (Throwable t) {
            LOG.warn("Could not use Netty Epoll event loop for benchmark {}", t.getMessage());
            eventLoop = new NioEventLoopGroup();
        }
    } else {
        eventLoop = new NioEventLoopGroup();
    }
    OrderedExecutor executor = OrderedExecutor.newBuilder().name("BenchBookieClientScheduler").numThreads(1).build();
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("BookKeeperClientScheduler"));
    ClientConfiguration conf = new ClientConfiguration();
    BookieClient bc = new BookieClient(conf, eventLoop, executor, scheduler, NullStatsLogger.INSTANCE);
    LatencyCallback lc = new LatencyCallback();
    ThroughputCallback tc = new ThroughputCallback();
    long ledger = getValidLedgerId(servers);
    for (long entry = 0; entry < warmUpCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), tc, null, BookieProtocol.FLAG_NONE);
    }
    LOG.info("Waiting for warmup");
    tc.waitFor(warmUpCount);
    ledger = getValidLedgerId(servers);
    LOG.info("Benchmarking latency");
    long startTime = System.nanoTime();
    for (long entry = 0; entry < latencyCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        lc.resetComplete();
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), lc, null, BookieProtocol.FLAG_NONE);
        lc.waitForComplete();
    }
    long endTime = System.nanoTime();
    LOG.info("Latency: " + (((double) (endTime - startTime)) / ((double) latencyCount)) / 1000000.0);
    ledger = getValidLedgerId(servers);
    LOG.info("Benchmarking throughput");
    startTime = System.currentTimeMillis();
    tc = new ThroughputCallback();
    for (long entry = 0; entry < throughputCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), tc, null, BookieProtocol.FLAG_NONE);
    }
    tc.waitFor(throughputCount);
    endTime = System.currentTimeMillis();
    LOG.info("Throughput: " + ((long) throughputCount) * 1000 / (endTime - startTime));
    bc.close();
    scheduler.shutdown();
    eventLoop.shutdownGracefully();
    executor.shutdown();
}

Example 7 with OrderedExecutor

use of org.apache.bookkeeper.common.util.OrderedExecutor in project bookkeeper by apache.

the class BookieClient method main.

/**
 * @param args
 * @throws IOException
 * @throws NumberFormatException
 * @throws InterruptedException
 */
public static void main(String[] args) throws NumberFormatException, IOException, InterruptedException {
    if (args.length != 3) {
        System.err.println("USAGE: BookieClient bookieHost port ledger#");
        return;
    }
    WriteCallback cb = new WriteCallback() {

        public void writeComplete(int rc, long ledger, long entry, BookieSocketAddress addr, Object ctx) {
            Counter counter = (Counter) ctx;
            counter.dec();
            if (rc != 0) {
                System.out.println("rc = " + rc + " for " + entry + "@" + ledger);
            }
        }
    };
    Counter counter = new Counter();
    byte[] hello = "hello".getBytes(UTF_8);
    long ledger = Long.parseLong(args[2]);
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup(1);
    OrderedExecutor executor = OrderedExecutor.newBuilder().name("BookieClientWorker").numThreads(1).build();
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("BookKeeperClientScheduler"));
    BookieClient bc = new BookieClient(new ClientConfiguration(), eventLoopGroup, executor, scheduler, NullStatsLogger.INSTANCE);
    BookieSocketAddress addr = new BookieSocketAddress(args[0], Integer.parseInt(args[1]));
    for (int i = 0; i < 100000; i++) {
        counter.inc();
        bc.addEntry(addr, ledger, new byte[0], i, ByteBufList.get(Unpooled.wrappedBuffer(hello)), cb, counter, 0);
    }
    counter.wait(0);
    System.out.println("Total = " + counter.total());
    scheduler.shutdown();
    eventLoopGroup.shutdownGracefully();
    executor.shutdown();
}

Example 8 with OrderedExecutor

use of org.apache.bookkeeper.common.util.OrderedExecutor in project bookkeeper by apache.

the class BookieRequestProcessor method processAddRequest.

private void processAddRequest(final BookieProtocol.ParsedAddRequest r, final Channel c) {
    WriteEntryProcessor write = WriteEntryProcessor.create(r, c, this);
    // If it's a high priority add (usually as part of recovery process), we want to make sure it gets
    // executed as fast as possible, so bypass the normal writeThreadPool and execute in highPriorityThreadPool
    final OrderedExecutor threadPool;
    if (r.isHighPriority()) {
        threadPool = highPriorityThreadPool;
    } else {
        threadPool = writeThreadPool;
    }
    if (null == threadPool) {
        write.run();
    } else {
        try {
            threadPool.executeOrdered(r.getLedgerId(), write);
        } catch (RejectedExecutionException e) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Failed to process request to add entry at {}:{}. Too many pending requests", r.ledgerId, r.entryId);
            }
            write.sendResponse(BookieProtocol.ETOOMANYREQUESTS, ResponseBuilder.buildErrorResponse(BookieProtocol.ETOOMANYREQUESTS, r), addRequestStats);
        }
    }
}

Example 9 with OrderedExecutor

use of org.apache.bookkeeper.common.util.OrderedExecutor in project bookkeeper by apache.

the class BookieRequestProcessor method processAddRequestV3.

private void processAddRequestV3(final BookkeeperProtocol.Request r, final Channel c) {
    WriteEntryProcessorV3 write = new WriteEntryProcessorV3(r, c, this);
    final OrderedExecutor threadPool;
    if (RequestUtils.isHighPriority(r)) {
        threadPool = highPriorityThreadPool;
    } else {
        threadPool = writeThreadPool;
    }
    if (null == threadPool) {
        write.run();
    } else {
        try {
            threadPool.executeOrdered(r.getAddRequest().getLedgerId(), write);
        } catch (RejectedExecutionException e) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Failed to process request to add entry at {}:{}. Too many pending requests", r.getAddRequest().getLedgerId(), r.getAddRequest().getEntryId());
            }
            BookkeeperProtocol.AddResponse.Builder addResponse = BookkeeperProtocol.AddResponse.newBuilder().setLedgerId(r.getAddRequest().getLedgerId()).setEntryId(r.getAddRequest().getEntryId()).setStatus(BookkeeperProtocol.StatusCode.ETOOMANYREQUESTS);
            BookkeeperProtocol.Response.Builder response = BookkeeperProtocol.Response.newBuilder().setHeader(write.getHeader()).setStatus(addResponse.getStatus()).setAddResponse(addResponse);
            BookkeeperProtocol.Response resp = response.build();
            write.sendResponse(addResponse.getStatus(), resp, addRequestStats);
        }
    }
}

Example 10 with OrderedExecutor

use of org.apache.bookkeeper.common.util.OrderedExecutor in project bookkeeper by apache.

the class BookieRequestProcessor method processReadRequest.

private void processReadRequest(final BookieProtocol.ReadRequest r, final Channel c) {
    ReadEntryProcessor read = ReadEntryProcessor.create(r, c, this);
    // If it's a high priority read (fencing or as part of recovery process), we want to make sure it
    // gets executed as fast as possible, so bypass the normal readThreadPool
    // and execute in highPriorityThreadPool
    final OrderedExecutor threadPool;
    if (r.isHighPriority() || r.isFencing()) {
        threadPool = highPriorityThreadPool;
    } else {
        threadPool = readThreadPool;
    }
    if (null == threadPool) {
        read.run();
    } else {
        try {
            threadPool.executeOrdered(r.getLedgerId(), read);
        } catch (RejectedExecutionException e) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Failed to process request to read entry at {}:{}. Too many pending requests", r.ledgerId, r.entryId);
            }
            read.sendResponse(BookieProtocol.ETOOMANYREQUESTS, ResponseBuilder.buildErrorResponse(BookieProtocol.ETOOMANYREQUESTS, r), readRequestStats);
        }
    }
}