Examples with CallableTool io.datarouter.util.concurrent.CallableTool used on opensource projects

Example 1 with CallableTool

use of io.datarouter.util.concurrent.CallableTool in project datarouter by hotpads.

the class LoadTestGetHandler method get.

@Handler(defaultHandler = true)
private Mav get(@Param(P_num) OptionalString num, @Param(P_max) OptionalString max, @Param(P_numThreads) OptionalString numThreads, @Param(P_batchSize) OptionalString batchSize, @Param(P_logPeriod) OptionalString logPeriod, @Param(P_submitAction) OptionalString submitAction) {
    var form = new HtmlForm().withMethod("post");
    form.addTextField().withDisplay("Num").withName(P_num).withPlaceholder("100,000").withValue(num.orElse(null));
    form.addTextField().withDisplay("Max").withName(P_max).withPlaceholder("10").withValue(max.orElse(null));
    form.addTextField().withDisplay("Num Threads").withName(P_numThreads).withPlaceholder("10").withValue(numThreads.orElse(null));
    form.addTextField().withDisplay("Batch Size").withName(P_batchSize).withPlaceholder("100").withValue(batchSize.orElse(null));
    form.addTextField().withDisplay("Log Period").withName(P_logPeriod).withPlaceholder("1,0000").withValue(logPeriod.orElse(null));
    form.addButton().withDisplay("Run Get").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Load Test - Get").withContent(Html.makeContent(form)).buildMav();
    }
    PhaseTimer timer = new PhaseTimer("get");
    // params
    int pNum = num.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_NUM);
    int pMax = max.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(pNum);
    int pNumThreads = numThreads.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_NUM_THREADS);
    int pBatchSize = batchSize.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    int pLogPeriod = logPeriod.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_LOG_PERIOD);
    // tracking
    AtomicInteger rowCounter = new AtomicInteger(0);
    AtomicLong lastBatchFinished = new AtomicLong(System.nanoTime());
    // execute
    int numBatches = LoadTestTool.numBatches(pNum, pBatchSize);
    ExecutorService executor = Executors.newFixedThreadPool(pNumThreads);
    Scanner.of(IntStream.range(0, numBatches).mapToObj(Integer::valueOf)).map(batchId -> LoadTestTool.makeRandomIdBatch(pNum, pMax, pBatchSize, batchId)).map(ids -> new GetBatchCallable(dao.getReaderNode(), ids, pLogPeriod, rowCounter, lastBatchFinished)).parallel(new ParallelScannerContext(executor, pNumThreads, true)).forEach(CallableTool::callUnchecked);
    ExecutorServiceTool.shutdown(executor, Duration.ofSeconds(5));
    timer.add("got " + rowCounter.get());
    var message = div(h2("Load Test Get Results"), div(h3("Results"), dl(dt("Total Time"), dd(timer.getElapsedString()), dt("Rows per second"), dd(timer.getItemsPerSecond(rowCounter.get()) + ""))), div(h3("Params"), dl(dt("Num"), dd(pNum + ""), dt("Max"), dd(pMax + ""), dt("Num Threads"), dd(pNumThreads + ""), dt("Batch Size"), dd(pBatchSize + ""), dt("Log Period"), dd(pLogPeriod + "")))).withClass("container");
    logger.warn("total={}, rps={}, num={}, max={}, numThreads={} batchSize={}, logPeriod={}", timer.getElapsedString(), timer.getItemsPerSecond(rowCounter.get()), pNum, pMax, pNumThreads, pBatchSize, pLogPeriod);
    return pageFactory.message(request, message);
}

Example 2 with CallableTool

use of io.datarouter.util.concurrent.CallableTool in project datarouter by hotpads.

the class LoadTestInsertHandler method insert.

@Handler(defaultHandler = true)
private Mav insert(@Param(P_num) OptionalString num, @Param(P_numThreads) OptionalString numThreads, @Param(P_batchSize) OptionalString batchSize, @Param(P_logPeriod) OptionalString logPeriod, @Param(P_persistentPut) OptionalString persistentPut, @Param(P_submitAction) OptionalString submitAction) {
    var form = new HtmlForm().withMethod("post");
    form.addTextField().withDisplay("Num").withName(P_num).withPlaceholder("100,000").withValue(num.orElse(null));
    form.addTextField().withDisplay("Num Threads").withName(P_numThreads).withPlaceholder("10").withValue(numThreads.orElse(null));
    form.addTextField().withDisplay("Batch Size").withName(P_batchSize).withPlaceholder("100").withValue(batchSize.orElse(null));
    form.addTextField().withDisplay("Log Period").withName(P_logPeriod).withPlaceholder("10,000").withValue(logPeriod.orElse(null));
    form.addTextField().withDisplay("Persistent Put").withName(P_persistentPut).withPlaceholder("true").withValue(persistentPut.orElse(null));
    form.addButton().withDisplay("Run Insert").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Load Test - Insert").withContent(Html.makeContent(form)).buildMav();
    }
    // params
    int pNum = num.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_NUM);
    int pNumThreads = numThreads.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_NUM_THREADS);
    int pBatchSize = batchSize.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    int pLogPeriod = logPeriod.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_LOG_PERIOD);
    boolean pPersistentPut = persistentPut.map(StringTool::nullIfEmpty).map(Boolean::valueOf).orElse(DEFAULT_PERSISTENT_PUT);
    PhaseTimer timer = new PhaseTimer("insert");
    // tracking
    AtomicInteger counter = new AtomicInteger(0);
    AtomicLong lastBatchFinished = new AtomicLong(System.nanoTime());
    // execute
    int numBatches = LoadTestTool.numBatches(pNum, pBatchSize);
    ExecutorService executor = Executors.newFixedThreadPool(pNumThreads);
    Scanner.of(IntStream.range(0, numBatches).mapToObj(Integer::valueOf)).map(batchId -> LoadTestTool.makePredictableIdBatch(pNum, pBatchSize, batchId)).map(ids -> new InsertBatchCallable(dao.getWriterNode(), ids, pPersistentPut, pLogPeriod, lastBatchFinished, counter)).parallel(new ParallelScannerContext(executor, pNumThreads, true)).forEach(CallableTool::callUnchecked);
    ExecutorServiceTool.shutdown(executor, Duration.ofSeconds(5));
    timer.add("inserted " + counter.get());
    // results
    DomContent message = div(h2("Load Test Insert Results"), div(h3("Results"), dl(dt("Total Time"), dd(timer.getElapsedString()), dt("Rows per second"), dd(timer.getItemsPerSecond(counter.get()) + ""))), div(h3("Params"), dl(dt("Num"), dd(pNum + ""), dt("Num Threads"), dd(pNumThreads + ""), dt("Batch Size"), dd(pBatchSize + ""), dt("Log Period"), dd(pLogPeriod + ""), dt("Persistent Put"), dd(pPersistentPut + "")))).withClass("container");
    logger.warn("total={}, rps={}, num={}, numThreads={} batchSize={}, logPeriod={}, persistentPut={}", timer.getElapsedString(), timer.getItemsPerSecond(counter.get()), pNum, pNumThreads, pBatchSize, pLogPeriod, pPersistentPut);
    return pageFactory.message(request, message);
}