Examples with StringTool io.datarouter.util.string.StringTool used on opensource projects

Example 1 with StringTool

use of io.datarouter.util.string.StringTool in project datarouter by hotpads.

the class LoadTestGetHandler method get.

@Handler(defaultHandler = true)
private Mav get(@Param(P_num) OptionalString num, @Param(P_max) OptionalString max, @Param(P_numThreads) OptionalString numThreads, @Param(P_batchSize) OptionalString batchSize, @Param(P_logPeriod) OptionalString logPeriod, @Param(P_submitAction) OptionalString submitAction) {
    var form = new HtmlForm().withMethod("post");
    form.addTextField().withDisplay("Num").withName(P_num).withPlaceholder("100,000").withValue(num.orElse(null));
    form.addTextField().withDisplay("Max").withName(P_max).withPlaceholder("10").withValue(max.orElse(null));
    form.addTextField().withDisplay("Num Threads").withName(P_numThreads).withPlaceholder("10").withValue(numThreads.orElse(null));
    form.addTextField().withDisplay("Batch Size").withName(P_batchSize).withPlaceholder("100").withValue(batchSize.orElse(null));
    form.addTextField().withDisplay("Log Period").withName(P_logPeriod).withPlaceholder("1,0000").withValue(logPeriod.orElse(null));
    form.addButton().withDisplay("Run Get").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Load Test - Get").withContent(Html.makeContent(form)).buildMav();
    }
    PhaseTimer timer = new PhaseTimer("get");
    // params
    int pNum = num.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_NUM);
    int pMax = max.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(pNum);
    int pNumThreads = numThreads.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_NUM_THREADS);
    int pBatchSize = batchSize.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    int pLogPeriod = logPeriod.map(StringTool::nullIfEmpty).map(number -> number.replaceAll(",", "")).map(Integer::valueOf).orElse(DEFAULT_LOG_PERIOD);
    // tracking
    AtomicInteger rowCounter = new AtomicInteger(0);
    AtomicLong lastBatchFinished = new AtomicLong(System.nanoTime());
    // execute
    int numBatches = LoadTestTool.numBatches(pNum, pBatchSize);
    ExecutorService executor = Executors.newFixedThreadPool(pNumThreads);
    Scanner.of(IntStream.range(0, numBatches).mapToObj(Integer::valueOf)).map(batchId -> LoadTestTool.makeRandomIdBatch(pNum, pMax, pBatchSize, batchId)).map(ids -> new GetBatchCallable(dao.getReaderNode(), ids, pLogPeriod, rowCounter, lastBatchFinished)).parallel(new ParallelScannerContext(executor, pNumThreads, true)).forEach(CallableTool::callUnchecked);
    ExecutorServiceTool.shutdown(executor, Duration.ofSeconds(5));
    timer.add("got " + rowCounter.get());
    var message = div(h2("Load Test Get Results"), div(h3("Results"), dl(dt("Total Time"), dd(timer.getElapsedString()), dt("Rows per second"), dd(timer.getItemsPerSecond(rowCounter.get()) + ""))), div(h3("Params"), dl(dt("Num"), dd(pNum + ""), dt("Max"), dd(pMax + ""), dt("Num Threads"), dd(pNumThreads + ""), dt("Batch Size"), dd(pBatchSize + ""), dt("Log Period"), dd(pLogPeriod + "")))).withClass("container");
    logger.warn("total={}, rps={}, num={}, max={}, numThreads={} batchSize={}, logPeriod={}", timer.getElapsedString(), timer.getItemsPerSecond(rowCounter.get()), pNum, pMax, pNumThreads, pBatchSize, pLogPeriod);
    return pageFactory.message(request, message);
}

Example 2 with StringTool

use of io.datarouter.util.string.StringTool in project datarouter by hotpads.

the class JobletCopyTableHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceNodeName, @Param(P_targetNodeName) OptionalString targetNodeName, @Param(P_putBatchSize) OptionalString putBatchSize, @Param(P_submitAction) OptionalString submitAction) {
    String errorPutBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (putBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(putBatchSize.get());
            }
        } catch (Exception e) {
            errorPutBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Source Node Name").withName(P_sourceNodeName).withValues(possibleNodes);
    form.addSelectField().withDisplay("Target Node Name").withName(P_targetNodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("Batch Size").withError(errorPutBatchSize).withName(P_putBatchSize).withPlaceholder(DEFAULT_BATCH_SIZE + "").withValue(putBatchSize.orElse(null));
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Copy Table - Joblets").withContent(Html.makeContent(form)).buildMav();
    }
    @SuppressWarnings("unchecked") PhysicalSortedStorageNode<PK, D, ?> sourceNode = (PhysicalSortedStorageNode<PK, D, ?>) nodes.getNode(sourceNodeName.get());
    String tableName = sourceNode.getFieldInfo().getTableName();
    List<TableSample> samples = tableSamplerService.scanSamplesForNode(sourceNode).list();
    TableSampleKey previousSampleKey = null;
    List<JobletPackage> jobletPackages = new ArrayList<>();
    // +1 for databeans beyond the final sample
    long numJoblets = samples.size() + 1;
    long counter = 1;
    int batchSize = putBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    for (TableSample sample : samples) {
        PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
        PK toKeyInclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, sample.getKey());
        jobletPackages.add(createJobletPackage(tableName, sourceNodeName.get(), targetNodeName.get(), fromKeyExclusive, toKeyInclusive, batchSize, sample.getNumRows(), counter, numJoblets));
        ++counter;
        previousSampleKey = sample.getKey();
    }
    // include any rows created since the last sample
    PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
    jobletPackages.add(createJobletPackage(tableName, sourceNodeName.get(), targetNodeName.get(), fromKeyExclusive, // open-ended
    null, batchSize, // we have no idea about the true estNumDatabeans
    1, counter, numJoblets));
    ++counter;
    // shuffle as optimization to spread write load.  could be optional
    Scanner.of(jobletPackages).shuffle().flush(jobletService::submitJobletPackages);
    changelogRecorderService.recordChangelog(getSessionInfo(), "Joblet", sourceNodeName.get(), targetNodeName.get());
    return pageFactory.message(request, "created " + numJoblets + " joblets");
}

Example 3 with StringTool

use of io.datarouter.util.string.StringTool in project datarouter by hotpads.

the class JobletTableProcessorHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_nodeName) OptionalString nodeName, @Param(P_scanBatchSize) OptionalString scanBatchSize, @Param(P_processorName) OptionalString processorName, @Param(P_submitAction) OptionalString submitAction) {
    String errorScanBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (scanBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(scanBatchSize.get());
            }
        } catch (Exception e) {
            errorScanBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleProcessors = Scanner.of(processorRegistry.getAll()).map(Class::getSimpleName).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Node Name").withName(P_nodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("Scan Batch Size").withError(errorScanBatchSize).withName(P_scanBatchSize).withPlaceholder(DEFAULT_SCAN_BATCH_SIZE + "").withValue(scanBatchSize.orElse(null));
    form.addSelectField().withDisplay("Processor Name").withName(P_processorName).withValues(possibleProcessors);
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Table Processor - Joblets").withContent(Html.makeContent(form)).buildMav();
    }
    @SuppressWarnings("unchecked") PhysicalSortedStorageNode<PK, D, ?> sourceNode = (PhysicalSortedStorageNode<PK, D, ?>) nodes.getNode(nodeName.get());
    String tableName = sourceNode.getFieldInfo().getTableName();
    List<TableSample> samples = tableSamplerService.scanSamplesForNode(sourceNode).list();
    TableSampleKey previousSampleKey = null;
    List<JobletPackage> jobletPackages = new ArrayList<>();
    long totalItemsProcessed = 1;
    long counter = 1;
    int actualScanBatchSize = scanBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_SCAN_BATCH_SIZE);
    long numJoblets = 0;
    for (TableSample sample : samples) {
        PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
        PK toKeyInclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, sample.getKey());
        var jobletPackage = createJobletPackage(tableName, nodeName.get(), fromKeyExclusive, toKeyInclusive, actualScanBatchSize, processorName.get(), sample.getNumRows(), counter, numJoblets);
        jobletPackages.add(jobletPackage);
        ++numJoblets;
        counter++;
        totalItemsProcessed++;
        previousSampleKey = sample.getKey();
    }
    // include any rows created since the last sample
    PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
    var jobletPackage = createJobletPackage(tableName, nodeName.get(), fromKeyExclusive, // open-ended
    null, actualScanBatchSize, processorName.get(), // we have no idea about the true estNumDatabeans
    1, counter, numJoblets);
    ++numJoblets;
    jobletPackages.add(jobletPackage);
    totalItemsProcessed++;
    // jobletPackage.size() == counter == numJoblets
    counter++;
    // shuffle as optimization to spread write load. could be optional
    Scanner.of(jobletPackages).shuffle().flush(jobletService::submitJobletPackages);
    changelogRecorderService.recordChangelogForTableProcessor(getSessionInfo(), "Joblet", nodeName.get(), processorName.get());
    return pageFactory.message(request, "jobletsCreated=" + numJoblets + " totalSamplesProcessed=" + totalItemsProcessed);
}

Example 4 with StringTool

use of io.datarouter.util.string.StringTool in project datarouter by hotpads.

the class SingleThreadCopyTableHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceNodeName, @Param(P_targetNodeName) OptionalString targetNodeName, @Param(P_lastKeyString) OptionalString lastKeyString, @Param(P_toEmail) OptionalString toEmail, @Param(P_numThreads) OptionalString numThreads, @Param(P_putBatchSize) OptionalString putBatchSize, @Param(P_submitAction) OptionalString submitAction) {
    String errorNumThreads = null;
    String errorPutBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (numThreads.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(numThreads.get());
            }
        } catch (Exception e) {
            errorNumThreads = "Please specify an integer";
        }
        try {
            if (putBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(putBatchSize.get());
            }
        } catch (Exception e) {
            errorPutBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleSourceNodes = tableSamplerService.scanAllSortedMapStorageNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleTargetNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Source Node Name").withName(P_sourceNodeName).withValues(possibleSourceNodes);
    form.addSelectField().withDisplay("Target Node Name").withName(P_targetNodeName).withValues(possibleTargetNodes);
    form.addTextField().withDisplay("Last Key String").withName(P_lastKeyString).withValue(lastKeyString.orElse(null));
    form.addTextField().withDisplay("Num Threads").withError(errorNumThreads).withName(P_numThreads).withPlaceholder(DEFAULT_NUM_THREADS + "").withValue(numThreads.orElse(null));
    form.addTextField().withDisplay("Batch Size").withError(errorPutBatchSize).withName(P_putBatchSize).withPlaceholder(DEFAULT_BATCH_SIZE + "").withValue(putBatchSize.orElse(null));
    form.addTextField().withDisplay("Email on Completion").withName(P_toEmail).withPlaceholder("you@email.com").withValue(toEmail.orElse(null));
    form.addButton().withDisplay("Copy").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Copy Table - Single Thread").withContent(Html.makeContent(form)).buildMav();
    }
    int actualNumThreads = numThreads.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_NUM_THREADS);
    int actualPutBatchSize = putBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    CopyTableSpanResult result = copyTableService.copyTableSpan(sourceNodeName.get(), targetNodeName.get(), lastKeyString.map(StringTool::nullIfEmpty).orElse(null), null, actualNumThreads, actualPutBatchSize, 1, 1);
    if (!result.success) {
        String message = String.format("The migration was interrupted unexpectedly with %s." + "  Please resume the migration with lastKey %s", result.exception.getMessage(), result.resumeFromKeyString);
        return pageFactory.message(request, message);
    }
    var header = standardDatarouterEmailHeaderService.makeStandardHeader();
    String message = String.format("Successfully migrated %s records from %s to %s", NumberFormatter.addCommas(result.numCopied), sourceNodeName.get(), targetNodeName.get());
    var body = body(header, p(message));
    if (toEmail.filter(str -> !str.isEmpty()).isPresent()) {
        String primaryHref = htmlEmailService.startLinkBuilder().withLocalPath(paths.datarouter.copyTableSingleThread).build();
        var emailBuilder = htmlEmailService.startEmailBuilder().withTitle("Copy Table").withTitleHref(primaryHref).withContent(body).fromAdmin().to(toEmail.get());
        htmlEmailService.trySendJ2Html(emailBuilder);
    }
    changelogRecorderService.recordChangelog(getSessionInfo(), "Single Thread", sourceNodeName.get(), targetNodeName.get());
    return pageFactory.message(request, message);
}

Example 5 with StringTool

use of io.datarouter.util.string.StringTool in project datarouter by hotpads.

the class SingleThreadTableProcessorHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceName, @Param(P_lastKeyString) OptionalString lastKeyString, @Param(P_scanBatchSize) OptionalString scanBatchSize, @Param(P_processorName) OptionalString processorName, @Param(P_toEmail) OptionalString toEmail, @Param(P_submitAction) OptionalString submitAction) {
    String errorScanBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (scanBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(scanBatchSize.get());
            }
        } catch (Exception e) {
            errorScanBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanAllSortedMapStorageNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleProcessors = Scanner.of(processorRegistry.getAll()).map(Class::getSimpleName).sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Node Name").withName(P_sourceNodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("From Key String").withName(P_lastKeyString).withValue(lastKeyString.orElse(null));
    form.addTextField().withDisplay("Scan Batch Size").withError(errorScanBatchSize).withName(P_scanBatchSize).withPlaceholder(DEFAULT_SCAN_BATCH_SIZE + "").withValue(scanBatchSize.orElse(null));
    form.addSelectField().withDisplay("Processor Name").withName(P_processorName).withValues(possibleProcessors);
    form.addTextField().withDisplay("Email on Completion").withName(P_toEmail).withPlaceholder("you@email.com").withValue(toEmail.orElse(null));
    form.addButton().withDisplay("Copy").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Table Processor - Single Thread").withContent(Html.makeContent(form)).buildMav();
    }
    int actualScanBatchSize = scanBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_SCAN_BATCH_SIZE);
    TableProcessor<?, ?> processor = injector.getInstance(processorRegistry.find(processorName.get()).get());
    TableProcessorSpanResult result = service.runTableProcessor(sourceName.get(), lastKeyString.map(StringTool::nullIfEmpty).orElse(null), null, actualScanBatchSize, processor, 1, 1);
    if (!result.success) {
        String message = String.format("The table processor was interrupted unexpectedly with %s." + "  Please resume the processor with lastKey %s", result.exception.getMessage(), result.resumeFromKeyString);
        return pageFactory.message(request, message);
    }
    var header = standardDatarouterEmailHeaderService.makeStandardHeader();
    String message = String.format("Successfully processed %s records for %s - %s", NumberFormatter.addCommas(result.numScanned), sourceName.get(), processorName.get());
    var body = body(header, p(message));
    if (toEmail.filter(str -> !str.isEmpty()).isPresent()) {
        String primaryHref = htmlEmailService.startLinkBuilder().withLocalPath(paths.datarouter.copyTableSingleThread).build();
        var emailBuilder = htmlEmailService.startEmailBuilder().withTitle("Table Processor").withTitleHref(primaryHref).withContent(body).fromAdmin().to(toEmail.get());
        htmlEmailService.trySendJ2Html(emailBuilder);
    }
    changelogRecorderService.recordChangelogForTableProcessor(getSessionInfo(), "Single Thread", sourceName.get(), processorName.get());
    return pageFactory.message(request, message);
}