Examples with PrimaryKey io.datarouter.model.key.primary.PrimaryKey used on opensource projects

Example 1 with PrimaryKey

use of io.datarouter.model.key.primary.PrimaryKey in project datarouter by hotpads.

the class JobletCopyTableHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceNodeName, @Param(P_targetNodeName) OptionalString targetNodeName, @Param(P_putBatchSize) OptionalString putBatchSize, @Param(P_submitAction) OptionalString submitAction) {
    String errorPutBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (putBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(putBatchSize.get());
            }
        } catch (Exception e) {
            errorPutBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Source Node Name").withName(P_sourceNodeName).withValues(possibleNodes);
    form.addSelectField().withDisplay("Target Node Name").withName(P_targetNodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("Batch Size").withError(errorPutBatchSize).withName(P_putBatchSize).withPlaceholder(DEFAULT_BATCH_SIZE + "").withValue(putBatchSize.orElse(null));
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Copy Table - Joblets").withContent(Html.makeContent(form)).buildMav();
    }
    @SuppressWarnings("unchecked") PhysicalSortedStorageNode<PK, D, ?> sourceNode = (PhysicalSortedStorageNode<PK, D, ?>) nodes.getNode(sourceNodeName.get());
    String tableName = sourceNode.getFieldInfo().getTableName();
    List<TableSample> samples = tableSamplerService.scanSamplesForNode(sourceNode).list();
    TableSampleKey previousSampleKey = null;
    List<JobletPackage> jobletPackages = new ArrayList<>();
    // +1 for databeans beyond the final sample
    long numJoblets = samples.size() + 1;
    long counter = 1;
    int batchSize = putBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    for (TableSample sample : samples) {
        PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
        PK toKeyInclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, sample.getKey());
        jobletPackages.add(createJobletPackage(tableName, sourceNodeName.get(), targetNodeName.get(), fromKeyExclusive, toKeyInclusive, batchSize, sample.getNumRows(), counter, numJoblets));
        ++counter;
        previousSampleKey = sample.getKey();
    }
    // include any rows created since the last sample
    PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
    jobletPackages.add(createJobletPackage(tableName, sourceNodeName.get(), targetNodeName.get(), fromKeyExclusive, // open-ended
    null, batchSize, // we have no idea about the true estNumDatabeans
    1, counter, numJoblets));
    ++counter;
    // shuffle as optimization to spread write load.  could be optional
    Scanner.of(jobletPackages).shuffle().flush(jobletService::submitJobletPackages);
    changelogRecorderService.recordChangelog(getSessionInfo(), "Joblet", sourceNodeName.get(), targetNodeName.get());
    return pageFactory.message(request, "created " + numJoblets + " joblets");
}

Example 2 with PrimaryKey

use of io.datarouter.model.key.primary.PrimaryKey in project datarouter by hotpads.

the class JobletTableProcessorHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_nodeName) OptionalString nodeName, @Param(P_scanBatchSize) OptionalString scanBatchSize, @Param(P_processorName) OptionalString processorName, @Param(P_submitAction) OptionalString submitAction) {
    String errorScanBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (scanBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(scanBatchSize.get());
            }
        } catch (Exception e) {
            errorScanBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleProcessors = Scanner.of(processorRegistry.getAll()).map(Class::getSimpleName).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Node Name").withName(P_nodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("Scan Batch Size").withError(errorScanBatchSize).withName(P_scanBatchSize).withPlaceholder(DEFAULT_SCAN_BATCH_SIZE + "").withValue(scanBatchSize.orElse(null));
    form.addSelectField().withDisplay("Processor Name").withName(P_processorName).withValues(possibleProcessors);
    form.addButton().withDisplay("Create Joblets").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Table Processor - Joblets").withContent(Html.makeContent(form)).buildMav();
    }
    @SuppressWarnings("unchecked") PhysicalSortedStorageNode<PK, D, ?> sourceNode = (PhysicalSortedStorageNode<PK, D, ?>) nodes.getNode(nodeName.get());
    String tableName = sourceNode.getFieldInfo().getTableName();
    List<TableSample> samples = tableSamplerService.scanSamplesForNode(sourceNode).list();
    TableSampleKey previousSampleKey = null;
    List<JobletPackage> jobletPackages = new ArrayList<>();
    long totalItemsProcessed = 1;
    long counter = 1;
    int actualScanBatchSize = scanBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_SCAN_BATCH_SIZE);
    long numJoblets = 0;
    for (TableSample sample : samples) {
        PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
        PK toKeyInclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, sample.getKey());
        var jobletPackage = createJobletPackage(tableName, nodeName.get(), fromKeyExclusive, toKeyInclusive, actualScanBatchSize, processorName.get(), sample.getNumRows(), counter, numJoblets);
        jobletPackages.add(jobletPackage);
        ++numJoblets;
        counter++;
        totalItemsProcessed++;
        previousSampleKey = sample.getKey();
    }
    // include any rows created since the last sample
    PK fromKeyExclusive = TableSamplerTool.extractPrimaryKeyFromSampleKey(sourceNode, previousSampleKey);
    var jobletPackage = createJobletPackage(tableName, nodeName.get(), fromKeyExclusive, // open-ended
    null, actualScanBatchSize, processorName.get(), // we have no idea about the true estNumDatabeans
    1, counter, numJoblets);
    ++numJoblets;
    jobletPackages.add(jobletPackage);
    totalItemsProcessed++;
    // jobletPackage.size() == counter == numJoblets
    counter++;
    // shuffle as optimization to spread write load. could be optional
    Scanner.of(jobletPackages).shuffle().flush(jobletService::submitJobletPackages);
    changelogRecorderService.recordChangelogForTableProcessor(getSessionInfo(), "Joblet", nodeName.get(), processorName.get());
    return pageFactory.message(request, "jobletsCreated=" + numJoblets + " totalSamplesProcessed=" + totalItemsProcessed);
}

Example 3 with PrimaryKey

use of io.datarouter.model.key.primary.PrimaryKey in project datarouter by hotpads.

the class TableProcessorService method runTableProcessor.

public <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> TableProcessorSpanResult runTableProcessor(String nodeName, String fromKeyExclusiveString, String toKeyInclusiveString, int scanBatchSize, TableProcessor<PK, D> tableProcessor, long batchId, long numBatches) {
    @SuppressWarnings("unchecked") SortedMapStorageNode<PK, D, ?> node = (SortedMapStorageNode<PK, D, ?>) nodes.getNode(nodeName);
    Objects.requireNonNull(node, nodeName + " not found");
    PK fromKeyExclusive = PrimaryKeyPercentCodecTool.decode(node.getFieldInfo().getPrimaryKeySupplier(), fromKeyExclusiveString);
    PK toKeyInclusive = PrimaryKeyPercentCodecTool.decode(node.getFieldInfo().getPrimaryKeySupplier(), toKeyInclusiveString);
    Range<PK> range = new Range<>(fromKeyExclusive, false, toKeyInclusive, true);
    var numScanned = new AtomicLong();
    AtomicReference<PK> lastKey = new AtomicReference<>();
    try {
        node.scan(range, new Config().setResponseBatchSize(scanBatchSize)).each($ -> Counters.inc("tableProcessor " + nodeName + " scanned")).each($ -> numScanned.incrementAndGet()).each(databean -> lastKey.set(databean.getKey())).each($ -> {
            if (numScanned.get() % 10_000 == 0) {
                logProgress(false, numScanned.get(), batchId, numBatches, nodeName, lastKey.get(), null);
            }
        }).then(tableProcessor::accept);
        logProgress(true, numScanned.get(), batchId, numBatches, nodeName, lastKey.get(), null);
        return new TableProcessorSpanResult(true, null, numScanned.get(), null);
    } catch (Throwable e) {
        PK pk = lastKey.get();
        logProgress(false, numScanned.get(), batchId, numBatches, nodeName, pk, e);
        String resumeFromKeyString = pk == null ? null : PrimaryKeyPercentCodecTool.encode(pk);
        return new TableProcessorSpanResult(false, e, numScanned.get(), resumeFromKeyString);
    }
}

Example 4 with PrimaryKey

use of io.datarouter.model.key.primary.PrimaryKey in project datarouter by hotpads.

the class SingleThreadCopyTableHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceNodeName, @Param(P_targetNodeName) OptionalString targetNodeName, @Param(P_lastKeyString) OptionalString lastKeyString, @Param(P_toEmail) OptionalString toEmail, @Param(P_numThreads) OptionalString numThreads, @Param(P_putBatchSize) OptionalString putBatchSize, @Param(P_submitAction) OptionalString submitAction) {
    String errorNumThreads = null;
    String errorPutBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (numThreads.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(numThreads.get());
            }
        } catch (Exception e) {
            errorNumThreads = "Please specify an integer";
        }
        try {
            if (putBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(putBatchSize.get());
            }
        } catch (Exception e) {
            errorPutBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleSourceNodes = tableSamplerService.scanAllSortedMapStorageNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleTargetNodes = tableSamplerService.scanCountableNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Source Node Name").withName(P_sourceNodeName).withValues(possibleSourceNodes);
    form.addSelectField().withDisplay("Target Node Name").withName(P_targetNodeName).withValues(possibleTargetNodes);
    form.addTextField().withDisplay("Last Key String").withName(P_lastKeyString).withValue(lastKeyString.orElse(null));
    form.addTextField().withDisplay("Num Threads").withError(errorNumThreads).withName(P_numThreads).withPlaceholder(DEFAULT_NUM_THREADS + "").withValue(numThreads.orElse(null));
    form.addTextField().withDisplay("Batch Size").withError(errorPutBatchSize).withName(P_putBatchSize).withPlaceholder(DEFAULT_BATCH_SIZE + "").withValue(putBatchSize.orElse(null));
    form.addTextField().withDisplay("Email on Completion").withName(P_toEmail).withPlaceholder("you@email.com").withValue(toEmail.orElse(null));
    form.addButton().withDisplay("Copy").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Copy Table - Single Thread").withContent(Html.makeContent(form)).buildMav();
    }
    int actualNumThreads = numThreads.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_NUM_THREADS);
    int actualPutBatchSize = putBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_BATCH_SIZE);
    CopyTableSpanResult result = copyTableService.copyTableSpan(sourceNodeName.get(), targetNodeName.get(), lastKeyString.map(StringTool::nullIfEmpty).orElse(null), null, actualNumThreads, actualPutBatchSize, 1, 1);
    if (!result.success) {
        String message = String.format("The migration was interrupted unexpectedly with %s." + "  Please resume the migration with lastKey %s", result.exception.getMessage(), result.resumeFromKeyString);
        return pageFactory.message(request, message);
    }
    var header = standardDatarouterEmailHeaderService.makeStandardHeader();
    String message = String.format("Successfully migrated %s records from %s to %s", NumberFormatter.addCommas(result.numCopied), sourceNodeName.get(), targetNodeName.get());
    var body = body(header, p(message));
    if (toEmail.filter(str -> !str.isEmpty()).isPresent()) {
        String primaryHref = htmlEmailService.startLinkBuilder().withLocalPath(paths.datarouter.copyTableSingleThread).build();
        var emailBuilder = htmlEmailService.startEmailBuilder().withTitle("Copy Table").withTitleHref(primaryHref).withContent(body).fromAdmin().to(toEmail.get());
        htmlEmailService.trySendJ2Html(emailBuilder);
    }
    changelogRecorderService.recordChangelog(getSessionInfo(), "Single Thread", sourceNodeName.get(), targetNodeName.get());
    return pageFactory.message(request, message);
}

Example 5 with PrimaryKey

use of io.datarouter.model.key.primary.PrimaryKey in project datarouter by hotpads.

the class SingleThreadTableProcessorHandler method defaultHandler.

@Handler(defaultHandler = true)
private <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> Mav defaultHandler(@Param(P_sourceNodeName) OptionalString sourceName, @Param(P_lastKeyString) OptionalString lastKeyString, @Param(P_scanBatchSize) OptionalString scanBatchSize, @Param(P_processorName) OptionalString processorName, @Param(P_toEmail) OptionalString toEmail, @Param(P_submitAction) OptionalString submitAction) {
    String errorScanBatchSize = null;
    if (submitAction.isPresent()) {
        try {
            if (scanBatchSize.map(StringTool::nullIfEmpty).isPresent()) {
                Integer.valueOf(scanBatchSize.get());
            }
        } catch (Exception e) {
            errorScanBatchSize = "Please specify an integer";
        }
    }
    List<String> possibleNodes = tableSamplerService.scanAllSortedMapStorageNodes().map(node -> node.getClientId().getName() + "." + node.getFieldInfo().getTableName()).append("").sort().list();
    List<String> possibleProcessors = Scanner.of(processorRegistry.getAll()).map(Class::getSimpleName).sort().list();
    var form = new HtmlForm().withMethod("post");
    form.addSelectField().withDisplay("Node Name").withName(P_sourceNodeName).withValues(possibleNodes);
    form.addTextField().withDisplay("From Key String").withName(P_lastKeyString).withValue(lastKeyString.orElse(null));
    form.addTextField().withDisplay("Scan Batch Size").withError(errorScanBatchSize).withName(P_scanBatchSize).withPlaceholder(DEFAULT_SCAN_BATCH_SIZE + "").withValue(scanBatchSize.orElse(null));
    form.addSelectField().withDisplay("Processor Name").withName(P_processorName).withValues(possibleProcessors);
    form.addTextField().withDisplay("Email on Completion").withName(P_toEmail).withPlaceholder("you@email.com").withValue(toEmail.orElse(null));
    form.addButton().withDisplay("Copy").withValue("anything");
    if (submitAction.isEmpty() || form.hasErrors()) {
        return pageFactory.startBuilder(request).withTitle("Table Processor - Single Thread").withContent(Html.makeContent(form)).buildMav();
    }
    int actualScanBatchSize = scanBatchSize.map(StringTool::nullIfEmpty).map(Integer::valueOf).orElse(DEFAULT_SCAN_BATCH_SIZE);
    TableProcessor<?, ?> processor = injector.getInstance(processorRegistry.find(processorName.get()).get());
    TableProcessorSpanResult result = service.runTableProcessor(sourceName.get(), lastKeyString.map(StringTool::nullIfEmpty).orElse(null), null, actualScanBatchSize, processor, 1, 1);
    if (!result.success) {
        String message = String.format("The table processor was interrupted unexpectedly with %s." + "  Please resume the processor with lastKey %s", result.exception.getMessage(), result.resumeFromKeyString);
        return pageFactory.message(request, message);
    }
    var header = standardDatarouterEmailHeaderService.makeStandardHeader();
    String message = String.format("Successfully processed %s records for %s - %s", NumberFormatter.addCommas(result.numScanned), sourceName.get(), processorName.get());
    var body = body(header, p(message));
    if (toEmail.filter(str -> !str.isEmpty()).isPresent()) {
        String primaryHref = htmlEmailService.startLinkBuilder().withLocalPath(paths.datarouter.copyTableSingleThread).build();
        var emailBuilder = htmlEmailService.startEmailBuilder().withTitle("Table Processor").withTitleHref(primaryHref).withContent(body).fromAdmin().to(toEmail.get());
        htmlEmailService.trySendJ2Html(emailBuilder);
    }
    changelogRecorderService.recordChangelogForTableProcessor(getSessionInfo(), "Single Thread", sourceName.get(), processorName.get());
    return pageFactory.message(request, message);
}