Examples with SortedMapStorageNode io.datarouter.storage.node.op.combo.SortedMapStorage.SortedMapStorageNode used on opensource projects

Example 1 with SortedMapStorageNode

use of io.datarouter.storage.node.op.combo.SortedMapStorage.SortedMapStorageNode in project datarouter by hotpads.

the class TableProcessorService method runTableProcessor.

public <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> TableProcessorSpanResult runTableProcessor(String nodeName, String fromKeyExclusiveString, String toKeyInclusiveString, int scanBatchSize, TableProcessor<PK, D> tableProcessor, long batchId, long numBatches) {
    @SuppressWarnings("unchecked") SortedMapStorageNode<PK, D, ?> node = (SortedMapStorageNode<PK, D, ?>) nodes.getNode(nodeName);
    Objects.requireNonNull(node, nodeName + " not found");
    PK fromKeyExclusive = PrimaryKeyPercentCodecTool.decode(node.getFieldInfo().getPrimaryKeySupplier(), fromKeyExclusiveString);
    PK toKeyInclusive = PrimaryKeyPercentCodecTool.decode(node.getFieldInfo().getPrimaryKeySupplier(), toKeyInclusiveString);
    Range<PK> range = new Range<>(fromKeyExclusive, false, toKeyInclusive, true);
    var numScanned = new AtomicLong();
    AtomicReference<PK> lastKey = new AtomicReference<>();
    try {
        node.scan(range, new Config().setResponseBatchSize(scanBatchSize)).each($ -> Counters.inc("tableProcessor " + nodeName + " scanned")).each($ -> numScanned.incrementAndGet()).each(databean -> lastKey.set(databean.getKey())).each($ -> {
            if (numScanned.get() % 10_000 == 0) {
                logProgress(false, numScanned.get(), batchId, numBatches, nodeName, lastKey.get(), null);
            }
        }).then(tableProcessor::accept);
        logProgress(true, numScanned.get(), batchId, numBatches, nodeName, lastKey.get(), null);
        return new TableProcessorSpanResult(true, null, numScanned.get(), null);
    } catch (Throwable e) {
        PK pk = lastKey.get();
        logProgress(false, numScanned.get(), batchId, numBatches, nodeName, pk, e);
        String resumeFromKeyString = pk == null ? null : PrimaryKeyPercentCodecTool.encode(pk);
        return new TableProcessorSpanResult(false, e, numScanned.get(), resumeFromKeyString);
    }
}

Example 2 with SortedMapStorageNode

use of io.datarouter.storage.node.op.combo.SortedMapStorage.SortedMapStorageNode in project datarouter by hotpads.

the class CopyTableService method copyTableSpan.

public <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> CopyTableSpanResult copyTableSpan(String sourceNodeName, String targetNodeName, String fromKeyExclusiveString, String toKeyInclusiveString, int numThreads, int batchSize, long batchId, long numBatches) {
    @SuppressWarnings("unchecked") SortedMapStorageNode<PK, D, ?> sourceNode = (SortedMapStorageNode<PK, D, ?>) nodes.getNode(sourceNodeName);
    Objects.requireNonNull(sourceNode, sourceNodeName + " not found");
    @SuppressWarnings("unchecked") SortedMapStorageNode<PK, D, ?> targetNode = (SortedMapStorageNode<PK, D, ?>) nodes.getNode(targetNodeName);
    Objects.requireNonNull(targetNode, targetNodeName + " not found");
    PK fromKeyExclusive = PrimaryKeyPercentCodecTool.decode(sourceNode.getFieldInfo().getPrimaryKeySupplier(), fromKeyExclusiveString);
    PK toKeyInclusive = PrimaryKeyPercentCodecTool.decode(sourceNode.getFieldInfo().getPrimaryKeySupplier(), toKeyInclusiveString);
    // hbase and bigtable put config
    Config putConfig = new Config();
    // .setPersistentPut(persistentPut)
    // .setIgnoreNullFields(true);//could be configurable
    var numSkipped = new AtomicLong();
    Range<PK> range = new Range<>(fromKeyExclusive, false, toKeyInclusive, true);
    var numScanned = new AtomicLong();
    var numCopied = new AtomicLong();
    AtomicReference<PK> lastKey = new AtomicReference<>();
    try {
        sourceNode.scan(range, SCAN_CONFIG).each($ -> numScanned.incrementAndGet()).each($ -> Counters.inc("copyTable " + sourceNodeName + " read")).batch(batchSize).parallel(putMultiScannerContext.get(numThreads)).each(batch -> targetNode.putMulti(batch, putConfig)).each($ -> Counters.inc("copyTable " + sourceNodeName + " write")).each(batch -> numCopied.addAndGet(batch.size())).each(batch -> lastKey.set(ListTool.getLast(batch).getKey())).sample(10, true).forEach(batch -> logProgress(false, numSkipped.get(), numScanned.get(), numCopied.get(), batchId, numBatches, sourceNodeName, targetNodeName, lastKey.get(), null));
        logProgress(true, numSkipped.get(), numScanned.get(), numCopied.get(), batchId, numBatches, sourceNodeName, targetNodeName, lastKey.get(), null);
        return new CopyTableSpanResult(true, null, numCopied.get(), null);
    } catch (Throwable e) {
        PK pk = lastKey.get();
        logProgress(false, numSkipped.get(), numScanned.get(), numCopied.get(), batchId, numBatches, sourceNodeName, targetNodeName, pk, e);
        String resumeFromKeyString = pk == null ? null : PrimaryKeyPercentCodecTool.encode(pk);
        return new CopyTableSpanResult(false, e, numCopied.get(), resumeFromKeyString);
    }
}