Examples with PrimaryKey io.datarouter.model.key.primary.PrimaryKey used on opensource projects

Example 6 with PrimaryKey

use of io.datarouter.model.key.primary.PrimaryKey in project datarouter by hotpads.

the class CopyTableService method copyTableSpan.

public <PK extends PrimaryKey<PK>, D extends Databean<PK, D>> CopyTableSpanResult copyTableSpan(String sourceNodeName, String targetNodeName, String fromKeyExclusiveString, String toKeyInclusiveString, int numThreads, int batchSize, long batchId, long numBatches) {
    @SuppressWarnings("unchecked") SortedMapStorageNode<PK, D, ?> sourceNode = (SortedMapStorageNode<PK, D, ?>) nodes.getNode(sourceNodeName);
    Objects.requireNonNull(sourceNode, sourceNodeName + " not found");
    @SuppressWarnings("unchecked") SortedMapStorageNode<PK, D, ?> targetNode = (SortedMapStorageNode<PK, D, ?>) nodes.getNode(targetNodeName);
    Objects.requireNonNull(targetNode, targetNodeName + " not found");
    PK fromKeyExclusive = PrimaryKeyPercentCodecTool.decode(sourceNode.getFieldInfo().getPrimaryKeySupplier(), fromKeyExclusiveString);
    PK toKeyInclusive = PrimaryKeyPercentCodecTool.decode(sourceNode.getFieldInfo().getPrimaryKeySupplier(), toKeyInclusiveString);
    // hbase and bigtable put config
    Config putConfig = new Config();
    // .setPersistentPut(persistentPut)
    // .setIgnoreNullFields(true);//could be configurable
    var numSkipped = new AtomicLong();
    Range<PK> range = new Range<>(fromKeyExclusive, false, toKeyInclusive, true);
    var numScanned = new AtomicLong();
    var numCopied = new AtomicLong();
    AtomicReference<PK> lastKey = new AtomicReference<>();
    try {
        sourceNode.scan(range, SCAN_CONFIG).each($ -> numScanned.incrementAndGet()).each($ -> Counters.inc("copyTable " + sourceNodeName + " read")).batch(batchSize).parallel(putMultiScannerContext.get(numThreads)).each(batch -> targetNode.putMulti(batch, putConfig)).each($ -> Counters.inc("copyTable " + sourceNodeName + " write")).each(batch -> numCopied.addAndGet(batch.size())).each(batch -> lastKey.set(ListTool.getLast(batch).getKey())).sample(10, true).forEach(batch -> logProgress(false, numSkipped.get(), numScanned.get(), numCopied.get(), batchId, numBatches, sourceNodeName, targetNodeName, lastKey.get(), null));
        logProgress(true, numSkipped.get(), numScanned.get(), numCopied.get(), batchId, numBatches, sourceNodeName, targetNodeName, lastKey.get(), null);
        return new CopyTableSpanResult(true, null, numCopied.get(), null);
    } catch (Throwable e) {
        PK pk = lastKey.get();
        logProgress(false, numSkipped.get(), numScanned.get(), numCopied.get(), batchId, numBatches, sourceNodeName, targetNodeName, pk, e);
        String resumeFromKeyString = pk == null ? null : PrimaryKeyPercentCodecTool.encode(pk);
        return new CopyTableSpanResult(false, e, numCopied.get(), resumeFromKeyString);
    }
}