Examples with SnapshotWriteResult io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult used on opensource projects

Example 1 with SnapshotWriteResult

use of io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult in project datarouter by hotpads.

the class SnapshotMerger method merge.

public void merge() {
    Map<SnapshotKey, SnapshotKeyAndNumRecords> summaryByKey = mergeGroup.keyReadOps(false).scanSnapshotKeysAndRootBlocks(readExec, 10).map(SnapshotKeyAndNumRecords::new).toMap(summary -> summary.key);
    while (summaryByKey.size() > 1) {
        SnapshotGroup outputGroup = summaryByKey.size() <= mergeFactor ? destinationGroup : mergeGroup;
        Scanner.of(summaryByKey.values()).minN(SnapshotKeyAndNumRecords.BY_NUM_RECORDS, mergeFactor).map(summary -> summary.key).flush(keys -> {
            SnapshotWriteResult result = combineSnapshots(keys, outputGroup);
            var newSummary = new SnapshotKeyAndNumRecords(result.toSnapshotKeyAndRoot());
            summaryByKey.put(result.key, newSummary);
        }).forEach(summaryByKey::remove);
    }
}

Example 2 with SnapshotWriteResult

use of io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult in project datarouter by hotpads.

the class SnapshotMerger method combineSnapshots.

private SnapshotWriteResult combineSnapshots(List<SnapshotKey> keys, SnapshotGroup outputGroup) {
    SnapshotWriteResult result = Scanner.of(keys).map(key -> new ScanningSnapshotReader(key, readExec, 10, mergeGroup, scanNumBlocks)).collate(reader -> reader.scanLeafRecords(0), SnapshotLeafRecord.KEY_COMPARATOR).deduplicateConsecutiveBy(leafRecord -> leafRecord.key, Arrays::equals).map(SnapshotLeafRecord::entry).batch(10_000).apply(batches -> outputGroup.writeOps().write(writerConfig, batches, writeExec, shouldStop));
    keys.forEach(key -> mergeGroup.deleteOps().deleteSnapshot(key, writeExec, 10));
    logger.warn("combined {}, {}", keys.size(), keys);
    return result;
}

Example 3 with SnapshotWriteResult

use of io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult in project datarouter by hotpads.

the class SnapshotGroupWriteOps method writeWithId.

// could be public if something like back-dating snapshotIds is necessary
private SnapshotWriteResult writeWithId(SnapshotWriterConfig config, Scanner<List<SnapshotEntry>> entries, String snapshotId, ExecutorService exec, Supplier<Boolean> shouldStop) {
    var snapshotKey = new SnapshotKey(groupId, snapshotId);
    SnapshotFileStorage snapshotFileStorage = group.makeSnapshotFileStorage(snapshotKey.snapshotId);
    SnapshotBlockStorage snapshotBlockStorage = cacheStorage == null ? null : group.makeCacheStorage(snapshotKey.snapshotId);
    try (var writer = new SnapshotWriter(snapshotKey, snapshotFileStorage, snapshotBlockStorage, config, exec)) {
        entries.advanceUntil($ -> shouldStop.get()).forEach(writer::addBatch);
        if (shouldStop.get()) {
            return SnapshotWriteResult.failure(snapshotKey);
        }
        return writer.complete().map(rootBlock -> {
            writeIdFile(snapshotKey.snapshotId);
            decodingBlockLoaderBySnapshotKey.put(snapshotKey, decodingBlockLoaderFactory.create(rootBlock, group.makeStorageReader(snapshotKey.snapshotId)));
            return SnapshotWriteResult.success(snapshotKey, rootBlock);
        }).orElseGet(() -> {
            logger.warn("snapshot {} had no entries and was not written", snapshotKey);
            return SnapshotWriteResult.empty(snapshotKey);
        });
    }
}

Example 4 with SnapshotWriteResult

use of io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult in project datarouter by hotpads.

the class SnapshotBenchmark method execute.

public RootBlock execute() {
    var timer = new PhaseTimer("writeSnapshot");
    SnapshotWriteResult result = makeEntryScanner(scannerExec, numInputThreads).apply(entries -> group.writeOps().write(makeSnapshotWriterConfig(), entries, writerExec, () -> false));
    snapshotKey = result.key;
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{} @{}/s", timer, NumberFormatter.addCommas(timer.getItemsPerSecond(numEntries)));
    return result.optRoot.get();
}

Example 5 with SnapshotWriteResult

use of io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method writeOutputSnapshot.

private RootBlock writeOutputSnapshot(Scanner<SnapshotRecord> records) {
    var timer = new PhaseTimer("writeOutputSnapshot");
    SnapshotWriterConfig config = makeSnapshotWriterConfig(true);
    SnapshotWriteResult result = records.map(SnapshotRecord::entry).batch(1000).apply(entries -> outputGroup.writeOps().write(config, entries, exec, () -> false));
    outputSnapshotKey = result.key;
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{}", timer);
    return result.optRoot.get();
}