Examples with SnapshotWriterConfig io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig used on opensource projects

Example 1 with SnapshotWriterConfig

use of io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig in project datarouter by hotpads.

the class SnapshotGroupWriteOps method writeWithId.

// could be public if something like back-dating snapshotIds is necessary
private SnapshotWriteResult writeWithId(SnapshotWriterConfig config, Scanner<List<SnapshotEntry>> entries, String snapshotId, ExecutorService exec, Supplier<Boolean> shouldStop) {
    var snapshotKey = new SnapshotKey(groupId, snapshotId);
    SnapshotFileStorage snapshotFileStorage = group.makeSnapshotFileStorage(snapshotKey.snapshotId);
    SnapshotBlockStorage snapshotBlockStorage = cacheStorage == null ? null : group.makeCacheStorage(snapshotKey.snapshotId);
    try (var writer = new SnapshotWriter(snapshotKey, snapshotFileStorage, snapshotBlockStorage, config, exec)) {
        entries.advanceUntil($ -> shouldStop.get()).forEach(writer::addBatch);
        if (shouldStop.get()) {
            return SnapshotWriteResult.failure(snapshotKey);
        }
        return writer.complete().map(rootBlock -> {
            writeIdFile(snapshotKey.snapshotId);
            decodingBlockLoaderBySnapshotKey.put(snapshotKey, decodingBlockLoaderFactory.create(rootBlock, group.makeStorageReader(snapshotKey.snapshotId)));
            return SnapshotWriteResult.success(snapshotKey, rootBlock);
        }).orElseGet(() -> {
            logger.warn("snapshot {} had no entries and was not written", snapshotKey);
            return SnapshotWriteResult.empty(snapshotKey);
        });
    }
}

Example 2 with SnapshotWriterConfig

use of io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method writeOutputSnapshot.

private RootBlock writeOutputSnapshot(Scanner<SnapshotRecord> records) {
    var timer = new PhaseTimer("writeOutputSnapshot");
    SnapshotWriterConfig config = makeSnapshotWriterConfig(true);
    SnapshotWriteResult result = records.map(SnapshotRecord::entry).batch(1000).apply(entries -> outputGroup.writeOps().write(config, entries, exec, () -> false));
    outputSnapshotKey = result.key;
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{}", timer);
    return result.optRoot.get();
}

Example 3 with SnapshotWriterConfig

use of io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method writeInputSnapshot.

private RootBlock writeInputSnapshot() {
    var timer = new PhaseTimer("writeInputSnapshot");
    SnapshotWriterConfig config = makeSnapshotWriterConfig(false);
    SnapshotWriteResult result = Scanner.iterate(0, i -> i + 1).limit(NUM_ENTRIES).shuffle().map(FilesystemSnapshotSortingTests::makeEntry).batch(1000).apply(entries -> inputGroup.writeOps().write(config, entries, exec, () -> false));
    inputSnapshotKey = result.key;
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{}", timer);
    return result.optRoot.get();
}

Example 4 with SnapshotWriterConfig

use of io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method writeChunkSnapshot.

private SnapshotKey writeChunkSnapshot(int chunkId, List<SnapshotRecord> records) {
    var timer = new PhaseTimer("writeChunkSnapshot " + chunkId);
    SnapshotWriterConfig config = makeSnapshotWriterConfig(true);
    SnapshotWriteResult result = Scanner.of(records).map(SnapshotRecord::entry).batch(1000).apply(entries -> chunkGroup.writeOps().write(config, entries, exec, () -> false));
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{}", timer);
    return result.key;
}