Search in sources :

Example 1 with SnapshotRecord

use of io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord in project datarouter by hotpads.

the class DatarouterSnapshotEntryHandler method buildTable.

private ContainerTag<?> buildTable(SnapshotKey snapshotKey, long id) {
    SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
    var reader = new SnapshotIdReader(snapshotKey, groups);
    SnapshotRecord record = reader.getRecord(id);
    SnapshotRecordStringDecoder decoder = ReflectionTool.create(group.getSnapshotEntryDecoderClass());
    SnapshotRecordStrings decoded = decoder.decode(record);
    List<Twin<String>> rows = new ArrayList<>();
    rows.add(new Twin<>("id", Long.toString(record.id)));
    rows.add(new Twin<>(decoder.keyName(), decoded.key));
    rows.add(new Twin<>(decoder.valueName(), decoded.value));
    IntStream.range(0, decoded.columnValues.size()).mapToObj(column -> new Twin<>(decoder.columnValueName(column), decoded.columnValues.get(column))).forEach(rows::add);
    var table = new J2HtmlTable<Twin<String>>().withClasses("sortable table table-sm table-striped my-4 border").withColumn("field", twin -> twin.getLeft()).withColumn("value", twin -> twin.getRight()).build(rows);
    return table;
}
Also used : IntStream(java.util.stream.IntStream) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) Twin(io.datarouter.util.tuple.Twin) Mav(io.datarouter.web.handler.mav.Mav) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) ReflectionTool(io.datarouter.util.lang.ReflectionTool) ArrayList(java.util.ArrayList) ContainerTag(j2html.tags.ContainerTag) Inject(javax.inject.Inject) List(java.util.List) DatarouterWebRequireJsV2(io.datarouter.web.requirejs.DatarouterWebRequireJsV2) SnapshotGroups(io.datarouter.filesystem.snapshot.group.SnapshotGroups) BaseHandler(io.datarouter.web.handler.BaseHandler) Bootstrap4PageFactory(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4PageFactory) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) Param(io.datarouter.web.handler.types.Param) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) SnapshotIdReader(io.datarouter.filesystem.snapshot.reader.SnapshotIdReader) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) ArrayList(java.util.ArrayList) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) SnapshotIdReader(io.datarouter.filesystem.snapshot.reader.SnapshotIdReader) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) Twin(io.datarouter.util.tuple.Twin)

Example 2 with SnapshotRecord

use of io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord in project datarouter by hotpads.

the class BaseSnapshotTests method testScan.

@Test
public void testScan() {
    if (!ENABLED_TESTS.contains(TestId.SCAN)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    List<SnapshotRecord> outputs = reader.scan(0).list();
    Assert.assertEquals(outputs.size(), sortedInputs.size());
    for (int i = 0; i < sortedInputs.size(); ++i) {
        Input input = sortedInputs.get(i);
        SnapshotRecord output = outputs.get(i);
        Assert.assertEquals(i, output.id);
        for (int column = 0; column < input.entry.columnValues.length; ++column) {
            if (!SnapshotEntry.equalColumnValue(input.entry, output.entry(), column)) {
                String message = String.format("%s, actual=%s, expected=%s", i, utf8(output.columnValues[column]), utf8(input.entry.columnValues[column]));
                throw new RuntimeException(message);
            }
        }
    }
}
Also used : ScanningSnapshotReader(io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader) BlockLoader(io.datarouter.filesystem.snapshot.reader.block.BlockLoader) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) Test(org.testng.annotations.Test)

Example 3 with SnapshotRecord

use of io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord in project datarouter by hotpads.

the class BaseSnapshotTests method testOperationInternal.

private void testOperationInternal(BlockLoader threadSafeBlockLoader, boolean random, boolean multiThreaded, Operation operation) {
    List<Input> searchKeys = random ? randomInputs : sortedInputs;
    int batchSize = 10_000;
    var parallelScannerContext = new ParallelScannerContext(exec, getNumThreads(), true, multiThreaded);
    var count = new AtomicLong();
    Scanner.of(searchKeys).batch(batchSize).parallel(parallelScannerContext).forEach(batch -> {
        var idReader = new SnapshotIdReader(snapshotKey, threadSafeBlockLoader);
        var keyReader = new SnapshotKeyReader(snapshotKey, threadSafeBlockLoader);
        for (int i = 0; i < batch.size(); ++i) {
            Input input = batch.get(i);
            long id = input.id;
            byte[] key = input.entry.key();
            byte[] value = input.entry.value();
            if (Operation.GET_LEAF_RECORD == operation) {
                SnapshotLeafRecord leafRecord = idReader.leafRecord(id);
                if (!Arrays.equals(key, leafRecord.key)) {
                    String message = String.format("%s, expected=%s, actual=%s", id, utf8(key), utf8(leafRecord.key));
                    throw new RuntimeException(message);
                }
                if (!Arrays.equals(value, leafRecord.value)) {
                    String message = String.format("%s, expected=%s, actual=%s", id, utf8(value), utf8(leafRecord.value));
                    throw new RuntimeException(message);
                }
            } else if (Operation.GET_RECORD == operation) {
                SnapshotRecord result = idReader.getRecord(id);
                if (id != result.id) {
                    String message = String.format("%s, expected=%s, actual=%s", id, id, result.id);
                    throw new RuntimeException(message);
                }
                if (!Arrays.equals(key, result.key)) {
                    String message = String.format("%s, expected=%s, actual=%s", id, utf8(key), utf8(result.key));
                    throw new RuntimeException(message);
                }
                if (!SnapshotEntry.equal(input.entry, result.entry())) {
                    String message = String.format("%s, expected=%s, actual=%s", i, // TODO print more than column 0
                    utf8(input.entry.columnValues[0]), utf8(result.columnValues[0]));
                    throw new RuntimeException(message);
                }
            } else if (Operation.FIND_ID == operation) {
                if (keyReader.findRecordId(key).isEmpty()) {
                    String message = String.format("%s, %s not found", i, utf8(key));
                    throw new RuntimeException(message);
                }
                if (id != keyReader.findRecordId(key).get().longValue()) {
                    String message = String.format("%s, %s not found", i, utf8(key));
                    throw new RuntimeException(message);
                }
            } else if (Operation.FIND_RECORD == operation) {
                Optional<SnapshotRecord> output = keyReader.findRecord(key);
                if (output.isEmpty()) {
                    String message = String.format("%s, %s not found", i, utf8(key));
                    throw new RuntimeException(message);
                }
                if (!SnapshotEntry.equal(input.entry, output.get().entry())) {
                    String message = String.format("%s, expected=%s, actual=%s", i, // TODO print more than column 0
                    utf8(batch.get(i).entry.columnValues[0]), utf8(output.get().columnValues[0]));
                    throw new RuntimeException(message);
                }
            }
        }
        count.addAndGet(batch.size());
        logger.warn("{}, {}, {} for {}/{} {}", random ? "random" : "sorted", multiThreaded ? "multi" : "single", operation.toString().toLowerCase(), NumberFormatter.addCommas(count.get()), NumberFormatter.addCommas(searchKeys.size()), utf8(ListTool.getLast(batch).entry.key()));
    });
}
Also used : SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Optional(java.util.Optional) ParallelScannerContext(io.datarouter.scanner.ParallelScannerContext) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) SnapshotIdReader(io.datarouter.filesystem.snapshot.reader.SnapshotIdReader) AtomicLong(java.util.concurrent.atomic.AtomicLong) SnapshotKeyReader(io.datarouter.filesystem.snapshot.reader.SnapshotKeyReader)

Example 4 with SnapshotRecord

use of io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method writeOutputSnapshot.

private RootBlock writeOutputSnapshot(Scanner<SnapshotRecord> records) {
    var timer = new PhaseTimer("writeOutputSnapshot");
    SnapshotWriterConfig config = makeSnapshotWriterConfig(true);
    SnapshotWriteResult result = records.map(SnapshotRecord::entry).batch(1000).apply(entries -> outputGroup.writeOps().write(config, entries, exec, () -> false));
    outputSnapshotKey = result.key;
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{}", timer);
    return result.optRoot.get();
}
Also used : PhaseTimer(io.datarouter.util.timer.PhaseTimer) SnapshotWriterConfig(io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig) SnapshotWriteResult(io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord)

Example 5 with SnapshotRecord

use of io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method writeChunkSnapshot.

private SnapshotKey writeChunkSnapshot(int chunkId, List<SnapshotRecord> records) {
    var timer = new PhaseTimer("writeChunkSnapshot " + chunkId);
    SnapshotWriterConfig config = makeSnapshotWriterConfig(true);
    SnapshotWriteResult result = Scanner.of(records).map(SnapshotRecord::entry).batch(1000).apply(entries -> chunkGroup.writeOps().write(config, entries, exec, () -> false));
    timer.add("wrote " + NumberFormatter.addCommas(result.optRoot.get().numItems()));
    logger.warn("{}", timer);
    return result.key;
}
Also used : PhaseTimer(io.datarouter.util.timer.PhaseTimer) SnapshotWriterConfig(io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig) SnapshotWriteResult(io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord)

Aggregations

SnapshotRecord (io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord)9 Bytes (io.datarouter.bytes.Bytes)3 BlockKey (io.datarouter.filesystem.snapshot.block.BlockKey)3 LeafBlock (io.datarouter.filesystem.snapshot.block.leaf.LeafBlock)3 ValueBlock (io.datarouter.filesystem.snapshot.block.value.ValueBlock)3 SnapshotWriteResult (io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult)3 SnapshotIdReader (io.datarouter.filesystem.snapshot.reader.SnapshotIdReader)3 SnapshotWriterConfig (io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig)3 PhaseTimer (io.datarouter.util.timer.PhaseTimer)3 List (java.util.List)3 IntStream (java.util.stream.IntStream)3 ValueLocation (io.datarouter.filesystem.snapshot.block.leaf.LeafBlock.ValueLocation)2 SnapshotGroup (io.datarouter.filesystem.snapshot.group.SnapshotGroup)2 SnapshotKey (io.datarouter.filesystem.snapshot.key.SnapshotKey)2 ScanningSnapshotReader (io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader)2 SnapshotKeyReader (io.datarouter.filesystem.snapshot.reader.SnapshotKeyReader)2 BlockLoader (io.datarouter.filesystem.snapshot.reader.block.BlockLoader)2 SnapshotLeafRecord (io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord)2 ParallelScannerContext (io.datarouter.scanner.ParallelScannerContext)2 Scanner (io.datarouter.scanner.Scanner)2