Examples with ScanningSnapshotReader io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader used on opensource projects

Example 6 with ScanningSnapshotReader

use of io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader in project datarouter by hotpads.

the class BaseSnapshotTests method testScanColumnValues.

@Test
public void testScanColumnValues() {
    if (!ENABLED_TESTS.contains(TestId.SCAN_COLUMN_VALUES)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    IntStream.range(0, NUM_COLUMNS).forEach(column -> {
        List<byte[]> actuals = reader.scanColumnValues(column).list();
        Assert.assertEquals(actuals.size(), sortedInputs.size());
        for (int i = 0; i < sortedInputs.size(); ++i) {
            Input input = sortedInputs.get(i);
            // TODO test all values after reader returns them
            if (!Arrays.equals(input.entry.columnValues[column], actuals.get(i))) {
                String message = String.format("%s, actual=%s, expected=%s", i, utf8(actuals.get(i)), utf8(input.entry.columnValues[column]));
                throw new RuntimeException(message);
            }
        }
    });
}

Example 7 with ScanningSnapshotReader

use of io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader in project datarouter by hotpads.

the class BaseSnapshotTests method testScanLeafRecords.

@Test
public void testScanLeafRecords() {
    if (!ENABLED_TESTS.contains(TestId.SCAN_LEAF_RECORDS)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    List<SnapshotLeafRecord> actuals = reader.scanLeafRecords(0).list();
    Assert.assertEquals(actuals.size(), sortedInputs.size());
    for (int i = 0; i < sortedInputs.size(); ++i) {
        Input input = sortedInputs.get(i);
        Assert.assertEquals(input.entry.key(), actuals.get(i).key);
        Assert.assertEquals(input.entry.value(), actuals.get(i).value);
    }
}

Example 8 with ScanningSnapshotReader

use of io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader in project datarouter by hotpads.

the class FilesystemSnapshotLargeTests method testScan.

@Test
public void testScan() {
    if (!PERSIST) {
        return;
    }
    BlockLoader blockLoader = SCAN_WITH_CACHE ? cache : group;
    int numThreads = Runtime.getRuntime().availableProcessors();
    var exec = Executors.newFixedThreadPool(numThreads);
    var reader = new ScanningSnapshotReader(benchmark.snapshotKey, exec, numThreads, blockLoader, SCAN_NUM_BLOCKS);
    var count = new AtomicLong();
    reader.scan(0).forEach(record -> {
        long id = count.getAndIncrement();
        Assert.assertEquals(record.id, id);
        Assert.assertEquals(record.key, SnapshotBenchmark.makeKey(id));
        Assert.assertEquals(record.value, SnapshotBenchmark.makeValue(id));
    });
    Assert.assertEquals(count.get(), benchmark.numEntries);
    ExecutorServiceTool.shutdown(exec, Duration.ofSeconds(2));
}

Example 9 with ScanningSnapshotReader

use of io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader in project datarouter by hotpads.

the class FilesystemSnapshotSortingTests method testSorting.

@Test
public void testSorting() {
    var timer = new PhaseTimer("testSorting");
    // write unsorted snapshot
    writeInputSnapshot();
    RootBlock inputRootBlock = inputGroup.root(BlockKey.root(inputSnapshotKey));
    Assert.assertEquals(inputRootBlock.numItems(), NUM_ENTRIES);
    timer.add("writeInputSnapshot");
    // write sorted chunks
    var inputReader = new ScanningSnapshotReader(inputSnapshotKey, exec, numThreads, inputGroup, SCAN_NUM_BLOCKS);
    var chunkId = new AtomicInteger();
    chunkSnapshotKeys = inputReader.scan(0).batch(CHUNK_SIZE).parallel(new ParallelScannerContext(sortExec, numThreads, true)).map(batch -> {
        Collections.sort(batch, SnapshotRecord.KEY_COMPARATOR);
        return writeChunkSnapshot(chunkId.getAndIncrement(), batch);
    }).list();
    timer.add("writeSortedChunks");
    RootBlock outputRootBlock = Scanner.of(chunkSnapshotKeys).map(chunkKey -> new ScanningSnapshotReader(chunkKey, exec, numThreads, chunkGroup, SCAN_NUM_BLOCKS)).collate(reader -> reader.scan(0), SnapshotRecord.KEY_COMPARATOR).apply(this::writeOutputSnapshot);
    Assert.assertEquals(outputRootBlock.numItems(), NUM_ENTRIES);
    timer.add("writeOutputSnapshot");
    var outputReader = new ScanningSnapshotReader(outputSnapshotKey, exec, numThreads, outputGroup, SCAN_NUM_BLOCKS);
    var outputCount = new Count("output");
    outputReader.scanKeys().map(FilesystemSnapshotSortingTests::parseKey).each(id -> Assert.assertEquals(id.intValue(), outputCount.intValue())).forEach(outputCount::increment);
    timer.add("assert output sorted");
    logger.warn("{}", timer);
}

Example 10 with ScanningSnapshotReader

use of io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader in project datarouter by hotpads.

the class BaseSnapshotTests method testScanKeys.

@Test
public void testScanKeys() {
    if (!ENABLED_TESTS.contains(TestId.SCAN_KEYS)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    List<byte[]> actuals = reader.scanKeys().list();
    Assert.assertEquals(actuals.size(), sortedInputs.size());
    for (int i = 0; i < sortedInputs.size(); ++i) {
        Input input = sortedInputs.get(i);
        Assert.assertEquals(input.entry.key(), actuals.get(i));
    }
}