Examples with BlockLoader io.datarouter.filesystem.snapshot.reader.block.BlockLoader used on opensource projects

Example 1 with BlockLoader

use of io.datarouter.filesystem.snapshot.reader.block.BlockLoader in project datarouter by hotpads.

the class BaseSnapshotTests method testScan.

@Test
public void testScan() {
    if (!ENABLED_TESTS.contains(TestId.SCAN)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    List<SnapshotRecord> outputs = reader.scan(0).list();
    Assert.assertEquals(outputs.size(), sortedInputs.size());
    for (int i = 0; i < sortedInputs.size(); ++i) {
        Input input = sortedInputs.get(i);
        SnapshotRecord output = outputs.get(i);
        Assert.assertEquals(i, output.id);
        for (int column = 0; column < input.entry.columnValues.length; ++column) {
            if (!SnapshotEntry.equalColumnValue(input.entry, output.entry(), column)) {
                String message = String.format("%s, actual=%s, expected=%s", i, utf8(output.columnValues[column]), utf8(input.entry.columnValues[column]));
                throw new RuntimeException(message);
            }
        }
    }
}

Example 2 with BlockLoader

use of io.datarouter.filesystem.snapshot.reader.block.BlockLoader in project datarouter by hotpads.

the class BaseSnapshotTests method testSearches.

@Test
public void testSearches() {
    if (!ENABLED_TESTS.contains(TestId.SEARCHES)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    int step = 1000;
    int limit = 1000;
    Scanner.iterate(0, fromId -> fromId += step).advanceWhile(fromId -> fromId < sortedInputs.size() - limit).parallel(new ParallelScannerContext(scanExec, getNumThreads(), true)).forEach(fromId -> {
        var idReader = new SnapshotIdReader(snapshotKey, blockLoader);
        // known first key inclusive
        byte[] searchKey = idReader.getRecord(fromId).key;
        List<SnapshotLeafRecord> outputsInclusive = reader.scanLeafRecords(searchKey, true).limit(limit).list();
        for (int i = 0; i < limit; ++i) {
            Input input = sortedInputs.get(fromId + i);
            SnapshotLeafRecord output = outputsInclusive.get(i);
            Assert.assertEquals(fromId + i, output.id);
            Assert.assertEquals(new Bytes(input.entry.key()), new Bytes(output.key));
        }
        // known first key exclusive
        List<SnapshotLeafRecord> outputsExclusive = reader.scanLeafRecords(searchKey, false).limit(limit).list();
        for (int i = 0; i < limit; ++i) {
            // plus one because exclusive
            Input input = sortedInputs.get(fromId + i + 1);
            SnapshotLeafRecord output = outputsExclusive.get(i);
            Assert.assertEquals(input.id, output.id);
            Assert.assertEquals(new Bytes(input.entry.key()), new Bytes(output.key));
        }
        // fake first key (should act like exclusive)
        byte[] nonExistentKey = ByteTool.concat(searchKey, new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0 });
        List<SnapshotLeafRecord> outputsNonExistentKey = reader.scanLeafRecords(nonExistentKey, true).limit(limit).list();
        for (int i = 0; i < limit; ++i) {
            // plus one because the first key didn't exist
            Input input = sortedInputs.get(fromId + i + 1);
            SnapshotLeafRecord output = outputsNonExistentKey.get(i);
            Assert.assertEquals(input.id, output.id);
            Assert.assertEquals(new Bytes(input.entry.key()), new Bytes(output.key));
        }
    });
}

Example 3 with BlockLoader

use of io.datarouter.filesystem.snapshot.reader.block.BlockLoader in project datarouter by hotpads.

the class BaseSnapshotTests method testScanValues.

@Test
public void testScanValues() {
    if (!ENABLED_TESTS.contains(TestId.SCAN_VALUES)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    List<byte[]> actuals = reader.scanValues().list();
    Assert.assertEquals(actuals.size(), sortedInputs.size());
    for (int i = 0; i < sortedInputs.size(); ++i) {
        Input input = sortedInputs.get(i);
        Assert.assertEquals(input.entry.value(), actuals.get(i));
    }
}

Example 4 with BlockLoader

use of io.datarouter.filesystem.snapshot.reader.block.BlockLoader in project datarouter by hotpads.

the class BaseSnapshotTests method testScanColumnValues.

@Test
public void testScanColumnValues() {
    if (!ENABLED_TESTS.contains(TestId.SCAN_COLUMN_VALUES)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    IntStream.range(0, NUM_COLUMNS).forEach(column -> {
        List<byte[]> actuals = reader.scanColumnValues(column).list();
        Assert.assertEquals(actuals.size(), sortedInputs.size());
        for (int i = 0; i < sortedInputs.size(); ++i) {
            Input input = sortedInputs.get(i);
            // TODO test all values after reader returns them
            if (!Arrays.equals(input.entry.columnValues[column], actuals.get(i))) {
                String message = String.format("%s, actual=%s, expected=%s", i, utf8(actuals.get(i)), utf8(input.entry.columnValues[column]));
                throw new RuntimeException(message);
            }
        }
    });
}

Example 5 with BlockLoader

use of io.datarouter.filesystem.snapshot.reader.block.BlockLoader in project datarouter by hotpads.

the class BaseSnapshotTests method testScanLeafRecords.

@Test
public void testScanLeafRecords() {
    if (!ENABLED_TESTS.contains(TestId.SCAN_LEAF_RECORDS)) {
        return;
    }
    BlockLoader blockLoader = makeBlockLoader(useMemoryCache(), shareMemoryCache());
    var reader = new ScanningSnapshotReader(snapshotKey, exec, getNumThreads(), blockLoader, SCAN_NUM_BLOCKS);
    List<SnapshotLeafRecord> actuals = reader.scanLeafRecords(0).list();
    Assert.assertEquals(actuals.size(), sortedInputs.size());
    for (int i = 0; i < sortedInputs.size(); ++i) {
        Input input = sortedInputs.get(i);
        Assert.assertEquals(input.entry.key(), actuals.get(i).key);
        Assert.assertEquals(input.entry.value(), actuals.get(i).value);
    }
}