Search in sources :

Example 1 with ValueBlock

use of io.datarouter.filesystem.snapshot.block.value.ValueBlock in project datarouter by hotpads.

the class WordTests method testValueBlockV1.

@Test
public void testValueBlockV1() {
    Supplier<ValueBlockV1Encoder> encoderSupplier = ValueBlockV1Encoder::new;
    Ref<ValueBlockV1Encoder> encoder = new Ref<>(encoderSupplier.get());
    int blockSize = 4096;
    List<String> inputs = WordDataset.scanWords(getClass().getSimpleName() + "-testValueBlockV1").list();
    List<byte[]> blocks = Scanner.of(inputs).map(str -> str.getBytes(StandardCharsets.UTF_8)).map(value -> new SnapshotEntry(EmptyArray.BYTE, EmptyArray.BYTE, new byte[][] { value })).concat(value -> {
        encoder.get().add(value, 0);
        if (encoder.get().numBytes() >= blockSize) {
            byte[] block = encoder.get().encode().concat();
            encoder.set(encoderSupplier.get());
            return Scanner.of(block);
        }
        return Scanner.empty();
    }).list();
    if (encoder.get().numRecords() > 0) {
        blocks.add(encoder.get().encode().concat());
    }
    logger.warn("encoded {} value blocks", blocks.size());
    List<String> outputs = Scanner.of(blocks).map(ValueBlockV1::new).concat(ValueBlock::valueCopies).map(bytes -> new String(bytes, StandardCharsets.UTF_8)).list();
    Require.equals(outputs.size(), inputs.size());
    for (int i = 0; i < inputs.size(); ++i) {
        if (ObjectTool.notEquals(outputs.get(i), inputs.get(i))) {
            logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
            String message = String.format("actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).getBytes(StandardCharsets.UTF_8)), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).getBytes(StandardCharsets.UTF_8)));
            throw new IllegalArgumentException(message);
        }
    }
}
Also used : ValueBlockV1(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1) RetainingGroup(io.datarouter.scanner.RetainingGroup) Scanner(io.datarouter.scanner.Scanner) Arrays(java.util.Arrays) LeafBlockV1Encoder(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1Encoder) FileIdsAndEndings(io.datarouter.filesystem.snapshot.writer.BlockQueue.FileIdsAndEndings) ByteTool(io.datarouter.bytes.ByteTool) ObjectTool(io.datarouter.util.lang.ObjectTool) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) Supplier(java.util.function.Supplier) LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) EmptyArray(io.datarouter.bytes.EmptyArray) Assert(org.testng.Assert) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Logger(org.slf4j.Logger) CsvIntByteStringCodec(io.datarouter.bytes.codec.bytestringcodec.CsvIntByteStringCodec) ValueBlockV1Encoder(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder) SnapshotEntry(io.datarouter.filesystem.snapshot.entry.SnapshotEntry) LeafBlockV1(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1) StandardCharsets(java.nio.charset.StandardCharsets) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) Ref(io.datarouter.scanner.Ref) Require(io.datarouter.util.Require) Ref(io.datarouter.scanner.Ref) SnapshotEntry(io.datarouter.filesystem.snapshot.entry.SnapshotEntry) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) ValueBlockV1Encoder(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder) Test(org.testng.annotations.Test)

Example 2 with ValueBlock

use of io.datarouter.filesystem.snapshot.block.value.ValueBlock in project datarouter by hotpads.

the class SnapshotIdReader method getRecord.

public SnapshotRecord getRecord(long recordId) {
    LeafBlock leafBlock = leafBlock(recordId);
    Bytes key = leafBlock.snapshotKey(recordId);
    Bytes value = leafBlock.snapshotValue(recordId);
    int numColumns = rootBlock.numColumns();
    byte[][] columnValues = new byte[numColumns][];
    for (int column = 0; column < numColumns; ++column) {
        ValueLocation valueLocation = leafBlock.getValueBlock(column, recordId);
        BlockKey valueBlockKey = leafBlock.valueBlockKey(snapshotKey, column, valueLocation.valueBlockId);
        ValueBlock valueBlock = blockLoader.value(valueBlockKey);
        Bytes columnValue = valueBlock.value(valueLocation.valueIndex);
        columnValues[column] = columnValue.toArray();
    }
    return new SnapshotRecord(recordId, key.toArray(), value.toArray(), columnValues);
}
Also used : Bytes(io.datarouter.bytes.Bytes) LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) ValueLocation(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock.ValueLocation) BlockKey(io.datarouter.filesystem.snapshot.block.BlockKey)

Example 3 with ValueBlock

use of io.datarouter.filesystem.snapshot.block.value.ValueBlock in project datarouter by hotpads.

the class SnapshotKeyReader method findRecord.

public Optional<SnapshotRecord> findRecord(byte[] searchKey) {
    LeafBlock leafBlock = leafBlock(searchKey);
    Optional<Long> optRecordId = leafBlock.findRecordId(searchKey);
    if (optRecordId.isEmpty()) {
        return Optional.empty();
    }
    long recordId = optRecordId.get();
    byte[] value = leafBlock.snapshotValue(recordId).toArray();
    int numColumns = rootBlock.numColumns();
    byte[][] columnValues = new byte[numColumns][];
    for (int column = 0; column < numColumns; ++column) {
        // TODO lookup value blocks using already-found keyId
        Optional<ValueLocation> optValueLocation = leafBlock.findValueBlock(column, searchKey);
        if (optValueLocation.isEmpty()) {
            return Optional.empty();
        }
        ValueLocation valueLocation = optValueLocation.get();
        BlockKey valueBlockKey = leafBlock.valueBlockKey(snapshotKey, column, valueLocation.valueBlockId);
        ValueBlock valueBlock = blockLoader.value(valueBlockKey);
        Bytes columnValue = valueBlock.value(valueLocation.valueIndex);
        columnValues[column] = columnValue.toArray();
    }
    return Optional.of(new SnapshotRecord(recordId, searchKey, value, columnValues));
}
Also used : LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) ValueLocation(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock.ValueLocation) BlockKey(io.datarouter.filesystem.snapshot.block.BlockKey) Bytes(io.datarouter.bytes.Bytes) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock)

Example 4 with ValueBlock

use of io.datarouter.filesystem.snapshot.block.value.ValueBlock in project datarouter by hotpads.

the class LeafBlockWithValueBlocks method scan.

public Scanner<SnapshotRecord> scan(int fromRecordIdInclusive) {
    return Scanner.iterate(0, recordIndex -> recordIndex + 1).limit(leafBlock.numRecords()).include(recordIndex -> leafBlock.recordId(recordIndex) >= fromRecordIdInclusive).map(recordIndex -> {
        long recordId = leafBlock.firstRecordId() + recordIndex;
        byte[] key = leafBlock.blockKey(recordIndex).toArray();
        byte[] value = leafBlock.blockValue(recordIndex).toArray();
        byte[][] columnValues = new byte[rootBlock.numColumns()][];
        for (int column = 0; column < rootBlock.numColumns(); ++column) {
            // TODO improve LeafBlock methods to skip looking up valueBlockOffset
            int valueBlockOffset = leafBlock.valueBlockOffsetForKey(column, recordIndex);
            int valueBlockId = leafBlock.firstValueBlockId(column) + valueBlockOffset;
            ValueBlock valueBlock = getValueBlock(column, valueBlockId);
            int valueIndex = leafBlock.valueIndex(column, valueBlockOffset, recordIndex);
            columnValues[column] = valueBlock.value(valueIndex).toArray();
        }
        return new SnapshotRecord(recordId, key, value, columnValues);
    });
}
Also used : IntStream(java.util.stream.IntStream) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) Scanner(io.datarouter.scanner.Scanner) List(java.util.List) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) RootBlock(io.datarouter.filesystem.snapshot.block.root.RootBlock) LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock)

Aggregations

LeafBlock (io.datarouter.filesystem.snapshot.block.leaf.LeafBlock)4 ValueBlock (io.datarouter.filesystem.snapshot.block.value.ValueBlock)4 SnapshotRecord (io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord)3 Bytes (io.datarouter.bytes.Bytes)2 BlockKey (io.datarouter.filesystem.snapshot.block.BlockKey)2 ValueLocation (io.datarouter.filesystem.snapshot.block.leaf.LeafBlock.ValueLocation)2 Scanner (io.datarouter.scanner.Scanner)2 List (java.util.List)2 ByteTool (io.datarouter.bytes.ByteTool)1 EmptyArray (io.datarouter.bytes.EmptyArray)1 CsvIntByteStringCodec (io.datarouter.bytes.codec.bytestringcodec.CsvIntByteStringCodec)1 LeafBlockV1 (io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1)1 LeafBlockV1Encoder (io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1Encoder)1 RootBlock (io.datarouter.filesystem.snapshot.block.root.RootBlock)1 ValueBlockV1 (io.datarouter.filesystem.snapshot.block.value.ValueBlockV1)1 ValueBlockV1Encoder (io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder)1 SnapshotEntry (io.datarouter.filesystem.snapshot.entry.SnapshotEntry)1 SnapshotLeafRecord (io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord)1 FileIdsAndEndings (io.datarouter.filesystem.snapshot.writer.BlockQueue.FileIdsAndEndings)1 Ref (io.datarouter.scanner.Ref)1