use of io.datarouter.filesystem.snapshot.block.leaf.LeafBlock in project datarouter by hotpads.
the class WordTests method testLeafBlockV1.
@Test
public void testLeafBlockV1() {
Supplier<LeafBlockV1Encoder> encoderSupplier = () -> new LeafBlockV1Encoder(32 * 1024);
Ref<LeafBlockV1Encoder> encoder = new Ref<>(encoderSupplier.get());
int blockSize = 4096;
String valuePrefix = "val_";
List<SnapshotEntry> inputs = WordDataset.scanWords(getClass().getSimpleName() + "-testLeafBlockV1").map(word -> {
byte[] keyBytes = word.getBytes(StandardCharsets.UTF_8);
byte[] valueBytes = ByteTool.concat(valuePrefix.getBytes(StandardCharsets.UTF_8), keyBytes);
return new SnapshotEntry(keyBytes, valueBytes, ByteTool.EMPTY_ARRAY_2);
}).list();
var keyId = new AtomicLong();
List<byte[]> blocks = Scanner.of(inputs).concat(entry -> {
// TODO use real value block references
encoder.get().add(0, keyId.getAndIncrement(), entry, new int[] { 0 }, new int[] { 0 });
if (encoder.get().numBytes() >= blockSize) {
var fileIdsAndEndings = new FileIdsAndEndings[] { new FileIdsAndEndings(new int[] { 0 }, new int[] { 0 }) };
byte[] block = encoder.get().encode(fileIdsAndEndings).concat();
encoder.set(encoderSupplier.get());
return Scanner.of(block);
}
return Scanner.empty();
}).list();
if (encoder.get().numRecords() > 0) {
var fileIdsAndEndings = new FileIdsAndEndings[] { new FileIdsAndEndings(new int[] { 0 }, new int[] { 0 }) };
blocks.add(encoder.get().encode(fileIdsAndEndings).concat());
}
logger.warn("encoded {} key blocks", blocks.size());
List<SnapshotLeafRecord> outputs = Scanner.of(blocks).map(LeafBlockV1::new).concat(LeafBlock::leafRecords).list();
Require.equals(outputs.size(), inputs.size());
for (int i = 0; i < outputs.size(); ++i) {
if (!Arrays.equals(outputs.get(i).key, inputs.get(i).key())) {
logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
String message = String.format("key: actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).key), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).key()));
throw new IllegalArgumentException(message);
}
if (!Arrays.equals(outputs.get(i).value, inputs.get(i).value())) {
logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
String message = String.format("value: actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).value), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).value()));
throw new IllegalArgumentException(message);
}
}
}
use of io.datarouter.filesystem.snapshot.block.leaf.LeafBlock in project datarouter by hotpads.
the class SnapshotIdReader method getRecord.
public SnapshotRecord getRecord(long recordId) {
LeafBlock leafBlock = leafBlock(recordId);
Bytes key = leafBlock.snapshotKey(recordId);
Bytes value = leafBlock.snapshotValue(recordId);
int numColumns = rootBlock.numColumns();
byte[][] columnValues = new byte[numColumns][];
for (int column = 0; column < numColumns; ++column) {
ValueLocation valueLocation = leafBlock.getValueBlock(column, recordId);
BlockKey valueBlockKey = leafBlock.valueBlockKey(snapshotKey, column, valueLocation.valueBlockId);
ValueBlock valueBlock = blockLoader.value(valueBlockKey);
Bytes columnValue = valueBlock.value(valueLocation.valueIndex);
columnValues[column] = columnValue.toArray();
}
return new SnapshotRecord(recordId, key.toArray(), value.toArray(), columnValues);
}
use of io.datarouter.filesystem.snapshot.block.leaf.LeafBlock in project datarouter by hotpads.
the class SnapshotKeyReader method findRecord.
public Optional<SnapshotRecord> findRecord(byte[] searchKey) {
LeafBlock leafBlock = leafBlock(searchKey);
Optional<Long> optRecordId = leafBlock.findRecordId(searchKey);
if (optRecordId.isEmpty()) {
return Optional.empty();
}
long recordId = optRecordId.get();
byte[] value = leafBlock.snapshotValue(recordId).toArray();
int numColumns = rootBlock.numColumns();
byte[][] columnValues = new byte[numColumns][];
for (int column = 0; column < numColumns; ++column) {
// TODO lookup value blocks using already-found keyId
Optional<ValueLocation> optValueLocation = leafBlock.findValueBlock(column, searchKey);
if (optValueLocation.isEmpty()) {
return Optional.empty();
}
ValueLocation valueLocation = optValueLocation.get();
BlockKey valueBlockKey = leafBlock.valueBlockKey(snapshotKey, column, valueLocation.valueBlockId);
ValueBlock valueBlock = blockLoader.value(valueBlockKey);
Bytes columnValue = valueBlock.value(valueLocation.valueIndex);
columnValues[column] = columnValue.toArray();
}
return Optional.of(new SnapshotRecord(recordId, searchKey, value, columnValues));
}
use of io.datarouter.filesystem.snapshot.block.leaf.LeafBlock in project datarouter by hotpads.
the class LeafBlockWithValueBlocks method scan.
public Scanner<SnapshotRecord> scan(int fromRecordIdInclusive) {
return Scanner.iterate(0, recordIndex -> recordIndex + 1).limit(leafBlock.numRecords()).include(recordIndex -> leafBlock.recordId(recordIndex) >= fromRecordIdInclusive).map(recordIndex -> {
long recordId = leafBlock.firstRecordId() + recordIndex;
byte[] key = leafBlock.blockKey(recordIndex).toArray();
byte[] value = leafBlock.blockValue(recordIndex).toArray();
byte[][] columnValues = new byte[rootBlock.numColumns()][];
for (int column = 0; column < rootBlock.numColumns(); ++column) {
// TODO improve LeafBlock methods to skip looking up valueBlockOffset
int valueBlockOffset = leafBlock.valueBlockOffsetForKey(column, recordIndex);
int valueBlockId = leafBlock.firstValueBlockId(column) + valueBlockOffset;
ValueBlock valueBlock = getValueBlock(column, valueBlockId);
int valueIndex = leafBlock.valueIndex(column, valueBlockOffset, recordIndex);
columnValues[column] = valueBlock.value(valueIndex).toArray();
}
return new SnapshotRecord(recordId, key, value, columnValues);
});
}
use of io.datarouter.filesystem.snapshot.block.leaf.LeafBlock in project datarouter by hotpads.
the class ScanningSnapshotReader method scanLeafRecords.
public Scanner<SnapshotLeafRecord> scanLeafRecords(byte[] startKey, boolean inclusive) {
var keyReader = new SnapshotKeyReader(snapshotKey, blockLoader);
LeafBlock leafBlock = keyReader.leafBlock(startKey);
SnapshotLeafSearchResult searchResult = leafBlock.search(startKey);
long fromRecordIdInclusive = searchResult.recordId(inclusive);
return scanLeafRecords(fromRecordIdInclusive);
}
Aggregations