Search in sources :

Example 1 with Ref

use of io.datarouter.scanner.Ref in project datarouter by hotpads.

the class WordTests method testValueBlockV1.

@Test
public void testValueBlockV1() {
    Supplier<ValueBlockV1Encoder> encoderSupplier = ValueBlockV1Encoder::new;
    Ref<ValueBlockV1Encoder> encoder = new Ref<>(encoderSupplier.get());
    int blockSize = 4096;
    List<String> inputs = WordDataset.scanWords(getClass().getSimpleName() + "-testValueBlockV1").list();
    List<byte[]> blocks = Scanner.of(inputs).map(str -> str.getBytes(StandardCharsets.UTF_8)).map(value -> new SnapshotEntry(EmptyArray.BYTE, EmptyArray.BYTE, new byte[][] { value })).concat(value -> {
        encoder.get().add(value, 0);
        if (encoder.get().numBytes() >= blockSize) {
            byte[] block = encoder.get().encode().concat();
            encoder.set(encoderSupplier.get());
            return Scanner.of(block);
        }
        return Scanner.empty();
    }).list();
    if (encoder.get().numRecords() > 0) {
        blocks.add(encoder.get().encode().concat());
    }
    logger.warn("encoded {} value blocks", blocks.size());
    List<String> outputs = Scanner.of(blocks).map(ValueBlockV1::new).concat(ValueBlock::valueCopies).map(bytes -> new String(bytes, StandardCharsets.UTF_8)).list();
    Require.equals(outputs.size(), inputs.size());
    for (int i = 0; i < inputs.size(); ++i) {
        if (ObjectTool.notEquals(outputs.get(i), inputs.get(i))) {
            logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
            String message = String.format("actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).getBytes(StandardCharsets.UTF_8)), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).getBytes(StandardCharsets.UTF_8)));
            throw new IllegalArgumentException(message);
        }
    }
}
Also used : ValueBlockV1(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1) RetainingGroup(io.datarouter.scanner.RetainingGroup) Scanner(io.datarouter.scanner.Scanner) Arrays(java.util.Arrays) LeafBlockV1Encoder(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1Encoder) FileIdsAndEndings(io.datarouter.filesystem.snapshot.writer.BlockQueue.FileIdsAndEndings) ByteTool(io.datarouter.bytes.ByteTool) ObjectTool(io.datarouter.util.lang.ObjectTool) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) Supplier(java.util.function.Supplier) LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) EmptyArray(io.datarouter.bytes.EmptyArray) Assert(org.testng.Assert) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Logger(org.slf4j.Logger) CsvIntByteStringCodec(io.datarouter.bytes.codec.bytestringcodec.CsvIntByteStringCodec) ValueBlockV1Encoder(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder) SnapshotEntry(io.datarouter.filesystem.snapshot.entry.SnapshotEntry) LeafBlockV1(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1) StandardCharsets(java.nio.charset.StandardCharsets) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) Ref(io.datarouter.scanner.Ref) Require(io.datarouter.util.Require) Ref(io.datarouter.scanner.Ref) SnapshotEntry(io.datarouter.filesystem.snapshot.entry.SnapshotEntry) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) ValueBlockV1Encoder(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder) Test(org.testng.annotations.Test)

Example 2 with Ref

use of io.datarouter.scanner.Ref in project datarouter by hotpads.

the class WordTests method testLeafBlockV1.

@Test
public void testLeafBlockV1() {
    Supplier<LeafBlockV1Encoder> encoderSupplier = () -> new LeafBlockV1Encoder(32 * 1024);
    Ref<LeafBlockV1Encoder> encoder = new Ref<>(encoderSupplier.get());
    int blockSize = 4096;
    String valuePrefix = "val_";
    List<SnapshotEntry> inputs = WordDataset.scanWords(getClass().getSimpleName() + "-testLeafBlockV1").map(word -> {
        byte[] keyBytes = word.getBytes(StandardCharsets.UTF_8);
        byte[] valueBytes = ByteTool.concat(valuePrefix.getBytes(StandardCharsets.UTF_8), keyBytes);
        return new SnapshotEntry(keyBytes, valueBytes, ByteTool.EMPTY_ARRAY_2);
    }).list();
    var keyId = new AtomicLong();
    List<byte[]> blocks = Scanner.of(inputs).concat(entry -> {
        // TODO use real value block references
        encoder.get().add(0, keyId.getAndIncrement(), entry, new int[] { 0 }, new int[] { 0 });
        if (encoder.get().numBytes() >= blockSize) {
            var fileIdsAndEndings = new FileIdsAndEndings[] { new FileIdsAndEndings(new int[] { 0 }, new int[] { 0 }) };
            byte[] block = encoder.get().encode(fileIdsAndEndings).concat();
            encoder.set(encoderSupplier.get());
            return Scanner.of(block);
        }
        return Scanner.empty();
    }).list();
    if (encoder.get().numRecords() > 0) {
        var fileIdsAndEndings = new FileIdsAndEndings[] { new FileIdsAndEndings(new int[] { 0 }, new int[] { 0 }) };
        blocks.add(encoder.get().encode(fileIdsAndEndings).concat());
    }
    logger.warn("encoded {} key blocks", blocks.size());
    List<SnapshotLeafRecord> outputs = Scanner.of(blocks).map(LeafBlockV1::new).concat(LeafBlock::leafRecords).list();
    Require.equals(outputs.size(), inputs.size());
    for (int i = 0; i < outputs.size(); ++i) {
        if (!Arrays.equals(outputs.get(i).key, inputs.get(i).key())) {
            logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
            String message = String.format("key: actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).key), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).key()));
            throw new IllegalArgumentException(message);
        }
        if (!Arrays.equals(outputs.get(i).value, inputs.get(i).value())) {
            logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
            String message = String.format("value: actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).value), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).value()));
            throw new IllegalArgumentException(message);
        }
    }
}
Also used : ValueBlockV1(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1) RetainingGroup(io.datarouter.scanner.RetainingGroup) Scanner(io.datarouter.scanner.Scanner) Arrays(java.util.Arrays) LeafBlockV1Encoder(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1Encoder) FileIdsAndEndings(io.datarouter.filesystem.snapshot.writer.BlockQueue.FileIdsAndEndings) ByteTool(io.datarouter.bytes.ByteTool) ObjectTool(io.datarouter.util.lang.ObjectTool) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) Supplier(java.util.function.Supplier) LeafBlock(io.datarouter.filesystem.snapshot.block.leaf.LeafBlock) EmptyArray(io.datarouter.bytes.EmptyArray) Assert(org.testng.Assert) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Logger(org.slf4j.Logger) CsvIntByteStringCodec(io.datarouter.bytes.codec.bytestringcodec.CsvIntByteStringCodec) ValueBlockV1Encoder(io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder) SnapshotEntry(io.datarouter.filesystem.snapshot.entry.SnapshotEntry) LeafBlockV1(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1) StandardCharsets(java.nio.charset.StandardCharsets) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) ValueBlock(io.datarouter.filesystem.snapshot.block.value.ValueBlock) Ref(io.datarouter.scanner.Ref) Require(io.datarouter.util.Require) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Ref(io.datarouter.scanner.Ref) AtomicLong(java.util.concurrent.atomic.AtomicLong) LeafBlockV1Encoder(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1Encoder) SnapshotEntry(io.datarouter.filesystem.snapshot.entry.SnapshotEntry) LeafBlockV1(io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1) FileIdsAndEndings(io.datarouter.filesystem.snapshot.writer.BlockQueue.FileIdsAndEndings) Test(org.testng.annotations.Test)

Aggregations

ByteTool (io.datarouter.bytes.ByteTool)2 EmptyArray (io.datarouter.bytes.EmptyArray)2 CsvIntByteStringCodec (io.datarouter.bytes.codec.bytestringcodec.CsvIntByteStringCodec)2 LeafBlock (io.datarouter.filesystem.snapshot.block.leaf.LeafBlock)2 LeafBlockV1 (io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1)2 LeafBlockV1Encoder (io.datarouter.filesystem.snapshot.block.leaf.LeafBlockV1Encoder)2 ValueBlock (io.datarouter.filesystem.snapshot.block.value.ValueBlock)2 ValueBlockV1 (io.datarouter.filesystem.snapshot.block.value.ValueBlockV1)2 ValueBlockV1Encoder (io.datarouter.filesystem.snapshot.block.value.ValueBlockV1Encoder)2 SnapshotEntry (io.datarouter.filesystem.snapshot.entry.SnapshotEntry)2 SnapshotLeafRecord (io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord)2 FileIdsAndEndings (io.datarouter.filesystem.snapshot.writer.BlockQueue.FileIdsAndEndings)2 Ref (io.datarouter.scanner.Ref)2 RetainingGroup (io.datarouter.scanner.RetainingGroup)2 Scanner (io.datarouter.scanner.Scanner)2 Require (io.datarouter.util.Require)2 ObjectTool (io.datarouter.util.lang.ObjectTool)2 StandardCharsets (java.nio.charset.StandardCharsets)2 Arrays (java.util.Arrays)2 List (java.util.List)2