Examples with ValueBlockV1 io.datarouter.filesystem.snapshot.block.value.ValueBlockV1 used on opensource projects

Example 1 with ValueBlockV1

use of io.datarouter.filesystem.snapshot.block.value.ValueBlockV1 in project datarouter by hotpads.

the class WordTests method testValueBlockV1.

@Test
public void testValueBlockV1() {
    Supplier<ValueBlockV1Encoder> encoderSupplier = ValueBlockV1Encoder::new;
    Ref<ValueBlockV1Encoder> encoder = new Ref<>(encoderSupplier.get());
    int blockSize = 4096;
    List<String> inputs = WordDataset.scanWords(getClass().getSimpleName() + "-testValueBlockV1").list();
    List<byte[]> blocks = Scanner.of(inputs).map(str -> str.getBytes(StandardCharsets.UTF_8)).map(value -> new SnapshotEntry(EmptyArray.BYTE, EmptyArray.BYTE, new byte[][] { value })).concat(value -> {
        encoder.get().add(value, 0);
        if (encoder.get().numBytes() >= blockSize) {
            byte[] block = encoder.get().encode().concat();
            encoder.set(encoderSupplier.get());
            return Scanner.of(block);
        }
        return Scanner.empty();
    }).list();
    if (encoder.get().numRecords() > 0) {
        blocks.add(encoder.get().encode().concat());
    }
    logger.warn("encoded {} value blocks", blocks.size());
    List<String> outputs = Scanner.of(blocks).map(ValueBlockV1::new).concat(ValueBlock::valueCopies).map(bytes -> new String(bytes, StandardCharsets.UTF_8)).list();
    Require.equals(outputs.size(), inputs.size());
    for (int i = 0; i < inputs.size(); ++i) {
        if (ObjectTool.notEquals(outputs.get(i), inputs.get(i))) {
            logger.warn("actual=[{}] expected=[{}]", outputs.get(i), inputs.get(i));
            String message = String.format("actual=[%s] does not equal expected=[%s]", CsvIntByteStringCodec.INSTANCE.encode(outputs.get(i).getBytes(StandardCharsets.UTF_8)), CsvIntByteStringCodec.INSTANCE.encode(inputs.get(i).getBytes(StandardCharsets.UTF_8)));
            throw new IllegalArgumentException(message);
        }
    }
}