Examples with GrowableByteBuffer com.yahoo.io.GrowableByteBuffer used on opensource projects

Example 61 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class ProgressToken method serialize.

public byte[] serialize() {
    DocumentSerializer out = DocumentSerializerFactory.create42(new GrowableByteBuffer());
    out.putInt(null, distributionBits);
    out.putLong(null, bucketCursor);
    out.putLong(null, finishedBucketCount);
    out.putLong(null, totalBucketCount);
    out.putInt(null, buckets.size());
    // Append individual bucket progress
    for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) {
        out.putLong(null, keyToBucketId(entry.getKey().getKey()));
        out.putLong(null, entry.getValue().getProgress().getRawId());
    }
    byte[] ret = new byte[out.getBuf().position()];
    out.getBuf().rewind();
    out.getBuf().get(ret);
    return ret;
}

Example 62 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentGenPluginTest method roundtripSerialize.

private static Document roundtripSerialize(Document docToSerialize, DocumentTypeManager mgr) {
    final GrowableByteBuffer outputBuffer = new GrowableByteBuffer();
    final DocumentSerializer serializer = DocumentSerializerFactory.createHead(outputBuffer);
    serializer.write(docToSerialize);
    outputBuffer.flip();
    return new Document(DocumentDeserializerFactory.createHead(mgr, outputBuffer));
}

Example 63 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class VespaDocumentDeserializer42 method read.

public void read(FieldBase fieldDef, Struct s) {
    s.setVersion(version);
    int startPos = position();
    if (version < 6) {
        throw new DeserializationException("Illegal document serialization version " + version);
    }
    int dataSize;
    if (version < 7) {
        long rSize = getInt2_4_8Bytes(null);
        // TODO: Look into how to support data segments larger than INT_MAX bytes
        if (rSize > Integer.MAX_VALUE) {
            throw new DeserializationException("Raw size of data block is too large.");
        }
        dataSize = (int) rSize;
    } else {
        dataSize = getInt(null);
    }
    byte comprCode = getByte(null);
    CompressionType compression = CompressionType.valueOf(comprCode);
    int uncompressedSize = 0;
    if (compression != CompressionType.NONE && compression != CompressionType.INCOMPRESSIBLE) {
        // uncompressedsize (full size of FIELDS only, after decompression)
        long pSize = getInt2_4_8Bytes(null);
        // TODO: Look into how to support data segments larger than INT_MAX bytes
        if (pSize > Integer.MAX_VALUE) {
            throw new DeserializationException("Uncompressed size of data block is too large.");
        }
        uncompressedSize = (int) pSize;
    }
    int numberOfFields = getInt1_4Bytes(null);
    List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields);
    for (int i = 0; i < numberOfFields; ++i) {
        // id, length (length only used for unknown fields
        fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null)));
    }
    // save a reference to the big buffer we're reading from:
    GrowableByteBuffer bigBuf = buf;
    if (version < 7) {
        // In V6 and earlier, the length included the header.
        int headerSize = position() - startPos;
        dataSize -= headerSize;
    }
    byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize));
    // set position in original buffer to after data
    position(position() + dataSize);
    // for a while: deserialize from this buffer instead:
    buf = GrowableByteBuffer.wrap(destination);
    s.clear();
    StructDataType type = s.getDataType();
    for (int i = 0; i < numberOfFields; ++i) {
        Field structField = type.getField(fieldIdsAndLengths.get(i).first, version);
        if (structField == null) {
            // ignoring unknown field:
            position(position() + fieldIdsAndLengths.get(i).second.intValue());
        } else {
            int posBefore = position();
            FieldValue value = structField.getDataType().createFieldValue();
            value.deserialize(structField, this);
            s.setFieldValue(structField, value);
            // jump to beginning of next field:
            position(posBefore + fieldIdsAndLengths.get(i).second.intValue());
        }
    }
    // restore the original buffer
    buf = bigBuf;
}

Example 64 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testCppDoc.

@Test
public void testCppDoc() throws IOException {
    docMan = setUpCppDocType();
    byte[] data = readFile("src/test/document/serializecpp.dat");
    ByteBuffer buf = ByteBuffer.wrap(data);
    Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
    validateCppDoc(doc);
}

Example 65 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testInheritance.

@Test
public void testInheritance() {
    // Create types that inherit each other.. And test that it works..
    DocumentType parentType = new DocumentType("parent");
    parentType.addField(new Field("parentbodyint", DataType.INT, false));
    parentType.addField(new Field("parentheaderint", DataType.INT, true));
    parentType.addField(new Field("overwritten", DataType.INT, true));
    DocumentType childType = new DocumentType("child");
    childType.addField(new Field("childbodyint", DataType.INT, false));
    childType.addField(new Field("childheaderint", DataType.INT, true));
    childType.addField(new Field("overwritten", DataType.INT, true));
    childType.inherit(parentType);
    DocumentTypeManager manager = new DocumentTypeManager();
    manager.register(childType);
    Document child = new Document(childType, new DocumentId("doc:what:test"));
    child.setFieldValue(childType.getField("parentbodyint"), new IntegerFieldValue(4));
    child.setFieldValue("parentheaderint", 6);
    child.setFieldValue("overwritten", 7);
    child.setFieldValue("childbodyint", 14);
    GrowableByteBuffer buffer = new GrowableByteBuffer(1024, 2f);
    child.serialize(buffer);
    buffer.flip();
    Document childCopy = manager.createDocument(buffer);
    // Test various ways of retrieving values
    assertEquals(new IntegerFieldValue(4), childCopy.getFieldValue(childType.getField("parentbodyint")));
    assertEquals(new IntegerFieldValue(6), childCopy.getFieldValue("parentheaderint"));
    assertEquals(new IntegerFieldValue(7), childCopy.getFieldValue("overwritten"));
    assertEquals(child, childCopy);
}