Examples with GrowableByteBuffer com.yahoo.io.GrowableByteBuffer used on opensource projects

Example 11 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testCppDocCompressed.

@Test
public void testCppDocCompressed() throws IOException {
    docMan = setUpCppDocType();
    byte[] data = readFile("src/test/document/serializecpp-lz4-level9.dat");
    ByteBuffer buf = ByteBuffer.wrap(data);
    Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
    validateCppDoc(doc);
}

Example 12 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testSerializeDeserialize.

@Test
public void testSerializeDeserialize() {
    setUpSertestDocType();
    Document doc = getSertestDocument();
    GrowableByteBuffer data = new GrowableByteBuffer();
    doc.serialize(data);
    int size = doc.getSerializedSize();
    assertEquals(size, data.position());
    data.flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
        fos.write(data.array(), 0, data.remaining());
        fos.close();
    } catch (Exception e) {
    }
    Document doc2 = docMan.createDocument(data);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    byte[] docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    byte[] doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    // Do the same thing, splitting document in two
    DocumentSerializer header = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true);
    DocumentSerializer body = DocumentSerializerFactory.create42(new GrowableByteBuffer());
    doc.serializeHeader(header);
    doc.serializeBody(body);
    header.getBuf().flip();
    body.getBuf().flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
        fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
        fos.close();
        fos = new FileOutputStream("src/test/files/testser-split.body.dat");
        fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
        fos.close();
    } catch (Exception e) {
    }
    DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
    doc2 = new Document(deser);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    Document docInDoc = (Document) doc.getFieldValue("docindoc");
    assert (docInDoc != null);
    assertEquals(new StringFieldValue("ball"), docInDoc.getFieldValue("tull"));
}

Example 13 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testEmptyStringsSerialization.

@Test
public void testEmptyStringsSerialization() {
    docMan = new DocumentTypeManager();
    DocumentType docType = new DocumentType("emptystrings");
    docType.addField(new Field("emptystring", DataType.STRING));
    docType.addField(new Field("nullstring", DataType.STRING));
    docType.addField(new Field("spacestring", DataType.STRING));
    docType.addField(new Field("astring", DataType.STRING));
    docMan.registerDocumentType(docType);
    GrowableByteBuffer grbuf = new GrowableByteBuffer();
    {
        Document doc = new Document(docType, new DocumentId("doc:a:b:emptystrings"));
        doc.setFieldValue("emptystring", "");
        doc.removeFieldValue("nullstring");
        doc.setFieldValue("spacestring", " ");
        doc.setFieldValue("astring", "a");
        assertEquals(new StringFieldValue(""), doc.getFieldValue("emptystring"));
        assertNull(doc.getFieldValue("nullstring"));
        assertEquals(new StringFieldValue(" "), doc.getFieldValue("spacestring"));
        assertEquals(new StringFieldValue("a"), doc.getFieldValue("astring"));
        doc.getSerializedSize();
        doc.serialize(grbuf);
        grbuf.flip();
    }
    {
        Document doc2 = docMan.createDocument(grbuf);
        assertEquals(new StringFieldValue(""), doc2.getFieldValue("emptystring"));
        assertNull(doc2.getFieldValue("nullstring"));
        assertEquals(new StringFieldValue(" "), doc2.getFieldValue("spacestring"));
        assertEquals(new StringFieldValue("a"), doc2.getFieldValue("astring"));
    }
}

Example 14 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testBug2354045.

@Test
public void testBug2354045() {
    DocumentTypeManager docMan = new DocumentTypeManager();
    DocumentType docType = new DocumentType("bug2354045");
    docType.addField(new Field("string", DataType.STRING));
    docMan.register(docType);
    GrowableByteBuffer grbuf = new GrowableByteBuffer();
    Document doc = new Document(docType, new DocumentId("doc:a:b:strings"));
    doc.removeFieldValue("string");
    assertNull(doc.getFieldValue("string"));
    doc.getSerializedSize();
    doc.serialize(grbuf);
    grbuf.flip();
    Document doc2 = docMan.createDocument(grbuf);
    assertNull(doc2.getFieldValue("string"));
    assertEquals(doc, doc2);
}

Example 15 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class DocumentTestCase method testSerializeDeserializeCompressed.

@Test
public void testSerializeDeserializeCompressed() {
    setUpSertestDocType();
    Document doc = getSertestDocument();
    CompressionConfig noncomp = new CompressionConfig();
    CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
    doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
    doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
    GrowableByteBuffer data = new GrowableByteBuffer();
    doc.serialize(data);
    int size = doc.getSerializedSize();
    doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
    doc.getDataType().getBodyType().setCompressionConfig(noncomp);
    assertEquals(size, data.position());
    data.flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
        fos.write(data.array(), 0, data.remaining());
        fos.close();
    } catch (Exception e) {
    }
    Document doc2 = docMan.createDocument(data);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    byte[] docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    byte[] doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    // Do the same thing, splitting document in two
    BufferSerializer header = new BufferSerializer(new GrowableByteBuffer());
    BufferSerializer body = new BufferSerializer(new GrowableByteBuffer());
    doc.serializeHeader(header);
    doc.serializeBody(body);
    header.getBuf().flip();
    body.getBuf().flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
        fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
        fos.close();
        fos = new FileOutputStream("src/test/files/testser-split.body.dat");
        fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
        fos.close();
    } catch (Exception e) {
    }
    DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
    doc2 = new Document(deser);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
}