Examples with GrowableByteBuffer com.yahoo.io.GrowableByteBuffer used on opensource projects

Example 56 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class SystemTestCase method testSystemTest.

public void testSystemTest() {
    DocumentType type = manager.getDocumentType("article");
    Document inDocument = new Document(type, "doc:article:boringarticle:longarticle");
    annotate(inDocument);
    GrowableByteBuffer buffer = new GrowableByteBuffer();
    DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
    serializer.write(inDocument);
    buffer.flip();
    DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
    Document outDocument = new Document(deserializer);
    consume(outDocument);
}

Example 57 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class StringTestCase method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    byte[] buf = new byte[1500];
    GrowableByteBuffer data = GrowableByteBuffer.wrap(buf);
    // short string
    java.lang.String foo = "foo";
    data.put((byte) 0);
    data.put((byte) (foo.length() + 1));
    data.put(foo.getBytes());
    data.put((byte) 0);
    int positionAfterPut = data.position();
    data.position(0);
    StringFieldValue tmp = new StringFieldValue();
    DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data);
    tmp.deserialize(deser);
    java.lang.String foo2 = tmp.getString();
    assertTrue(foo.equals(foo2));
    assertEquals(data.position(), positionAfterPut);
    // =====================
    buf = new byte[1500];
    data = GrowableByteBuffer.wrap(buf);
    // long string
    java.lang.String blah = "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah" + "blahblahblahblahblahblahblahblahblahblahblah";
    int length = blah.length() + 1;
    data.put((byte) 0);
    data.putInt(length | 0x80000000);
    data.put(blah.getBytes());
    data.put((byte) 0);
    positionAfterPut = data.position();
    data.position(0);
    tmp = new StringFieldValue();
    deser = DocumentDeserializerFactory.create42(null, data);
    tmp.deserialize(deser);
    java.lang.String blah2 = tmp.getString();
    assertEquals(data.position(), positionAfterPut);
    assertTrue(blah.equals(blah2));
}

Example 58 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class StringTestCase method serializeAndDeserialize.

private Document serializeAndDeserialize(Document doc, DocumentTypeManager manager) {
    GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
    DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
    serializer.write(doc);
    buffer.flip();
    DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
    return new Document(deserializer);
}

Example 59 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class SerializationHelperTestCase method testSerializeRawField.

public void testSerializeRawField() throws UnsupportedEncodingException {
    GrowableByteBuffer gbuf = new GrowableByteBuffer();
    ByteBuffer rawValue = ByteBuffer.wrap(Utf8.toBytes("0123456789"));
    rawValue.position(7);
    Raw value = new Raw(rawValue);
    value.serialize(gbuf);
    assertEquals(7, gbuf.position());
    assertEquals(7, rawValue.position());
    value = new Raw(rawValue);
    value.serialize(gbuf);
    assertEquals(14, gbuf.position());
    assertEquals(7, rawValue.position());
}

Example 60 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class SerializeAnnotationsTestCase method testSerializeAdvancedTree.

public void testSerializeAdvancedTree() throws IOException {
    SpanList root = new SpanList();
    SpanTree tree = new SpanTree("html", root);
    DataType positionType = docMan.getDataType("myposition");
    StructDataType cityDataType = (StructDataType) docMan.getDataType("annotation.city");
    AnnotationTypeRegistry registry = docMan.getAnnotationTypeRegistry();
    AnnotationType textType = registry.getType("text");
    AnnotationType beginTag = registry.getType("begintag");
    AnnotationType endTag = registry.getType("endtag");
    AnnotationType bodyType = registry.getType("body");
    AnnotationType paragraphType = registry.getType("paragraph");
    AnnotationType cityType = registry.getType("city");
    AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) docMan.getDataType("annotationreference<text>");
    Struct position = new Struct(positionType);
    position.setFieldValue("latitude", new DoubleFieldValue(37.774929));
    position.setFieldValue("longitude", new DoubleFieldValue(-122.419415));
    Annotation sanAnnotation = new Annotation(textType);
    Annotation franciscoAnnotation = new Annotation(textType);
    Struct positionWithRef = cityDataType.createFieldValue();
    positionWithRef.setFieldValue("position", position);
    Field referencesField = cityDataType.getField("references");
    Array<FieldValue> refList = new Array<FieldValue>(referencesField.getDataType());
    refList.add(new AnnotationReference(annRefType, sanAnnotation));
    refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
    positionWithRef.setFieldValue(referencesField, refList);
    Annotation city = new Annotation(cityType, positionWithRef);
    AlternateSpanList paragraph = new AlternateSpanList();
    paragraph.addChildren(new ArrayList<SpanNode>(), 0);
    paragraph.setProbability(0, 0.9);
    paragraph.setProbability(1, 0.1);
    {
        Span span1 = new Span(6, 3);
        Span span2 = new Span(9, 10);
        Span span3 = new Span(19, 4);
        Span span4 = new Span(23, 4);
        paragraph.add(0, span1).add(0, span2).add(0, span3).add(0, span4);
        Span alt_span1 = new Span(6, 13);
        Span alt_span2 = new Span(19, 8);
        paragraph.add(1, alt_span1).add(1, alt_span2);
        tree.annotate(span1, beginTag).annotate(span2, textType).annotate(span3, sanAnnotation).annotate(span4, endTag).annotate(alt_span1, textType).annotate(alt_span2, bodyType).annotate(paragraph, paragraphType);
    }
    {
        Span span1 = new Span(0, 6);
        Span span2 = new Span(27, 9);
        Span span3 = new Span(36, 8);
        root.add(span1).add(paragraph).add(span2).add(span3);
        tree.annotate(span1, beginTag).annotate(span2, franciscoAnnotation).annotate(span3, endTag).annotate(root, bodyType).annotate(city);
    }
    StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " + "lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " + "lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " + "oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " + "oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" + "aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
    value.setSpanTree(tree);
    // important! call readFile() before writeFile()!
    ByteBuffer serializedFromFile = readFile("test_data_serialized_advanced");
    ByteBuffer serialized = writeFile(value, "test_data_serialized_advanced");
    assertEquals(serialized.limit(), serializedFromFile.limit());
    StringFieldValue valueFromFile = new StringFieldValue();
    DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile));
    deserializer.read(null, valueFromFile);
    assertEquals(value, valueFromFile);
}