Examples with FieldValue com.yahoo.document.datatypes.FieldValue used on opensource projects

Example 91 with FieldValue

use of com.yahoo.document.datatypes.FieldValue in project vespa by vespa-engine.

the class VespaDocumentDeserializer42 method read.

public <T extends FieldValue> void read(FieldBase field, Array<T> array) {
    int numElements = getNumCollectionElems();
    ArrayList<T> list = new ArrayList<T>(numElements);
    ArrayDataType type = array.getDataType();
    for (int i = 0; i < numElements; i++) {
        if (version < 7) {
            // We don't need size for anything
            getInt(null);
        }
        FieldValue fv = type.getNestedType().createFieldValue();
        fv.deserialize(null, this);
        list.add((T) fv);
    }
    array.clear();
    array.addAll(list);
}

Example 92 with FieldValue

use of com.yahoo.document.datatypes.FieldValue in project vespa by vespa-engine.

the class VespaDocumentDeserializer42 method read.

public void read(FieldBase fieldDef, Struct s) {
    s.setVersion(version);
    int startPos = position();
    if (version < 6) {
        throw new DeserializationException("Illegal document serialization version " + version);
    }
    int dataSize;
    if (version < 7) {
        long rSize = getInt2_4_8Bytes(null);
        // TODO: Look into how to support data segments larger than INT_MAX bytes
        if (rSize > Integer.MAX_VALUE) {
            throw new DeserializationException("Raw size of data block is too large.");
        }
        dataSize = (int) rSize;
    } else {
        dataSize = getInt(null);
    }
    byte comprCode = getByte(null);
    CompressionType compression = CompressionType.valueOf(comprCode);
    int uncompressedSize = 0;
    if (compression != CompressionType.NONE && compression != CompressionType.INCOMPRESSIBLE) {
        // uncompressedsize (full size of FIELDS only, after decompression)
        long pSize = getInt2_4_8Bytes(null);
        // TODO: Look into how to support data segments larger than INT_MAX bytes
        if (pSize > Integer.MAX_VALUE) {
            throw new DeserializationException("Uncompressed size of data block is too large.");
        }
        uncompressedSize = (int) pSize;
    }
    int numberOfFields = getInt1_4Bytes(null);
    List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields);
    for (int i = 0; i < numberOfFields; ++i) {
        // id, length (length only used for unknown fields
        fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null)));
    }
    // save a reference to the big buffer we're reading from:
    GrowableByteBuffer bigBuf = buf;
    if (version < 7) {
        // In V6 and earlier, the length included the header.
        int headerSize = position() - startPos;
        dataSize -= headerSize;
    }
    byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize));
    // set position in original buffer to after data
    position(position() + dataSize);
    // for a while: deserialize from this buffer instead:
    buf = GrowableByteBuffer.wrap(destination);
    s.clear();
    StructDataType type = s.getDataType();
    for (int i = 0; i < numberOfFields; ++i) {
        Field structField = type.getField(fieldIdsAndLengths.get(i).first, version);
        if (structField == null) {
            // ignoring unknown field:
            position(position() + fieldIdsAndLengths.get(i).second.intValue());
        } else {
            int posBefore = position();
            FieldValue value = structField.getDataType().createFieldValue();
            value.deserialize(structField, this);
            s.setFieldValue(structField, value);
            // jump to beginning of next field:
            position(posBefore + fieldIdsAndLengths.get(i).second.intValue());
        }
    }
    // restore the original buffer
    buf = bigBuf;
}

Example 93 with FieldValue

use of com.yahoo.document.datatypes.FieldValue in project vespa by vespa-engine.

the class DocumentTestCase method testModifyDocument.

@Test
public void testModifyDocument() {
    Document doc = new Document(testDocType, new DocumentId("doc:ns:testdoc"));
    doc.setFieldValue("primitive1", 1);
    Struct l1s1 = new Struct(doc.getField("l1s1").getDataType());
    l1s1.setFieldValue("primitive1", 2);
    Struct l2s1 = new Struct(doc.getField("struct2").getDataType());
    l2s1.setFieldValue("primitive1", 3);
    l2s1.setFieldValue("primitive2", 4);
    Array<IntegerFieldValue> iarr1 = new Array<>(l2s1.getField("iarray").getDataType());
    iarr1.add(new IntegerFieldValue(11));
    iarr1.add(new IntegerFieldValue(12));
    iarr1.add(new IntegerFieldValue(13));
    l2s1.setFieldValue("iarray", iarr1);
    ArrayDataType dt = (ArrayDataType) l2s1.getField("sarray").getDataType();
    Array<Struct> sarr1 = new Array<>(dt);
    {
        Struct l3s1 = new Struct(dt.getNestedType());
        l3s1.setFieldValue("primitive1", 1);
        l3s1.setFieldValue("primitive2", 2);
        sarr1.add(l3s1);
    }
    {
        Struct l3s1 = new Struct(dt.getNestedType());
        l3s1.setFieldValue("primitive1", 1);
        l3s1.setFieldValue("primitive2", 2);
        sarr1.add(l3s1);
    }
    l2s1.setFieldValue("sarray", sarr1);
    MapFieldValue<StringFieldValue, StringFieldValue> smap1 = new MapFieldValue<>((MapDataType) l2s1.getField("smap").getDataType());
    smap1.put(new StringFieldValue("leonardo"), new StringFieldValue("dicaprio"));
    smap1.put(new StringFieldValue("ellen"), new StringFieldValue("page"));
    smap1.put(new StringFieldValue("joseph"), new StringFieldValue("gordon-levitt"));
    l2s1.setFieldValue("smap", smap1);
    l1s1.setFieldValue("ss", l2s1.clone());
    MapFieldValue<StringFieldValue, Struct> structmap1 = new MapFieldValue<>((MapDataType) l1s1.getField("structmap").getDataType());
    structmap1.put(new StringFieldValue("test"), l2s1.clone());
    l1s1.setFieldValue("structmap", structmap1);
    WeightedSet<StringFieldValue> wset1 = new WeightedSet<>(l1s1.getField("wset").getDataType());
    wset1.add(new StringFieldValue("foo"));
    wset1.add(new StringFieldValue("bar"));
    wset1.add(new StringFieldValue("zoo"));
    l1s1.setFieldValue("wset", wset1);
    Struct l2s2 = new Struct(doc.getField("struct2").getDataType());
    l2s2.setFieldValue("primitive1", 5);
    l2s2.setFieldValue("primitive2", 6);
    WeightedSet<Struct> wset2 = new WeightedSet<>(l1s1.getField("structwset").getDataType());
    wset2.add(l2s1.clone());
    wset2.add(l2s2.clone());
    l1s1.setFieldValue("structwset", wset2);
    doc.setFieldValue("l1s1", l1s1.clone());
    {
        ModifyIteratorHandler handler = new ModifyIteratorHandler();
        FieldPath path = doc.getDataType().buildFieldPath("l1s1.structmap.value.smap{leonardo}");
        doc.iterateNested(path, 0, handler);
        FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap{leonardo}");
        assertEquals(new StringFieldValue("newvalue"), fv);
    }
    {
        AddIteratorHandler handler = new AddIteratorHandler();
        FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray");
        doc.iterateNested(path, 0, handler);
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
        assertTrue(((Array) fv).contains(new IntegerFieldValue(32)));
        assertEquals(4, ((Array) fv).size());
    }
    {
        RemoveIteratorHandler handler = new RemoveIteratorHandler();
        FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray[1]");
        doc.iterateNested(path, 0, handler);
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
        assertFalse(((Array) fv).contains(new Integer(12)));
        assertEquals(3, ((Array) fv).size());
    }
    {
        RemoveIteratorHandler handler = new RemoveIteratorHandler();
        FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray[$x]");
        doc.iterateNested(path, 0, handler);
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
        assertEquals(0, ((Array) fv).size());
    }
    {
        RemoveIteratorHandler handler = new RemoveIteratorHandler();
        FieldPath path = doc.getDataType().buildFieldPath("l1s1.structmap.value.smap{leonardo}");
        doc.iterateNested(path, 0, handler);
        FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap");
        assertFalse(((MapFieldValue) fv).contains(new StringFieldValue("leonardo")));
    }
    {
        RemoveIteratorHandler handler = new RemoveIteratorHandler();
        FieldPath path = doc.getDataType().buildFieldPath("l1s1.wset{foo}");
        doc.iterateNested(path, 0, handler);
        FieldValue fv = doc.getRecursiveValue("l1s1.wset");
        assertFalse(((WeightedSet) fv).contains(new StringFieldValue("foo")));
    }
}

Example 94 with FieldValue

use of com.yahoo.document.datatypes.FieldValue in project vespa by vespa-engine.

the class Bug4261985TestCase method annotate.

public void annotate(Document document, DocumentTypeManager manager) {
    AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
    AnnotationType company = registry.getType("company");
    AnnotationType industry = registry.getType("industry");
    AnnotationType person = registry.getType("person");
    AnnotationType location = registry.getType("location");
    AnnotationType bigshots = registry.getType("bigshots");
    if (company.inherits(industry)) {
        System.out.println("Company Inherits Industry");
    } else {
        System.out.println("FAIL: COMPANY DOES NOT INHERIT INDUSTRY");
        throw new RuntimeException("FAIL: COMPANY DOES NOT INHERIT INDUSTRY, though it does in SD file");
    }
    SpanTree tree = new SpanTree("testannotations");
    SpanList root = (SpanList) tree.getRoot();
    SpanNode span1 = new Span(0, 5);
    SpanNode span2 = new Span(5, 10);
    SpanNode span3 = new Span(10, 15);
    SpanNode span4 = new Span(15, 20);
    SpanNode span5 = new Span(6, 10);
    SpanNode span6 = new Span(8, 4);
    SpanNode span7 = new Span(4, 2);
    SpanNode span8 = new Span(12, 6);
    root.add(span1);
    root.add(span2);
    // root.add(span3);
    root.add(span4);
    root.add(span5);
    root.add(span6);
    // root.add(span7);
    root.add(span8);
    AlternateSpanList aspl = new AlternateSpanList();
    aspl.add(span7);
    List<SpanNode> subtree1 = new ArrayList<SpanNode>();
    subtree1.add(span3);
    aspl.addChildren(1, subtree1, 33.0d);
    root.add(aspl);
    Struct personValue = (Struct) person.getDataType().createFieldValue();
    personValue.setFieldValue("name", "Richard Bair");
    Annotation personAn = new Annotation(person, personValue);
    tree.annotate(span1, personAn);
    Struct companyValue = (Struct) company.getDataType().createFieldValue();
    companyValue.setFieldValue("name", "Sun");
    Struct locationVal = new Struct(manager.getDataType("annotation.location"));
    locationVal.setFieldValue("lat", 37.774929);
    locationVal.setFieldValue("lon", -122.419415);
    Annotation locAnnotation = new Annotation(location, locationVal);
    Field compLocField = ((StructDataType) company.getDataType()).getField("place");
    // FieldValue compLocFieldVal = new FieldValue(compLocField.getDataType());
    AnnotationReferenceDataType annType = (AnnotationReferenceDataType) compLocField.getDataType();
    FieldValue compLocFieldVal = null;
    // if (scenario.equals("createFieldValue")) {
    // compLocFieldVal = annType.createFieldValue(new AnnotationReference(annType, locAnnotation));
    // } else {
    compLocFieldVal = new AnnotationReference(annType, locAnnotation);
    // }
    companyValue.setFieldValue(compLocField, compLocFieldVal);
    companyValue.setFieldValue("vertical", "software");
    Struct dirValue1 = new Struct(manager.getDataType("annotation.person"));
    dirValue1.setFieldValue("name", "Jonathan Schwartz");
    Annotation dirAnnotation1 = new Annotation(person, dirValue1);
    Struct dirValue2 = new Struct(manager.getDataType("annotation.person"));
    dirValue2.setFieldValue("name", "Scott Mcnealy");
    Annotation dirAnnotation2 = new Annotation(person, dirValue2);
    Field dirField = ((StructDataType) company.getDataType()).getField("directors");
    Array<FieldValue> dirFieldVal = new Array<FieldValue>(dirField.getDataType());
    AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) dirField.getDataType()).getNestedType();
    dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation1));
    dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation2));
    companyValue.setFieldValue(dirField, dirFieldVal);
    Annotation compAn = new Annotation(company, companyValue);
    tree.annotate(span2, compAn);
    Struct bigshotsValue = (Struct) bigshots.getDataType().createFieldValue();
    Field ceosField = ((StructDataType) bigshots.getDataType()).getField("ceos");
    // FieldValue compLocFieldVal = new FieldValue(compLocField.getDataType());
    AnnotationReferenceDataType annType1 = (AnnotationReferenceDataType) ceosField.getDataType();
    FieldValue ceosFieldVal = new AnnotationReference(annType1, compAn);
    bigshotsValue.setFieldValue(ceosField, ceosFieldVal);
    Annotation bigshotsAn = new Annotation(bigshots, bigshotsValue);
    tree.annotate(span8, bigshotsAn);
    Field selfField = ((StructDataType) bigshots.getDataType()).getField("self");
    AnnotationReferenceDataType annType2 = (AnnotationReferenceDataType) selfField.getDataType();
    FieldValue selfFieldVal = new AnnotationReference(annType2, bigshotsAn);
    bigshotsValue.setFieldValue(selfField, selfFieldVal);
    bigshotsAn = new Annotation(bigshots, bigshotsValue);
    tree.annotate(span8, bigshotsAn);
    tree.annotate(span3, locAnnotation);
    tree.annotate(span5, dirAnnotation1);
    tree.annotate(span6, dirAnnotation2);
    Struct indValue = new Struct(manager.getDataType("annotation.industry"));
    indValue.setFieldValue("vertical", "Manufacturing");
    Annotation indAn = new Annotation(industry, indValue);
    tree.annotate(span4, indAn);
    StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
    body.setSpanTree(tree);
    document.setFieldValue(document.getDataType().getField("body"), body);
}

Example 95 with FieldValue

use of com.yahoo.document.datatypes.FieldValue in project vespa by vespa-engine.

the class DocTestCase method simple5.

public void simple5() {
    // the following two lines work inside process(Document, Arguments, Processing) in a DocumentProcessor
    DocumentTypeManager dtm = processing.getService().getDocumentTypeManager();
    AnnotationTypeRegistry atr = dtm.getAnnotationTypeRegistry();
    StringFieldValue text = new StringFieldValue("<body><p>I live in San </p>Francisco</body>");
    // 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
    SpanList root = new SpanList();
    SpanTree tree = new SpanTree("html", root);
    StructDataType positionType = (StructDataType) dtm.getDataType("position");
    AnnotationType textType = atr.getType("text");
    AnnotationType beginTag = atr.getType("begintag");
    AnnotationType endTag = atr.getType("endtag");
    AnnotationType bodyType = atr.getType("body");
    AnnotationType paragraphType = atr.getType("paragraph");
    AnnotationType cityType = atr.getType("city");
    Struct position = new Struct(positionType);
    position.setFieldValue("latitude", 37.774929);
    position.setFieldValue("longitude", -122.419415);
    Annotation sanAnnotation = new Annotation(textType);
    Annotation franciscoAnnotation = new Annotation(textType);
    Struct positionWithRef = (Struct) cityType.getDataType().createFieldValue();
    positionWithRef.setFieldValue("position", position);
    Field referencesField = ((StructDataType) cityType.getDataType()).getField("references");
    Array<FieldValue> refList = new Array<FieldValue>(referencesField.getDataType());
    AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) referencesField.getDataType()).getNestedType();
    refList.add(new AnnotationReference(annRefType, sanAnnotation));
    refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
    positionWithRef.setFieldValue(referencesField, refList);
    Annotation city = new Annotation(cityType, positionWithRef);
    SpanList paragraph = new SpanList();
    {
        Span span1 = new Span(6, 3);
        Span span2 = new Span(9, 10);
        Span span3 = new Span(19, 4);
        Span span4 = new Span(23, 4);
        paragraph.add(span1).add(span2).add(span3).add(span4);
        tree.annotate(span1, beginTag).annotate(span2, textType).annotate(span3, sanAnnotation).annotate(span4, endTag).annotate(paragraph, paragraphType);
    }
    {
        Span span1 = new Span(0, 6);
        Span span2 = new Span(27, 9);
        Span span3 = new Span(36, 8);
        root.add(span1).add(paragraph).add(span2).add(span3);
        tree.annotate(span1, beginTag).annotate(span2, franciscoAnnotation).annotate(span3, endTag).annotate(root, bodyType).annotate(city);
    }
    text.setSpanTree(tree);
}