Examples with StructDataType com.yahoo.document.StructDataType used on opensource projects

Example 26 with StructDataType

use of com.yahoo.document.StructDataType in project vespa by vespa-engine.

the class DocumentScriptTestCase method requireThatLinguisticsSpanTreeIsRemovedFromStructStringFields.

@Test
public void requireThatLinguisticsSpanTreeIsRemovedFromStructStringFields() {
    StructDataType structType = new StructDataType("myStruct");
    structType.addField(new Field("myString", DataType.STRING));
    Struct in = new Struct(structType);
    in.setFieldValue("myString", newString(SpanTrees.LINGUISTICS, "mySpanTree"));
    Struct out = (Struct) processDocument(in);
    assertSpanTrees(out.getFieldValue("myString"), "mySpanTree");
    StringFieldValue str = (StringFieldValue) ((MapValueUpdate) processFieldUpdate(in)).getUpdate().getValue();
    assertSpanTrees(str, "mySpanTree");
    str = (StringFieldValue) ((MapValueUpdate) processFieldUpdate(in)).getUpdate().getValue();
    assertSpanTrees(str, "mySpanTree");
}

Example 27 with StructDataType

use of com.yahoo.document.StructDataType in project vespa by vespa-engine.

the class JsonReaderTestCase method setUp.

@Before
public void setUp() throws Exception {
    parserFactory = new JsonFactory();
    types = new DocumentTypeManager();
    {
        DocumentType x = new DocumentType("smoke");
        x.addField(new Field("something", DataType.STRING));
        x.addField(new Field("nalle", DataType.STRING));
        x.addField(new Field("int1", DataType.INT));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("mirrors");
        StructDataType woo = new StructDataType("woo");
        woo.addField(new Field("sandra", DataType.STRING));
        woo.addField(new Field("cloud", DataType.STRING));
        x.addField(new Field("skuggsjaa", woo));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testarray");
        DataType d = new ArrayDataType(DataType.STRING);
        x.addField(new Field("actualarray", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testset");
        DataType d = new WeightedSetDataType(DataType.STRING, true, true);
        x.addField(new Field("actualset", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testmap");
        DataType d = new MapDataType(DataType.STRING, DataType.STRING);
        x.addField(new Field("actualmap", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testraw");
        DataType d = DataType.RAW;
        x.addField(new Field("actualraw", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testMapStringToArrayOfInt");
        DataType value = new ArrayDataType(DataType.INT);
        DataType d = new MapDataType(DataType.STRING, value);
        x.addField(new Field("actualMapStringToArrayOfInt", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testsinglepos");
        DataType d = PositionDataType.INSTANCE;
        x.addField(new Field("singlepos", d));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testtensor");
        x.addField(new Field("mappedtensorfield", new TensorDataType(new TensorType.Builder().mapped("x").mapped("y").build())));
        x.addField(new Field("indexedtensorfield", new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build())));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testpredicate");
        x.addField(new Field("boolean", DataType.PREDICATE));
        types.registerDocumentType(x);
    }
    {
        DocumentType x = new DocumentType("testint");
        x.addField(new Field("integerfield", DataType.INT));
        types.registerDocumentType(x);
    }
}

Example 28 with StructDataType

use of com.yahoo.document.StructDataType in project vespa by vespa-engine.

the class JsonWriterTestCase method registerMirrorsDocumentType.

private void registerMirrorsDocumentType() {
    DocumentType x = new DocumentType("mirrors");
    StructDataType woo = new StructDataType("woo");
    woo.addField(new Field("sandra", DataType.STRING));
    woo.addField(new Field("cloud", DataType.STRING));
    x.addField(new Field("skuggsjaa", woo));
    types.registerDocumentType(x);
}

Example 29 with StructDataType

use of com.yahoo.document.StructDataType in project vespa by vespa-engine.

the class SerializeAnnotationsTestCase method testSerializeAdvancedTree.

public void testSerializeAdvancedTree() throws IOException {
    SpanList root = new SpanList();
    SpanTree tree = new SpanTree("html", root);
    DataType positionType = docMan.getDataType("myposition");
    StructDataType cityDataType = (StructDataType) docMan.getDataType("annotation.city");
    AnnotationTypeRegistry registry = docMan.getAnnotationTypeRegistry();
    AnnotationType textType = registry.getType("text");
    AnnotationType beginTag = registry.getType("begintag");
    AnnotationType endTag = registry.getType("endtag");
    AnnotationType bodyType = registry.getType("body");
    AnnotationType paragraphType = registry.getType("paragraph");
    AnnotationType cityType = registry.getType("city");
    AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) docMan.getDataType("annotationreference<text>");
    Struct position = new Struct(positionType);
    position.setFieldValue("latitude", new DoubleFieldValue(37.774929));
    position.setFieldValue("longitude", new DoubleFieldValue(-122.419415));
    Annotation sanAnnotation = new Annotation(textType);
    Annotation franciscoAnnotation = new Annotation(textType);
    Struct positionWithRef = cityDataType.createFieldValue();
    positionWithRef.setFieldValue("position", position);
    Field referencesField = cityDataType.getField("references");
    Array<FieldValue> refList = new Array<FieldValue>(referencesField.getDataType());
    refList.add(new AnnotationReference(annRefType, sanAnnotation));
    refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
    positionWithRef.setFieldValue(referencesField, refList);
    Annotation city = new Annotation(cityType, positionWithRef);
    AlternateSpanList paragraph = new AlternateSpanList();
    paragraph.addChildren(new ArrayList<SpanNode>(), 0);
    paragraph.setProbability(0, 0.9);
    paragraph.setProbability(1, 0.1);
    {
        Span span1 = new Span(6, 3);
        Span span2 = new Span(9, 10);
        Span span3 = new Span(19, 4);
        Span span4 = new Span(23, 4);
        paragraph.add(0, span1).add(0, span2).add(0, span3).add(0, span4);
        Span alt_span1 = new Span(6, 13);
        Span alt_span2 = new Span(19, 8);
        paragraph.add(1, alt_span1).add(1, alt_span2);
        tree.annotate(span1, beginTag).annotate(span2, textType).annotate(span3, sanAnnotation).annotate(span4, endTag).annotate(alt_span1, textType).annotate(alt_span2, bodyType).annotate(paragraph, paragraphType);
    }
    {
        Span span1 = new Span(0, 6);
        Span span2 = new Span(27, 9);
        Span span3 = new Span(36, 8);
        root.add(span1).add(paragraph).add(span2).add(span3);
        tree.annotate(span1, beginTag).annotate(span2, franciscoAnnotation).annotate(span3, endTag).annotate(root, bodyType).annotate(city);
    }
    StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " + "lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " + "lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " + "oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " + "oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" + "aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
    value.setSpanTree(tree);
    // important! call readFile() before writeFile()!
    ByteBuffer serializedFromFile = readFile("test_data_serialized_advanced");
    ByteBuffer serialized = writeFile(value, "test_data_serialized_advanced");
    assertEquals(serialized.limit(), serializedFromFile.limit());
    StringFieldValue valueFromFile = new StringFieldValue();
    DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile));
    deserializer.read(null, valueFromFile);
    assertEquals(value, valueFromFile);
}

Example 30 with StructDataType

use of com.yahoo.document.StructDataType in project vespa by vespa-engine.

the class StructTestCase method testSetUnknownType.

@Test
public void testSetUnknownType() {
    StructDataType type = new StructDataType("teststr");
    type.addField(new Field("int", 0, DataType.INT, true));
    Struct struct = new Struct(type);
    try {
        struct.setFieldValue(new Field("alien", DataType.STRING), new StringFieldValue("foo"));
        fail("Alien type worked");
    } catch (IllegalArgumentException expected) {
        assertTrue(expected.getMessage().matches(".*No such field.*"));
    }
}