Examples with SpanNode com.yahoo.document.annotation.SpanNode used on opensource projects

Example 1 with SpanNode

use of com.yahoo.document.annotation.SpanNode in project vespa by vespa-engine.

the class StringTestCase method annotate.

public Document annotate(Document document, DocumentTypeManager manager) {
    AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
    AnnotationType company = registry.getType("company");
    AnnotationType industry = registry.getType("industry");
    AnnotationType person = registry.getType("person");
    AnnotationType location = registry.getType("location");
    Map<String, AnnotationType> m = registry.getTypes();
    for (String key : m.keySet()) {
        System.out.println("Key: " + key);
        AnnotationType val = m.get(key);
        parseAnnotationType(val);
    }
    SpanTree tree = new SpanTree("testannotations");
    SpanList root = (SpanList) tree.getRoot();
    SpanNode companySpan = new Span(0, 5);
    SpanNode industrySpan = new Span(5, 10);
    SpanNode personSpan = new Span(10, 15);
    SpanNode locationSpan = new Span(15, 20);
    root.add(companySpan);
    root.add(industrySpan);
    root.add(personSpan);
    root.add(locationSpan);
    Struct companyValue = (Struct) company.getDataType().createFieldValue();
    companyValue.setFieldValue("name", new StringFieldValue("Sun"));
    companyValue.setFieldValue("ceo", new StringFieldValue("Scott Mcnealy"));
    companyValue.setFieldValue("lat", new DoubleFieldValue(37.7));
    companyValue.setFieldValue("lon", new DoubleFieldValue(-122.44));
    companyValue.setFieldValue("vertical", new StringFieldValue("software"));
    Annotation compAn = new Annotation(company, companyValue);
    tree.annotate(companySpan, compAn);
    Struct personValue = new Struct(manager.getDataType("annotation.person"));
    personValue.setFieldValue("name", new StringFieldValue("Richard Bair"));
    Annotation personAn = new Annotation(person, personValue);
    tree.annotate(personSpan, personAn);
    Struct locValue = new Struct(manager.getDataType("annotation.location"));
    locValue.setFieldValue("name", new StringFieldValue("Prinsens Gate"));
    Annotation loc = new Annotation(location, locValue);
    tree.annotate(locationSpan, loc);
    Struct locValue2 = new Struct(manager.getDataType("annotation.location"));
    locValue2.setFieldValue("name", new StringFieldValue("Kongens Gate"));
    Annotation locAn = new Annotation(location, locValue2);
    tree.annotate(locationSpan, locAn);
    SpanList branch = new SpanList();
    SpanNode span1 = new Span(0, 3);
    SpanNode span2 = new Span(1, 9);
    SpanNode span3 = new Span(12, 10);
    branch.add(span1);
    branch.add(span3);
    branch.add(span2);
    Struct industryValue = new Struct(manager.getDataType("annotation.industry"));
    industryValue.setFieldValue("vertical", new StringFieldValue("Manufacturing"));
    Annotation ind = new Annotation(industry, industryValue);
    tree.annotate(span1, ind);
    Struct pValue = new Struct(manager.getDataType("annotation.person"));
    pValue.setFieldValue("name", new StringFieldValue("Praveen Mohan"));
    Annotation pAn = new Annotation(person, pValue);
    tree.annotate(span2, pAn);
    Struct lValue = new Struct(manager.getDataType("annotation.location"));
    lValue.setFieldValue("name", new StringFieldValue("Embassy Golf Links"));
    Annotation locn = new Annotation(location, lValue);
    tree.annotate(span3, locn);
    Struct cValue = (Struct) company.getDataType().createFieldValue();
    cValue.setFieldValue("name", new StringFieldValue("Yahoo"));
    cValue.setFieldValue("ceo", new StringFieldValue("Carol Bartz"));
    cValue.setFieldValue("lat", new DoubleFieldValue(127.7));
    cValue.setFieldValue("lon", new DoubleFieldValue(-42.44));
    cValue.setFieldValue("vertical", new StringFieldValue("search"));
    Annotation cAn = new Annotation(company, cValue);
    tree.annotate(branch, cAn);
    Struct pVal = new Struct(manager.getDataType("annotation.person"));
    pVal.setFieldValue("name", new StringFieldValue("Kim Omar"));
    Annotation an = new Annotation(person, pVal);
    tree.annotate(root, an);
    root.add(branch);
    StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
    root.remove(branch);
    tree.cleanup();
    System.out.println("No. Of Annotations: " + tree.numAnnotations());
    body.setSpanTree(tree);
    document.setFieldValue(document.getField("body"), body);
    return document;
}

Example 2 with SpanNode

use of com.yahoo.document.annotation.SpanNode in project vespa by vespa-engine.

the class StringTestCase method consumeAnnotations.

public void consumeAnnotations(SpanTree tree, SpanList root) {
    System.out.println("\n\nSpanList: " + root + " num Children: " + root.numChildren());
    System.out.println("-------------------");
    Iterator<SpanNode> childIterator = root.childIterator();
    while (childIterator.hasNext()) {
        SpanNode node = childIterator.next();
        // + " Span Text: " + node.getText(fieldValStr));
        System.out.println("Span Node: " + node);
        if (node instanceof SpanList) {
            System.out.println("Encountered another span list");
            SpanList spl = (SpanList) node;
            ListIterator<SpanNode> lli = spl.childIterator();
            while (lli.hasNext()) {
                System.out.print(" " + lli.next() + " ");
            }
            consumeAnnotations(tree, (SpanList) node);
        } else {
            System.out.println("\nGetting annotations for this span node: " + node);
            getAnnotationsForNode(tree, node);
        }
    }
    System.out.println("\nGetting annotations for the SpanList itself : " + root);
    getAnnotationsForNode(tree, root);
}

Example 3 with SpanNode

use of com.yahoo.document.annotation.SpanNode in project vespa by vespa-engine.

the class VespaDocumentDeserializer42 method read.

public void read(Annotation annotation) {
    int annotationTypeId = buf.getInt();
    AnnotationType type = manager.getAnnotationTypeRegistry().getType(annotationTypeId);
    if (type == null) {
        throw new DeserializationException("Cannot deserialize annotation of type " + annotationTypeId + " (unknown type)");
    }
    annotation.setType(type);
    byte features = buf.get();
    int length = buf.getInt1_2_4Bytes();
    if ((features & (byte) 1) == (byte) 1) {
        // we have a span node
        int spanNodeId = buf.getInt1_2_4Bytes();
        try {
            SpanNode node = spanNodes.get(spanNodeId);
            annotation.setSpanNode(node);
        } catch (IndexOutOfBoundsException ioobe) {
            throw new DeserializationException("Could not deserialize annotation, associated span node not found ", ioobe);
        }
    }
    if ((features & (byte) 2) == (byte) 2) {
        // we have a value:
        int dataTypeId = buf.getInt();
        // if this data type ID the same as the one in our config?
        if (dataTypeId != type.getDataType().getId()) {
            // not the same, but we will handle it gracefully, and just skip past the data:
            buf.position(buf.position() + length - 4);
        } else {
            FieldValue value = type.getDataType().createFieldValue();
            value.deserialize(this);
            annotation.setFieldValue(value);
        }
    }
}

Example 4 with SpanNode

use of com.yahoo.document.annotation.SpanNode in project vespa by vespa-engine.

the class VespaDocumentDeserializer42 method read.

public void read(AlternateSpanList altSpanList) {
    byte type = buf.get();
    if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) {
        throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type);
    }
    int numSubTrees = buf.getInt1_2_4Bytes();
    for (int i = 0; i < numSubTrees; i++) {
        double prob = buf.getDouble();
        List<SpanNode> list = readSpanList(altSpanList);
        if (i == 0) {
            for (SpanNode node : list) {
                altSpanList.add(node);
            }
            altSpanList.setProbability(0, prob);
        } else {
            altSpanList.addChildren(i, list, prob);
        }
    }
}

Example 5 with SpanNode

use of com.yahoo.document.annotation.SpanNode in project vespa by vespa-engine.

the class VespaDocumentDeserializer42 method readSpanTree.

private void readSpanTree(SpanTree tree, boolean readName) {
    // we don't support serialization of nested span trees:
    if (spanNodes != null || annotations != null) {
        throw new SerializationException("Deserialization of nested SpanTrees is not supported.");
    }
    // we're going to write a new SpanTree, create a new Map for nodes:
    spanNodes = new ArrayList<SpanNode>();
    annotations = new ArrayList<Annotation>();
    try {
        if (readName) {
            StringFieldValue treeName = new StringFieldValue();
            treeName.deserialize(this);
            tree.setName(treeName.getString());
        }
        SpanNode root = readSpanNode();
        tree.setRoot(root);
        int numAnnotations = buf.getInt1_2_4Bytes();
        for (int i = 0; i < numAnnotations; i++) {
            Annotation a = new Annotation();
            annotations.add(a);
        }
        for (int i = 0; i < numAnnotations; i++) {
            read(annotations.get(i));
        }
        for (Annotation a : annotations) {
            tree.annotate(a);
        }
        for (SpanNode node : spanNodes) {
            if (node instanceof Span) {
                correctIndexes((Span) node);
            }
        }
    } finally {
        // we're done, let's set this to null to save memory and prevent madness:
        spanNodes = null;
        annotations = null;
    }
}