Examples with PhraseSegmentItem com.yahoo.prelude.query.PhraseSegmentItem used on opensource projects

Example 6 with PhraseSegmentItem

use of com.yahoo.prelude.query.PhraseSegmentItem in project vespa by vespa-engine.

the class LowercasingTestCase method slightlyMoreComplexTree.

@Test
public void slightlyMoreComplexTree() {
    Query q = new Query();
    AndItem a0 = new AndItem();
    OrItem o0 = new OrItem();
    PhraseItem p0 = new PhraseItem();
    p0.setIndexName(BAMSE);
    PhraseSegmentItem p1 = new PhraseSegmentItem("Overbuljongterningpakkmesterassistent", true, false);
    p1.setIndexName(BAMSE);
    WordItem tmp;
    tmp = new WordItem("Nalle0", BAMSE, true);
    a0.addItem(tmp);
    tmp = new WordItem("Nalle1", BAMSE, true);
    o0.addItem(tmp);
    tmp = new WordItem("Nalle2", BAMSE, true);
    o0.addItem(tmp);
    a0.addItem(o0);
    tmp = new WordItem("Nalle3", BAMSE, true);
    p0.addItem(tmp);
    p1.addItem(new WordItem("Over", BAMSE, true));
    p1.addItem(new WordItem("buljong", BAMSE, true));
    p1.addItem(new WordItem("terning", BAMSE, true));
    p1.addItem(new WordItem("pakk", BAMSE, true));
    p1.addItem(new WordItem("Mester", BAMSE, true));
    p1.addItem(new WordItem("assistent", BAMSE, true));
    p1.lock();
    p0.addItem(p1);
    a0.addItem(p0);
    q.getModel().getQueryTree().setRoot(a0);
    Result r = execution.search(q);
    AndItem root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot();
    tmp = (WordItem) root.getItem(0);
    assertEquals("nalle0", tmp.getWord());
    OrItem orElement = (OrItem) root.getItem(1);
    tmp = (WordItem) orElement.getItem(0);
    assertEquals("nalle1", tmp.getWord());
    tmp = (WordItem) orElement.getItem(1);
    assertEquals("nalle2", tmp.getWord());
    PhraseItem phrase = (PhraseItem) root.getItem(2);
    tmp = (WordItem) phrase.getItem(0);
    assertEquals("nalle3", tmp.getWord());
    PhraseSegmentItem locked = (PhraseSegmentItem) phrase.getItem(1);
    assertEquals("over", ((WordItem) locked.getItem(0)).getWord());
    assertEquals("buljong", ((WordItem) locked.getItem(1)).getWord());
    assertEquals("terning", ((WordItem) locked.getItem(2)).getWord());
    assertEquals("pakk", ((WordItem) locked.getItem(3)).getWord());
    assertEquals("mester", ((WordItem) locked.getItem(4)).getWord());
    assertEquals("assistent", ((WordItem) locked.getItem(5)).getWord());
}

Example 7 with PhraseSegmentItem

use of com.yahoo.prelude.query.PhraseSegmentItem in project vespa by vespa-engine.

the class SegmentItemTestCase method test.

@Test
public final void test() {
    PhraseSegmentItem item = new PhraseSegmentItem("a b c", false, true);
    item.addItem(new WordItem("a"));
    item.addItem(new WordItem("b"));
    item.addItem(new WordItem("c"));
    assertEquals(100, item.getItem(0).getWeight());
    item.setWeight(150);
    assertEquals(150, item.getItem(0).getWeight());
    assertEquals(item.getItem(0).getWeight(), item.getItem(1).getWeight());
    assertEquals(item.getItem(0).getWeight(), item.getItem(2).getWeight());
}

Example 8 with PhraseSegmentItem

use of com.yahoo.prelude.query.PhraseSegmentItem in project vespa by vespa-engine.

the class VespaSerializerTestCase method testAnnotatedPhraseSegment.

@Test
public final void testAnnotatedPhraseSegment() {
    PhraseSegmentItem phraseSegment = new PhraseSegmentItem("abc", true, false);
    phraseSegment.addItem(new WordItem("a", "indexNamePlaceholder"));
    phraseSegment.addItem(new WordItem("b", "indexNamePlaceholder"));
    phraseSegment.setIndexName("someIndexName");
    phraseSegment.setLabel("labeled");
    phraseSegment.lock();
    String q = VespaSerializer.serialize(phraseSegment);
    assertEquals("someIndexName contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"label\": \"labeled\"}]phrase(\"a\", \"b\"))", q);
}

Example 9 with PhraseSegmentItem

use of com.yahoo.prelude.query.PhraseSegmentItem in project vespa by vespa-engine.

the class CJKSearcher method transform.

private Item transform(Item root) {
    if (root instanceof PhraseItem) {
        PhraseItem asPhrase = (PhraseItem) root;
        if (asPhrase.isExplicit() || hasOverlappingTokens(asPhrase))
            return root;
        AndItem replacement = new AndItem();
        for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext(); ) {
            Item item = i.next();
            if (item instanceof WordItem)
                replacement.addItem(item);
            else if (item instanceof PhraseSegmentItem) {
                replacement.addItem(new AndSegmentItem((PhraseSegmentItem) item));
            } else
                // should never run, but hey... just convert and hope it's OK :)
                replacement.addItem(item);
        }
        return replacement;
    } else if (root instanceof PhraseSegmentItem) {
        PhraseSegmentItem asSegment = (PhraseSegmentItem) root;
        if (asSegment.isExplicit() || hasOverlappingTokens(asSegment))
            return root;
        else
            return new AndSegmentItem(asSegment);
    } else if (root instanceof SegmentItem) {
        // avoid descending into AndSegmentItems and similar
        return root;
    } else if (root instanceof CompositeItem) {
        for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext(); ) {
            Item item = i.next();
            Item transformedItem = transform(item);
            if (item != transformedItem) {
                i.set(transformedItem);
            }
        }
        return root;
    }
    return root;
}

Example 10 with PhraseSegmentItem

use of com.yahoo.prelude.query.PhraseSegmentItem in project vespa by vespa-engine.

the class YqlParser method instantiatePhraseSegmentItem.

@NonNull
private Item instantiatePhraseSegmentItem(String field, OperatorNode<ExpressionOperator> ast, boolean forcePhrase) {
    Substring origin = getOrigin(ast);
    Boolean stem = getAnnotation(ast, STEM, Boolean.class, Boolean.TRUE, STEM_DESCRIPTION);
    Boolean andSegmenting = getAnnotation(ast, AND_SEGMENTING, Boolean.class, Boolean.FALSE, "setting for whether to force using AND for segments on and off");
    SegmentItem phrase;
    List<String> words = null;
    if (forcePhrase || !andSegmenting) {
        phrase = new PhraseSegmentItem(origin.getValue(), origin.getValue(), true, !stem, origin);
    } else {
        phrase = new AndSegmentItem(origin.getValue(), true, !stem);
    }
    phrase.setIndexName(field);
    if (resegment && getAnnotation(ast, IMPLICIT_TRANSFORMS, Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION)) {
        words = segmenter.segment(origin.getValue(), currentlyParsing.getLanguage());
    }
    if (words != null && words.size() > 0) {
        for (String word : words) {
            phrase.addItem(new WordItem(word, field, true));
        }
    } else {
        for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>>getArgument(1)) {
            phrase.addItem(instantiateWordItem(field, word, phrase.getClass(), SegmentWhen.NEVER));
        }
    }
    if (phrase instanceof TaggableItem) {
        leafStyleSettings(ast, (TaggableItem) phrase);
    }
    phrase.lock();
    return phrase;
}