Examples with WordItem com.yahoo.prelude.query.WordItem used on opensource projects

Example 61 with WordItem

use of com.yahoo.prelude.query.WordItem in project vespa by vespa-engine.

the class LiteralPhraseProduction method produce.

public void produce(RuleEvaluation e, int offset) {
    PhraseItem newPhrase = new PhraseItem();
    newPhrase.setIndexName(getLabel());
    for (String term : terms) newPhrase.addItem(new WordItem(term));
    if (replacing) {
        Match matched = e.getNonreferencedMatch(0);
        insertMatch(e, matched, newPhrase, offset);
    } else {
        newPhrase.setWeight(getWeight());
        if (e.getTraceLevel() >= 6)
            e.trace(6, "Adding '" + newPhrase + "'");
        e.addItem(newPhrase, getTermType());
    }
}

Example 62 with WordItem

use of com.yahoo.prelude.query.WordItem in project vespa by vespa-engine.

the class YqlParser method instantiatePhraseSegmentItem.

@NonNull
private Item instantiatePhraseSegmentItem(String field, OperatorNode<ExpressionOperator> ast, boolean forcePhrase) {
    Substring origin = getOrigin(ast);
    Boolean stem = getAnnotation(ast, STEM, Boolean.class, Boolean.TRUE, STEM_DESCRIPTION);
    Boolean andSegmenting = getAnnotation(ast, AND_SEGMENTING, Boolean.class, Boolean.FALSE, "setting for whether to force using AND for segments on and off");
    SegmentItem phrase;
    List<String> words = null;
    if (forcePhrase || !andSegmenting) {
        phrase = new PhraseSegmentItem(origin.getValue(), origin.getValue(), true, !stem, origin);
    } else {
        phrase = new AndSegmentItem(origin.getValue(), true, !stem);
    }
    phrase.setIndexName(field);
    if (resegment && getAnnotation(ast, IMPLICIT_TRANSFORMS, Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION)) {
        words = segmenter.segment(origin.getValue(), currentlyParsing.getLanguage());
    }
    if (words != null && words.size() > 0) {
        for (String word : words) {
            phrase.addItem(new WordItem(word, field, true));
        }
    } else {
        for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>>getArgument(1)) {
            phrase.addItem(instantiateWordItem(field, word, phrase.getClass(), SegmentWhen.NEVER));
        }
    }
    if (phrase instanceof TaggableItem) {
        leafStyleSettings(ast, (TaggableItem) phrase);
    }
    phrase.lock();
    return phrase;
}

Example 63 with WordItem

use of com.yahoo.prelude.query.WordItem in project vespa by vespa-engine.

the class YqlParser method resegment.

@NonNull
private TaggableItem resegment(String field, OperatorNode<ExpressionOperator> ast, String wordData, boolean fromQuery, Class<?> parent, Language language) {
    String toSegment = wordData;
    Substring s = getOrigin(ast);
    Language usedLanguage = language == null ? currentlyParsing.getLanguage() : language;
    if (s != null) {
        toSegment = s.getValue();
    }
    List<String> words = segmenter.segment(toSegment, usedLanguage);
    TaggableItem wordItem;
    if (words.size() == 0) {
        wordItem = new WordItem(wordData, fromQuery);
    } else if (words.size() == 1 || !phraseArgumentSupported(parent)) {
        wordItem = new WordItem(words.get(0), fromQuery);
    } else {
        wordItem = new PhraseSegmentItem(toSegment, fromQuery, false);
        ((PhraseSegmentItem) wordItem).setIndexName(field);
        for (String w : words) {
            WordItem segment = new WordItem(w, fromQuery);
            prepareWord(field, ast, fromQuery, segment);
            ((PhraseSegmentItem) wordItem).addItem(segment);
        }
        ((PhraseSegmentItem) wordItem).lock();
    }
    return wordItem;
}

Example 64 with WordItem

use of com.yahoo.prelude.query.WordItem in project vespa by vespa-engine.

the class QueryTestCase method testEncodeQueryPacketWithLabelsConnectivityAndSignificance.

@Test
public void testEncodeQueryPacketWithLabelsConnectivityAndSignificance() {
    Query query = new Query();
    AndItem and = new AndItem();
    WeightedSetItem taggable1 = new WeightedSetItem("field1");
    taggable1.setLabel("foo");
    WeightedSetItem taggable2 = new WeightedSetItem("field2");
    taggable1.setLabel("bar");
    and.addItem(taggable1);
    and.addItem(taggable2);
    WordItem word1 = new WordItem("word1", "field3");
    word1.setSignificance(0.37);
    WordItem word2 = new WordItem("word1", "field3");
    word2.setSignificance(0.81);
    word2.setConnectivity(word1, 0.15);
    and.addItem(word1);
    and.addItem(word2);
    query.getModel().getQueryTree().setRoot(and);
    query.prepare();
    QueryPacket packet = QueryPacket.create(query);
    byte[] encoded = packetToBytes(packet);
    byte[] correctBuffer = new byte[] { 0, 0, 1, 16, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x03, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 1, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 18, 'v', 'e', 's', 'p', 'a', 46, 'l', 'a', 'b', 'e', 'l', 46, 'b', 'a', 'r', 46, 'i', 'd', 0, 0, 0, 1, '1', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 1, '3', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 4, '0', 46, '1', '5', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '3', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '3', '7', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '8', '1', 0, 0, 0, 5, 0, 0, 0, '4', 1, 4, 79, 1, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 79, 2, 0, 6, 'f', 'i', 'e', 'l', 'd', '2', 68, 3, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', '1', 68, 4, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', 49 };
    assertEqualArrays(correctBuffer, encoded);
}

Example 65 with WordItem

use of com.yahoo.prelude.query.WordItem in project vespa by vespa-engine.

the class QueryTestCase method testPhraseEqualsPhraseWithPhraseSegment.

@Test
public void testPhraseEqualsPhraseWithPhraseSegment() throws BufferTooSmallException {
    Query query = new Query();
    PhraseItem p = new PhraseItem();
    PhraseSegmentItem ps = new PhraseSegmentItem("a b", false, false);
    ps.addItem(new WordItem("a"));
    ps.addItem(new WordItem("b"));
    p.addItem(ps);
    query.getModel().getQueryTree().setRoot(p);
    query.setTimeout(0);
    QueryPacket queryPacket = QueryPacket.create(query);
    ByteBuffer buffer1 = ByteBuffer.allocate(1024);
    queryPacket.encode(buffer1, 0);
    query = new Query();
    p = new PhraseItem();
    p.addItem(new WordItem("a"));
    p.addItem(new WordItem("b"));
    query.getModel().getQueryTree().setRoot(p);
    query.setTimeout(0);
    queryPacket = QueryPacket.create(query);
    assertNotNull(queryPacket);
    ByteBuffer buffer2 = ByteBuffer.allocate(1024);
    queryPacket.encode(buffer2, 0);
    byte[] encoded1 = new byte[buffer1.position()];
    buffer1.rewind();
    buffer1.get(encoded1);
    byte[] encoded2 = new byte[buffer2.position()];
    buffer2.rewind();
    buffer2.get(encoded2);
    assertEqualArrays(encoded2, encoded1);
}