Search in sources :

Example 41 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class QueryTestCase method testUnicodeNormalization.

@Test
public void testUnicodeNormalization() {
    Linguistics linguistics = new SimpleLinguistics();
    Query query = newQueryFromEncoded("?query=content:%EF%BC%B3%EF%BC%AF%EF%BC%AE%EF%BC%B9", Language.ENGLISH, linguistics);
    assertEquals("SONY", ((WordItem) query.getModel().getQueryTree().getRoot()).getWord());
    query = newQueryFromEncoded("?query=foo&filter=+%EF%BC%B3%EF%BC%AF%EF%BC%AE%EF%BC%B9", Language.ENGLISH, linguistics);
    assertEquals("RANK foo |SONY", query.getModel().getQueryTree().getRoot().toString());
    query = newQueryFromEncoded("?query=foo+AND+%EF%BC%B3%EF%BC%AF%EF%BC%AE%EF%BC%B9)&type=adv", Language.ENGLISH, linguistics);
    assertEquals("AND foo SONY", query.getModel().getQueryTree().getRoot().toString());
}
Also used : SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Query(com.yahoo.search.Query) Linguistics(com.yahoo.language.Linguistics) SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Test(org.junit.Test)

Example 42 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class ExpressionConverterTestCase method requireThatAllExpressionTypesCanBeTraversed.

@SuppressWarnings("unchecked")
@Test
public void requireThatAllExpressionTypesCanBeTraversed() {
    assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, new InputExpression("bar")));
    assertConvertable(new AttributeExpression("foo"));
    assertConvertable(new Base64DecodeExpression());
    assertConvertable(new Base64EncodeExpression());
    assertConvertable(new CatExpression(new InputExpression("foo"), new IndexExpression("bar")));
    assertConvertable(new ClearStateExpression());
    assertConvertable(new EchoExpression());
    assertConvertable(new ForEachExpression(new IndexExpression("foo")));
    assertConvertable(new GetFieldExpression("foo"));
    assertConvertable(new GetVarExpression("foo"));
    assertConvertable(new GuardExpression(new IndexExpression("foo")));
    assertConvertable(new HexDecodeExpression());
    assertConvertable(new HexEncodeExpression());
    assertConvertable(new HostNameExpression());
    assertConvertable(new IfThenExpression(new InputExpression("foo"), IfThenExpression.Comparator.EQ, new InputExpression("bar"), new IndexExpression("baz"), new IndexExpression("cox")));
    assertConvertable(new IndexExpression("foo"));
    assertConvertable(new InputExpression("foo"));
    assertConvertable(new JoinExpression("foo"));
    assertConvertable(new LowerCaseExpression());
    assertConvertable(new NormalizeExpression(new SimpleLinguistics()));
    assertConvertable(new NowExpression());
    assertConvertable(new OptimizePredicateExpression());
    assertConvertable(new ParenthesisExpression(new InputExpression("foo")));
    assertConvertable(new RandomExpression(69));
    assertConvertable(new ScriptExpression(new StatementExpression(new InputExpression("foo"))));
    assertConvertable(new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")), new Pair<String, Expression>("bar", new IndexExpression("foo"))));
    assertConvertable(new SetLanguageExpression());
    assertConvertable(new SetValueExpression(new IntegerFieldValue(69)));
    assertConvertable(new SetVarExpression("foo"));
    assertConvertable(new SplitExpression("foo"));
    assertConvertable(new StatementExpression(new InputExpression("foo")));
    assertConvertable(new SubstringExpression(6, 9));
    assertConvertable(new SummaryExpression("foo"));
    assertConvertable(new SwitchExpression(Collections.singletonMap("foo", (Expression) new IndexExpression("bar")), new InputExpression("baz")));
    assertConvertable(new ThisExpression());
    assertConvertable(new ToArrayExpression());
    assertConvertable(new ToByteExpression());
    assertConvertable(new ToDoubleExpression());
    assertConvertable(new ToFloatExpression());
    assertConvertable(new ToIntegerExpression());
    assertConvertable(new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()));
    assertConvertable(new ToLongExpression());
    assertConvertable(new ToPositionExpression());
    assertConvertable(new ToStringExpression());
    assertConvertable(new ToWsetExpression(false, false));
    assertConvertable(new TrimExpression());
    assertConvertable(new ZCurveExpression());
}
Also used : SummaryExpression(com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression) SwitchExpression(com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression) AnnotatorConfig(com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig) StatementExpression(com.yahoo.vespa.indexinglanguage.expressions.StatementExpression) IntegerFieldValue(com.yahoo.document.datatypes.IntegerFieldValue) OptimizePredicateExpression(com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression) SplitExpression(com.yahoo.vespa.indexinglanguage.expressions.SplitExpression) ToPositionExpression(com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression) ThisExpression(com.yahoo.vespa.indexinglanguage.expressions.ThisExpression) ToByteExpression(com.yahoo.vespa.indexinglanguage.expressions.ToByteExpression) TokenizeExpression(com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression) IfThenExpression(com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression) ToStringExpression(com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression) SubstringExpression(com.yahoo.vespa.indexinglanguage.expressions.SubstringExpression) EchoExpression(com.yahoo.vespa.indexinglanguage.expressions.EchoExpression) SetVarExpression(com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression) HexDecodeExpression(com.yahoo.vespa.indexinglanguage.expressions.HexDecodeExpression) SelectInputExpression(com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression) InputExpression(com.yahoo.vespa.indexinglanguage.expressions.InputExpression) AttributeExpression(com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression) ScriptExpression(com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression) SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) ToArrayExpression(com.yahoo.vespa.indexinglanguage.expressions.ToArrayExpression) SetLanguageExpression(com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression) LowerCaseExpression(com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression) ToIntegerExpression(com.yahoo.vespa.indexinglanguage.expressions.ToIntegerExpression) ForEachExpression(com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression) SelectInputExpression(com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression) ToWsetExpression(com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression) Base64DecodeExpression(com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression) IndexExpression(com.yahoo.vespa.indexinglanguage.expressions.IndexExpression) GetFieldExpression(com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression) RandomExpression(com.yahoo.vespa.indexinglanguage.expressions.RandomExpression) TrimExpression(com.yahoo.vespa.indexinglanguage.expressions.TrimExpression) JoinExpression(com.yahoo.vespa.indexinglanguage.expressions.JoinExpression) HexEncodeExpression(com.yahoo.vespa.indexinglanguage.expressions.HexEncodeExpression) Pair(com.yahoo.collections.Pair) SetValueExpression(com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression) NowExpression(com.yahoo.vespa.indexinglanguage.expressions.NowExpression) ToDoubleExpression(com.yahoo.vespa.indexinglanguage.expressions.ToDoubleExpression) ToLongExpression(com.yahoo.vespa.indexinglanguage.expressions.ToLongExpression) ToFloatExpression(com.yahoo.vespa.indexinglanguage.expressions.ToFloatExpression) Base64EncodeExpression(com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression) CatExpression(com.yahoo.vespa.indexinglanguage.expressions.CatExpression) GuardExpression(com.yahoo.vespa.indexinglanguage.expressions.GuardExpression) ParenthesisExpression(com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression) NormalizeExpression(com.yahoo.vespa.indexinglanguage.expressions.NormalizeExpression) ArithmeticExpression(com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression) HostNameExpression(com.yahoo.vespa.indexinglanguage.expressions.HostNameExpression) GetVarExpression(com.yahoo.vespa.indexinglanguage.expressions.GetVarExpression) ClearStateExpression(com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression) ZCurveExpression(com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression) Test(org.junit.Test)

Aggregations

SimpleLinguistics (com.yahoo.language.simple.SimpleLinguistics)42 Test (org.junit.Test)37 Token (com.yahoo.prelude.query.parser.Token)17 Tokenizer (com.yahoo.prelude.query.parser.Tokenizer)17 Linguistics (com.yahoo.language.Linguistics)10 Index (com.yahoo.prelude.Index)7 IndexFacts (com.yahoo.prelude.IndexFacts)7 StringFieldValue (com.yahoo.document.datatypes.StringFieldValue)6 AnnotatorConfig (com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig)5 SpecialTokenRegistry (com.yahoo.prelude.query.parser.SpecialTokenRegistry)3 Query (com.yahoo.search.Query)3 Execution (com.yahoo.search.searchchain.Execution)3 SimpleTestAdapter (com.yahoo.vespa.indexinglanguage.SimpleTestAdapter)3 InputExpression (com.yahoo.vespa.indexinglanguage.expressions.InputExpression)3 Pair (com.yahoo.collections.Pair)2 FieldValue (com.yahoo.document.datatypes.FieldValue)2 IntegerFieldValue (com.yahoo.document.datatypes.IntegerFieldValue)2 RendererRegistry (com.yahoo.search.rendering.RendererRegistry)2 ArithmeticExpression (com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression)2 AttributeExpression (com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression)2