Examples with TokenType com.yahoo.language.process.TokenType used on opensource projects

Example 6 with TokenType

use of com.yahoo.language.process.TokenType in project vespa by vespa-engine.

the class LinguisticsAnnotatorTestCase method requireThatTermReplacementsAreApplied.

@Test
public void requireThatTermReplacementsAreApplied() {
    SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
    expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
    for (boolean specialToken : Arrays.asList(true, false)) {
        for (TokenType type : TokenType.values()) {
            if (!specialToken && !type.isIndexable()) {
                continue;
            }
            assertAnnotations(expected, "foo", newLinguistics(Arrays.asList(newToken("foo", "foo", type, specialToken)), Collections.singletonMap("foo", "bar")));
        }
    }
}

Example 7 with TokenType

use of com.yahoo.language.process.TokenType in project vespa by vespa-engine.

the class LinguisticsAnnotatorTestCase method requireThatSpecialTokenStringsAreAnnotatedRegardlessOfType.

@Test
public void requireThatSpecialTokenStringsAreAnnotatedRegardlessOfType() {
    SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
    expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
    for (TokenType type : TokenType.values()) {
        assertAnnotations(expected, "foo", newToken("foo", "bar", type, true));
    }
}