Examples with AnalyzeResponse org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse used on opensource projects

Example 21 with AnalyzeResponse

use of org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse in project elasticsearch by elastic.

the class AnalyzeActionIT method testDetailAnalyzeSpecifyAttributes.

public void testDetailAnalyzeSpecifyAttributes() throws Exception {
    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled").setExplain(true).setTokenizer("standard").addTokenFilter("snowball").setAttributes("keyword").get();
    assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball"));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl"));
    String[] expectedAttributesKey = { "keyword" };
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length));
    Object extendedAttribute;
    for (String key : expectedAttributesKey) {
        extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key);
        assertThat(extendedAttribute, notNullValue());
    }
}

Example 22 with AnalyzeResponse

use of org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse in project elasticsearch by elastic.

the class AnalyzeActionIT method testDetailAnalyzeCustomAnalyzerWithNoIndex.

public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception {
    //analyzer only
    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setExplain(true).setAnalyzer("simple").get();
    assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue());
    assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue());
    assertThat(analyzeResponse.detail().charfilters(), IsNull.nullValue());
    assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple"));
    assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(4));
    //custom analyzer
    analyzeResponse = client().admin().indices().prepareAnalyze("<text>THIS IS A TEST</text>").setExplain(true).addCharFilter("html_strip").setTokenizer("keyword").addTokenFilter("lowercase").get();
    assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue());
    //charfilters
    assertThat(analyzeResponse.detail().charfilters().length, equalTo(1));
    assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("html_strip"));
    assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1));
    assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("\nTHIS IS A TEST\n"));
    //tokenizer
    assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword"));
    assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1));
    assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("\nTHIS IS A TEST\n"));
    //tokenfilters
    assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase"));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("\nthis is a test\n"));
    //check other attributes
    analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled").setExplain(true).setTokenizer("standard").addTokenFilter("snowball").get();
    assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball"));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3));
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl"));
    String[] expectedAttributesKey = { "bytes", "positionLength", "keyword" };
    assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length));
    Object extendedAttribute;
    for (String key : expectedAttributesKey) {
        extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key);
        assertThat(extendedAttribute, notNullValue());
    }
}

Example 23 with AnalyzeResponse

use of org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse in project elasticsearch by elastic.

the class AnalyzeActionIT method testDetailAnalyzeWithMultiValues.

public void testDetailAnalyzeWithMultiValues() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
    ensureGreen();
    client().admin().indices().preparePutMapping("test").setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get();
    String[] texts = new String[] { "THIS IS A TEST", "THE SECOND TEXT" };
    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts).setExplain(true).setField("simple").setText(texts).execute().get();
    assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple"));
    assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(7));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.detail().analyzer().getTokens()[3];
    assertThat(token.getTerm(), equalTo("test"));
    assertThat(token.getPosition(), equalTo(3));
    assertThat(token.getStartOffset(), equalTo(10));
    assertThat(token.getEndOffset(), equalTo(14));
    assertThat(token.getPositionLength(), equalTo(1));
    token = analyzeResponse.detail().analyzer().getTokens()[5];
    assertThat(token.getTerm(), equalTo("second"));
    assertThat(token.getPosition(), equalTo(105));
    assertThat(token.getStartOffset(), equalTo(19));
    assertThat(token.getEndOffset(), equalTo(25));
    assertThat(token.getPositionLength(), equalTo(1));
}

Example 24 with AnalyzeResponse

use of org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse in project graylog2-server by Graylog2.

the class Messages method analyze.

public List<String> analyze(String string, String index, String analyzer) {
    final AnalyzeResponse response = c.admin().indices().prepareAnalyze(index, string).setAnalyzer(analyzer).get();
    final List<AnalyzeToken> tokens = response.getTokens();
    final List<String> terms = new ArrayList<>(tokens.size());
    for (AnalyzeToken token : tokens) {
        terms.add(token.getTerm());
    }
    return terms;
}

Example 25 with AnalyzeResponse

use of org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse in project core-ng-project by neowu.

the class ElasticSearchTypeImpl method analyze.

@Override
public List<String> analyze(AnalyzeRequest request) {
    StopWatch watch = new StopWatch();
    String index = request.index == null ? this.index : request.index;
    try {
        AnalyzeResponse response = client().admin().indices().prepareAnalyze(index, request.text).setAnalyzer(request.analyzer).get();
        return response.getTokens().stream().map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toList());
    } catch (ElasticsearchException e) {
        // due to elastic search uses async executor to run, we have to wrap the exception to retain the original place caused the exception
        throw new SearchException(e);
    } finally {
        long elapsedTime = watch.elapsedTime();
        ActionLogContext.track("elasticsearch", elapsedTime);
        logger.debug("analyze, index={}, analyzer={}, elapsedTime={}", index, request.analyzer, elapsedTime);
        checkSlowOperation(elapsedTime);
    }
}