use of org.opensearch.indices.analysis.AnalysisModule in project OpenSearch by opensearch-project.
the class CompoundAnalysisTests method analyze.
private List<String> analyze(Settings settings, String analyzerName, String text) throws IOException {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
AnalysisModule analysisModule = createAnalysisModule(settings);
IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(idxSettings);
Analyzer analyzer = indexAnalyzers.get(analyzerName).analyzer();
TokenStream stream = analyzer.tokenStream("", text);
stream.reset();
CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
List<String> terms = new ArrayList<>();
while (stream.incrementToken()) {
String tokText = termAtt.toString();
terms.add(tokText);
}
return terms;
}
use of org.opensearch.indices.analysis.AnalysisModule in project OpenSearch by opensearch-project.
the class EdgeNGramTokenizerTests method buildAnalyzers.
private IndexAnalyzers buildAnalyzers(Version version, String tokenizer) throws IOException {
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).put("index.analysis.analyzer.my_analyzer.tokenizer", tokenizer).build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
return new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings);
}
use of org.opensearch.indices.analysis.AnalysisModule in project OpenSearch by opensearch-project.
the class AnalysisRegistryTests method setUp.
@Override
public void setUp() throws Exception {
super.setUp();
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
emptyRegistry = emptyAnalysisRegistry(settings);
// Module loaded to register in-built normalizers for testing
AnalysisModule module = new AnalysisModule(TestEnvironment.newEnvironment(settings), singletonList(new MockAnalysisPlugin()));
nonEmptyRegistry = module.getAnalysisRegistry();
}
use of org.opensearch.indices.analysis.AnalysisModule in project OpenSearch by opensearch-project.
the class AnalysisTestsHelper method createTestAnalysisFromSettings.
public static OpenSearchTestCase.TestAnalysis createTestAnalysisFromSettings(final Settings settings, final Path configPath, final AnalysisPlugin... plugins) throws IOException {
final Settings actualSettings;
if (settings.get(IndexMetadata.SETTING_VERSION_CREATED) == null) {
actualSettings = Settings.builder().put(settings).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
} else {
actualSettings = settings;
}
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", actualSettings);
final AnalysisRegistry analysisRegistry = new AnalysisModule(new Environment(actualSettings, configPath), Arrays.asList(plugins)).getAnalysisRegistry();
return new OpenSearchTestCase.TestAnalysis(analysisRegistry.build(indexSettings), analysisRegistry.buildTokenFilterFactories(indexSettings), analysisRegistry.buildTokenizerFactories(indexSettings), analysisRegistry.buildCharFilterFactories(indexSettings));
}
use of org.opensearch.indices.analysis.AnalysisModule in project OpenSearch by opensearch-project.
the class OpenSearchTestCase method createTestAnalysis.
/**
* Creates an TestAnalysis with all the default analyzers configured.
*/
public static TestAnalysis createTestAnalysis(IndexSettings indexSettings, Settings nodeSettings, AnalysisPlugin... analysisPlugins) throws IOException {
Environment env = TestEnvironment.newEnvironment(nodeSettings);
AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins));
AnalysisRegistry analysisRegistry = analysisModule.getAnalysisRegistry();
return new TestAnalysis(analysisRegistry.build(indexSettings), analysisRegistry.buildTokenFilterFactories(indexSettings), analysisRegistry.buildTokenizerFactories(indexSettings), analysisRegistry.buildCharFilterFactories(indexSettings));
}
Aggregations