Examples with Environment org.opensearch.env.Environment used on opensource projects

Example 56 with Environment

use of org.opensearch.env.Environment in project OpenSearch by opensearch-project.

the class RemoveCorruptedShardDataCommandTests method testCorruptedTranslog.

public void testCorruptedTranslog() throws Exception {
    final int numDocsToKeep = indexDocs(indexShard, false);
    // close shard
    closeShards(indexShard);
    TestTranslog.corruptRandomTranslogFile(logger, random(), translogPath);
    // test corrupted shard
    final IndexShard corruptedShard = reopenIndexShard(true);
    allowShardFailures();
    // it has to fail on start up due to index.shard.check_on_startup = checksum
    final Exception exception = expectThrows(Exception.class, () -> newStartedShard(p -> corruptedShard, true));
    final Throwable cause = exception.getCause() instanceof EngineException ? exception.getCause().getCause() : exception.getCause();
    assertThat(cause, instanceOf(TranslogCorruptedException.class));
    // translog is corrupted already - do not check consistency
    closeShard(corruptedShard, false);
    final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
    final MockTerminal t = new MockTerminal();
    final OptionParser parser = command.getParser();
    final OptionSet options = parser.parse("-d", translogPath.toString());
    // run command with dry-run
    // mean dry run
    t.addTextInput("n");
    t.setVerbosity(Terminal.Verbosity.VERBOSE);
    try {
        command.execute(t, options, environment);
        fail();
    } catch (OpenSearchException e) {
        assertThat(e.getMessage(), containsString("aborted by user"));
        assertThat(t.getOutput(), containsString("Continue and remove corrupted data from the shard ?"));
    }
    logger.info("--> output:\n{}", t.getOutput());
    // run command without dry-run
    t.reset();
    t.addTextInput("y");
    command.execute(t, options, environment);
    final String output = t.getOutput();
    logger.info("--> output:\n{}", output);
    // reopen shard
    failOnShardFailures();
    final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
    final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
    assertThat(shardDocUIDs.size(), equalTo(numDocsToKeep));
    closeShards(newShard);
}

Example 57 with Environment

use of org.opensearch.env.Environment in project OpenSearch by opensearch-project.

the class RemoveCorruptedShardDataCommandTests method testCorruptedIndex.

public void testCorruptedIndex() throws Exception {
    final int numDocs = indexDocs(indexShard, true);
    // close shard
    closeShards(indexShard);
    final boolean corruptSegments = randomBoolean();
    CorruptionUtils.corruptIndex(random(), indexPath, corruptSegments);
    if (randomBoolean()) {
        // test corrupted shard and add corruption marker
        final IndexShard corruptedShard = reopenIndexShard(true);
        allowShardFailures();
        expectThrows(IndexShardRecoveryException.class, () -> newStartedShard(p -> corruptedShard, true));
        closeShards(corruptedShard);
    }
    final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
    final MockTerminal t = new MockTerminal();
    final OptionParser parser = command.getParser();
    // run command with dry-run
    // mean dry run
    t.addTextInput("n");
    final OptionSet options = parser.parse("-d", indexPath.toString());
    t.setVerbosity(Terminal.Verbosity.VERBOSE);
    try {
        command.execute(t, options, environment);
        fail();
    } catch (OpenSearchException e) {
        if (corruptSegments) {
            assertThat(e.getMessage(), either(is("Index is unrecoverable")).or(startsWith("unable to list commits")));
        } else {
            assertThat(e.getMessage(), containsString("aborted by user"));
        }
    } finally {
        logger.info("--> output:\n{}", t.getOutput());
    }
    if (corruptSegments == false) {
        // run command without dry-run
        t.addTextInput("y");
        command.execute(t, options, environment);
        final String output = t.getOutput();
        logger.info("--> output:\n{}", output);
        // reopen shard
        failOnShardFailures();
        final IndexShard newShard = newStartedShard(p -> reopenIndexShard(false), true);
        final Set<String> shardDocUIDs = getShardDocUIDs(newShard);
        final Matcher matcher = NUM_CORRUPT_DOCS_PATTERN.matcher(output);
        assertThat(matcher.find(), equalTo(true));
        final int expectedNumDocs = numDocs - Integer.parseInt(matcher.group("docs"));
        assertThat(shardDocUIDs.size(), equalTo(expectedNumDocs));
        closeShards(newShard);
    }
}

Example 58 with Environment

use of org.opensearch.env.Environment in project OpenSearch by opensearch-project.

the class AnalysisModuleTests method testRegisterHunspellDictionary.

public void testRegisterHunspellDictionary() throws Exception {
    Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
    Environment environment = TestEnvironment.newEnvironment(settings);
    InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff");
    InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic");
    Dictionary dictionary;
    try (Directory tmp = new NIOFSDirectory(environment.tmpFile())) {
        dictionary = new Dictionary(tmp, "hunspell", aff, dic);
    }
    AnalysisModule module = new AnalysisModule(environment, singletonList(new AnalysisPlugin() {

        @Override
        public Map<String, Dictionary> getHunspellDictionaries() {
            return singletonMap("foo", dictionary);
        }
    }));
    assertSame(dictionary, module.getHunspellService().getDictionary("foo"));
}

Example 59 with Environment

use of org.opensearch.env.Environment in project OpenSearch by opensearch-project.

the class AnalysisModuleTests method testWordListPath.

public void testWordListPath() throws Exception {
    Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
    Environment env = TestEnvironment.newEnvironment(settings);
    String[] words = new String[] { "donau", "dampf", "schiff", "spargel", "creme", "suppe" };
    Path wordListFile = generateWordList(words);
    settings = Settings.builder().loadFromSource("index: \n  word_list_path: " + wordListFile.toAbsolutePath(), XContentType.YAML).build();
    Set<?> wordList = Analysis.getWordSet(env, settings, "index.word_list");
    MatcherAssert.assertThat(wordList.size(), equalTo(6));
    // MatcherAssert.assertThat(wordList, hasItems(words));
    Files.delete(wordListFile);
}

Example 60 with Environment

use of org.opensearch.env.Environment in project OpenSearch by opensearch-project.

the class AnalysisModuleTests method testPluginPreConfiguredCharFilters.

/**
 * Tests that plugins can register pre-configured char filters that vary in behavior based on OpenSearch version, Lucene version,
 * and that do not vary based on version at all.
 */
public void testPluginPreConfiguredCharFilters() throws IOException {
    boolean noVersionSupportsMultiTerm = randomBoolean();
    boolean luceneVersionSupportsMultiTerm = randomBoolean();
    boolean opensearchVersionSupportsMultiTerm = randomBoolean();
    AnalysisRegistry registry = new AnalysisModule(TestEnvironment.newEnvironment(emptyNodeSettings), singletonList(new AnalysisPlugin() {

        @Override
        public List<PreConfiguredCharFilter> getPreConfiguredCharFilters() {
            return Arrays.asList(PreConfiguredCharFilter.singleton("no_version", noVersionSupportsMultiTerm, tokenStream -> new AppendCharFilter(tokenStream, "no_version")), PreConfiguredCharFilter.luceneVersion("lucene_version", luceneVersionSupportsMultiTerm, (tokenStream, luceneVersion) -> new AppendCharFilter(tokenStream, luceneVersion.toString())), PreConfiguredCharFilter.openSearchVersion("opensearch_version", opensearchVersionSupportsMultiTerm, (tokenStream, esVersion) -> new AppendCharFilter(tokenStream, esVersion.toString())));
        }

        @Override
        public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
            // Need mock keyword tokenizer here, because alpha / beta versions are broken up by the dash.
            return singletonMap("keyword", (indexSettings, environment, name, settings) -> TokenizerFactory.newFactory(name, () -> new MockTokenizer(MockTokenizer.KEYWORD, false)));
        }
    })).getAnalysisRegistry();
    Version version = VersionUtils.randomVersion(random());
    IndexAnalyzers analyzers = getIndexAnalyzers(registry, Settings.builder().put("index.analysis.analyzer.no_version.tokenizer", "keyword").put("index.analysis.analyzer.no_version.char_filter", "no_version").put("index.analysis.analyzer.lucene_version.tokenizer", "keyword").put("index.analysis.analyzer.lucene_version.char_filter", "lucene_version").put("index.analysis.analyzer.opensearch_version.tokenizer", "keyword").put("index.analysis.analyzer.opensearch_version.char_filter", "opensearch_version").put(IndexMetadata.SETTING_VERSION_CREATED, version).build());
    assertTokenStreamContents(analyzers.get("no_version").tokenStream("", "test"), new String[] { "testno_version" });
    assertTokenStreamContents(analyzers.get("lucene_version").tokenStream("", "test"), new String[] { "test" + version.luceneVersion });
    assertTokenStreamContents(analyzers.get("opensearch_version").tokenStream("", "test"), new String[] { "test" + version });
    assertEquals("test" + (noVersionSupportsMultiTerm ? "no_version" : ""), analyzers.get("no_version").normalize("", "test").utf8ToString());
    assertEquals("test" + (luceneVersionSupportsMultiTerm ? version.luceneVersion.toString() : ""), analyzers.get("lucene_version").normalize("", "test").utf8ToString());
    assertEquals("test" + (opensearchVersionSupportsMultiTerm ? version.toString() : ""), analyzers.get("opensearch_version").normalize("", "test").utf8ToString());
}