Examples with IndexMetadata org.opensearch.cluster.metadata.IndexMetadata used on opensource projects

Example 1 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class PercolatorFieldMapperTests method testExtractTerms.

public void testExtractTerms() throws Exception {
    addQueryFieldMappings();
    BooleanQuery.Builder bq = new BooleanQuery.Builder();
    TermQuery termQuery1 = new TermQuery(new Term("field", "term1"));
    bq.add(termQuery1, Occur.SHOULD);
    TermQuery termQuery2 = new TermQuery(new Term("field", "term2"));
    bq.add(termQuery2, Occur.SHOULD);
    DocumentMapper documentMapper = mapperService.documentMapper("doc");
    PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
    IndexMetadata build = IndexMetadata.builder("").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
    IndexSettings settings = new IndexSettings(build, Settings.EMPTY);
    ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(settings, mapperService.documentMapperParser(), documentMapper, null, null);
    fieldMapper.processQuery(bq.build(), parseContext);
    ParseContext.Document document = parseContext.doc();
    PercolatorFieldMapper.PercolatorFieldType fieldType = (PercolatorFieldMapper.PercolatorFieldType) fieldMapper.fieldType();
    assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_COMPLETE));
    List<IndexableField> fields = new ArrayList<>(Arrays.asList(document.getFields(fieldType.queryTermsField.name())));
    fields.sort(Comparator.comparing(IndexableField::binaryValue));
    assertThat(fields.size(), equalTo(2));
    assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field\u0000term1"));
    assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field\u0000term2"));
    fields = new ArrayList<>(Arrays.asList(document.getFields(fieldType.minimumShouldMatchField.name())));
    assertThat(fields.size(), equalTo(1));
    assertThat(fields.get(0).numericValue(), equalTo(1L));
    // Now test conjunction:
    bq = new BooleanQuery.Builder();
    bq.add(termQuery1, Occur.MUST);
    bq.add(termQuery2, Occur.MUST);
    parseContext = new ParseContext.InternalParseContext(settings, mapperService.documentMapperParser(), documentMapper, null, null);
    fieldMapper.processQuery(bq.build(), parseContext);
    document = parseContext.doc();
    assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_COMPLETE));
    fields = new ArrayList<>(Arrays.asList(document.getFields(fieldType.queryTermsField.name())));
    fields.sort(Comparator.comparing(IndexableField::binaryValue));
    assertThat(fields.size(), equalTo(2));
    assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field\u0000term1"));
    assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field\u0000term2"));
    fields = new ArrayList<>(Arrays.asList(document.getFields(fieldType.minimumShouldMatchField.name())));
    assertThat(fields.size(), equalTo(1));
    assertThat(fields.get(0).numericValue(), equalTo(2L));
}

Example 2 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class PercolatorFieldMapperTests method testExtractTermsAndRanges_failed.

public void testExtractTermsAndRanges_failed() throws Exception {
    addQueryFieldMappings();
    TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
    DocumentMapper documentMapper = mapperService.documentMapper("doc");
    PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
    IndexMetadata build = IndexMetadata.builder("").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
    IndexSettings settings = new IndexSettings(build, Settings.EMPTY);
    ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(settings, mapperService.documentMapperParser(), documentMapper, null, null);
    fieldMapper.processQuery(query, parseContext);
    ParseContext.Document document = parseContext.doc();
    PercolatorFieldMapper.PercolatorFieldType fieldType = (PercolatorFieldMapper.PercolatorFieldType) fieldMapper.fieldType();
    assertThat(document.getFields().size(), equalTo(1));
    assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_FAILED));
}

Example 3 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class GatewayIndexStateIT method testRecoverBrokenIndexMetadata.

/**
 * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being
 * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
 * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
 */
public void testRecoverBrokenIndexMetadata() throws Exception {
    logger.info("--> starting one node");
    internalCluster().startNode();
    logger.info("--> indexing a simple document");
    client().prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
    logger.info("--> waiting for green status");
    if (usually()) {
        ensureYellow();
    } else {
        internalCluster().startNode();
        client().admin().cluster().health(Requests.clusterHealthRequest().waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true).waitForNodes("2")).actionGet();
    }
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    final IndexMetadata metadata = state.getMetadata().index("test");
    final IndexMetadata.Builder brokenMeta = IndexMetadata.builder(metadata).settings(Settings.builder().put(metadata.getSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.minimumIndexCompatibilityVersion().id).put("index.similarity.BM25.type", "classic").put("index.analysis.filter.myCollator.type", "icu_collation"));
    restartNodesOnBrokenClusterState(ClusterState.builder(state).metadata(Metadata.builder(state.getMetadata()).put(brokenMeta)));
    // check that the cluster does not keep reallocating shards
    assertBusy(() -> {
        final RoutingTable routingTable = client().admin().cluster().prepareState().get().getState().routingTable();
        final IndexRoutingTable indexRoutingTable = routingTable.index("test");
        assertNotNull(indexRoutingTable);
        for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
            assertTrue(shardRoutingTable.primaryShard().unassigned());
            assertEquals(UnassignedInfo.AllocationStatus.DECIDERS_NO, shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus());
            assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
        }
    }, 60, TimeUnit.SECONDS);
    client().admin().indices().prepareClose("test").get();
    state = client().admin().cluster().prepareState().get().getState();
    assertEquals(IndexMetadata.State.CLOSE, state.getMetadata().index(metadata.getIndex()).getState());
    assertEquals("classic", state.getMetadata().index(metadata.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
    // try to open it with the broken setting - fail again!
    OpenSearchException ex = expectThrows(OpenSearchException.class, () -> client().admin().indices().prepareOpen("test").get());
    assertEquals(ex.getMessage(), "Failed to verify index " + metadata.getIndex());
    assertNotNull(ex.getCause());
    assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
    assertEquals(ex.getCause().getMessage(), "Unknown filter type [icu_collation] for [myCollator]");
}

Example 4 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class SimpleClusterStateIT method testFilteringByIndexWorks.

/**
 * Retrieves the cluster state for the given indices and then checks
 * that the cluster state returns coherent data for both routing table and metadata.
 */
private void testFilteringByIndexWorks(String[] indices, String[] expected) {
    ClusterStateResponse clusterState = client().admin().cluster().prepareState().clear().setMetadata(true).setRoutingTable(true).setIndices(indices).get();
    ImmutableOpenMap<String, IndexMetadata> metadata = clusterState.getState().getMetadata().indices();
    assertThat(metadata.size(), is(expected.length));
    RoutingTable routingTable = clusterState.getState().getRoutingTable();
    assertThat(routingTable.indicesRouting().size(), is(expected.length));
    for (String expectedIndex : expected) {
        assertThat(metadata, CollectionAssertions.hasKey(expectedIndex));
        assertThat(routingTable.hasIndex(expectedIndex), is(true));
    }
}

Example 5 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class UnsafeBootstrapAndDetachCommandIT method test3MasterNodes2Failed.

public void test3MasterNodes2Failed() throws Exception {
    internalCluster().setBootstrapMasterNodeIndex(2);
    List<String> masterNodes = new ArrayList<>();
    logger.info("--> start 1st master-eligible node");
    masterNodes.add(internalCluster().startMasterOnlyNode(Settings.builder().put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()));
    // node ordinal 0
    logger.info("--> start one data-only node");
    String dataNode = internalCluster().startDataOnlyNode(Settings.builder().put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build());
    // node ordinal 1
    logger.info("--> start 2nd and 3rd master-eligible nodes and bootstrap");
    // node ordinals 2 and 3
    masterNodes.addAll(internalCluster().startMasterOnlyNodes(2));
    logger.info("--> wait for all nodes to join the cluster");
    ensureStableCluster(4);
    List<String> currentClusterNodes = new ArrayList<>(masterNodes);
    currentClusterNodes.add(dataNode);
    currentClusterNodes.forEach(node -> ensureReadOnlyBlock(false, node));
    logger.info("--> create index test");
    createIndex("test");
    ensureGreen("test");
    Settings master1DataPathSettings = internalCluster().dataPathSettings(masterNodes.get(0));
    Settings master2DataPathSettings = internalCluster().dataPathSettings(masterNodes.get(1));
    Settings master3DataPathSettings = internalCluster().dataPathSettings(masterNodes.get(2));
    Settings dataNodeDataPathSettings = internalCluster().dataPathSettings(dataNode);
    logger.info("--> stop 2nd and 3d master eligible node");
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1)));
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(2)));
    logger.info("--> ensure NO_MASTER_BLOCK on data-only node");
    assertBusy(() -> {
        ClusterState state = internalCluster().client(dataNode).admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
        assertTrue(state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID));
    });
    logger.info("--> try to unsafely bootstrap 1st master-eligible node, while node lock is held");
    Environment environmentMaster1 = TestEnvironment.newEnvironment(Settings.builder().put(internalCluster().getDefaultSettings()).put(master1DataPathSettings).build());
    expectThrows(() -> unsafeBootstrap(environmentMaster1), UnsafeBootstrapMasterCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG);
    logger.info("--> stop 1st master-eligible node and data-only node");
    NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class);
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(0)));
    assertBusy(() -> internalCluster().getInstance(GatewayMetaState.class, dataNode).allPendingAsyncStatesWritten());
    internalCluster().stopRandomDataNode();
    logger.info("--> unsafely-bootstrap 1st master-eligible node");
    MockTerminal terminal = unsafeBootstrap(environmentMaster1, false, true);
    Metadata metadata = OpenSearchNodeCommand.createPersistedClusterStateService(Settings.EMPTY, nodeEnvironment.nodeDataPaths()).loadBestOnDiskState().metadata;
    assertThat(terminal.getOutput(), containsString(String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, metadata.coordinationMetadata().term(), metadata.version())));
    logger.info("--> start 1st master-eligible node");
    String masterNode2 = internalCluster().startMasterOnlyNode(master1DataPathSettings);
    logger.info("--> detach-cluster on data-only node");
    Environment environmentData = TestEnvironment.newEnvironment(Settings.builder().put(internalCluster().getDefaultSettings()).put(dataNodeDataPathSettings).build());
    detachCluster(environmentData, false);
    logger.info("--> start data-only node");
    String dataNode2 = internalCluster().startDataOnlyNode(dataNodeDataPathSettings);
    logger.info("--> ensure there is no NO_MASTER_BLOCK and unsafe-bootstrap is reflected in cluster state");
    assertBusy(() -> {
        ClusterState state = internalCluster().client(dataNode2).admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
        assertFalse(state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID));
        assertTrue(state.metadata().persistentSettings().getAsBoolean(UnsafeBootstrapMasterCommand.UNSAFE_BOOTSTRAP.getKey(), false));
    });
    List<String> bootstrappedNodes = new ArrayList<>();
    bootstrappedNodes.add(dataNode2);
    bootstrappedNodes.add(masterNode2);
    bootstrappedNodes.forEach(node -> ensureReadOnlyBlock(true, node));
    logger.info("--> ensure index test is green");
    ensureGreen("test");
    IndexMetadata indexMetadata = clusterService().state().metadata().index("test");
    assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
    logger.info("--> detach-cluster on 2nd and 3rd master-eligible nodes");
    Environment environmentMaster2 = TestEnvironment.newEnvironment(Settings.builder().put(internalCluster().getDefaultSettings()).put(master2DataPathSettings).build());
    detachCluster(environmentMaster2, false);
    Environment environmentMaster3 = TestEnvironment.newEnvironment(Settings.builder().put(internalCluster().getDefaultSettings()).put(master3DataPathSettings).build());
    detachCluster(environmentMaster3, false);
    logger.info("--> start 2nd and 3rd master-eligible nodes and ensure 4 nodes stable cluster");
    bootstrappedNodes.add(internalCluster().startMasterOnlyNode(master2DataPathSettings));
    bootstrappedNodes.add(internalCluster().startMasterOnlyNode(master3DataPathSettings));
    ensureStableCluster(4);
    bootstrappedNodes.forEach(node -> ensureReadOnlyBlock(true, node));
    removeBlock();
}