Examples with IndexRoutingTable org.opensearch.cluster.routing.IndexRoutingTable used on opensource projects

Example 1 with IndexRoutingTable

use of org.opensearch.cluster.routing.IndexRoutingTable in project OpenSearch by opensearch-project.

the class GatewayIndexStateIT method testRecoverBrokenIndexMetadata.

/**
 * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being
 * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
 * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
 */
public void testRecoverBrokenIndexMetadata() throws Exception {
    logger.info("--> starting one node");
    internalCluster().startNode();
    logger.info("--> indexing a simple document");
    client().prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
    logger.info("--> waiting for green status");
    if (usually()) {
        ensureYellow();
    } else {
        internalCluster().startNode();
        client().admin().cluster().health(Requests.clusterHealthRequest().waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true).waitForNodes("2")).actionGet();
    }
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    final IndexMetadata metadata = state.getMetadata().index("test");
    final IndexMetadata.Builder brokenMeta = IndexMetadata.builder(metadata).settings(Settings.builder().put(metadata.getSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.minimumIndexCompatibilityVersion().id).put("index.similarity.BM25.type", "classic").put("index.analysis.filter.myCollator.type", "icu_collation"));
    restartNodesOnBrokenClusterState(ClusterState.builder(state).metadata(Metadata.builder(state.getMetadata()).put(brokenMeta)));
    // check that the cluster does not keep reallocating shards
    assertBusy(() -> {
        final RoutingTable routingTable = client().admin().cluster().prepareState().get().getState().routingTable();
        final IndexRoutingTable indexRoutingTable = routingTable.index("test");
        assertNotNull(indexRoutingTable);
        for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
            assertTrue(shardRoutingTable.primaryShard().unassigned());
            assertEquals(UnassignedInfo.AllocationStatus.DECIDERS_NO, shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus());
            assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
        }
    }, 60, TimeUnit.SECONDS);
    client().admin().indices().prepareClose("test").get();
    state = client().admin().cluster().prepareState().get().getState();
    assertEquals(IndexMetadata.State.CLOSE, state.getMetadata().index(metadata.getIndex()).getState());
    assertEquals("classic", state.getMetadata().index(metadata.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
    // try to open it with the broken setting - fail again!
    OpenSearchException ex = expectThrows(OpenSearchException.class, () -> client().admin().indices().prepareOpen("test").get());
    assertEquals(ex.getMessage(), "Failed to verify index " + metadata.getIndex());
    assertNotNull(ex.getCause());
    assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
    assertEquals(ex.getCause().getMessage(), "Unknown filter type [icu_collation] for [myCollator]");
}

Example 2 with IndexRoutingTable

use of org.opensearch.cluster.routing.IndexRoutingTable in project OpenSearch by opensearch-project.

the class AwarenessAllocationIT method testAwarenessZones.

public void testAwarenessZones() {
    Settings commonSettings = Settings.builder().put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a,b").put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone").build();
    logger.info("--> starting 4 nodes on different zones");
    List<String> nodes = internalCluster().startNodes(Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "a").build());
    String A_0 = nodes.get(0);
    String B_0 = nodes.get(1);
    String B_1 = nodes.get(2);
    String A_1 = nodes.get(3);
    logger.info("--> waiting for nodes to form a cluster");
    ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("4").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    createIndex("test", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
    if (randomBoolean()) {
        assertAcked(client().admin().indices().prepareClose("test"));
    }
    logger.info("--> waiting for shards to be allocated");
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    ObjectIntHashMap<String> counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_1), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(B_1), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(A_0), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(B_0), anyOf(equalTo(2), equalTo(3)));
}

Example 3 with IndexRoutingTable

use of org.opensearch.cluster.routing.IndexRoutingTable in project OpenSearch by opensearch-project.

the class AwarenessAllocationIT method testAwarenessZonesIncrementalNodes.

public void testAwarenessZonesIncrementalNodes() {
    Settings commonSettings = Settings.builder().put("cluster.routing.allocation.awareness.force.zone.values", "a,b").put("cluster.routing.allocation.awareness.attributes", "zone").build();
    logger.info("--> starting 2 nodes on zones 'a' & 'b'");
    List<String> nodes = internalCluster().startNodes(Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build());
    String A_0 = nodes.get(0);
    String B_0 = nodes.get(1);
    createIndex("test", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
    if (randomBoolean()) {
        assertAcked(client().admin().indices().prepareClose("test"));
    }
    ClusterHealthResponse health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2").setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    ObjectIntHashMap<String> counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(5));
    assertThat(counts.get(B_0), equalTo(5));
    logger.info("--> starting another node in zone 'b'");
    String B_1 = internalCluster().startNode(Settings.builder().put(commonSettings).put("node.attr.zone", "b").build());
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("3").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    client().admin().cluster().prepareReroute().get();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("3").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(5));
    assertThat(counts.get(B_0), equalTo(3));
    assertThat(counts.get(B_1), equalTo(2));
    String noZoneNode = internalCluster().startNode();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    client().admin().cluster().prepareReroute().get();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(5));
    assertThat(counts.get(B_0), equalTo(3));
    assertThat(counts.get(B_1), equalTo(2));
    assertThat(counts.containsKey(noZoneNode), equalTo(false));
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.awareness.attributes", "").build()).get();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(3));
    assertThat(counts.get(B_0), equalTo(3));
    assertThat(counts.get(B_1), equalTo(2));
    assertThat(counts.get(noZoneNode), equalTo(2));
}

Example 4 with IndexRoutingTable

use of org.opensearch.cluster.routing.IndexRoutingTable in project OpenSearch by opensearch-project.

the class TransportIndicesShardStoresAction method masterOperation.

@Override
protected void masterOperation(IndicesShardStoresRequest request, ClusterState state, ActionListener<IndicesShardStoresResponse> listener) {
    final RoutingTable routingTables = state.routingTable();
    final RoutingNodes routingNodes = state.getRoutingNodes();
    final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
    final Set<Tuple<ShardId, String>> shardsToFetch = new HashSet<>();
    logger.trace("using cluster state version [{}] to determine shards", state.version());
    // collect relevant shard ids of the requested indices for fetching store infos
    for (String index : concreteIndices) {
        IndexRoutingTable indexShardRoutingTables = routingTables.index(index);
        if (indexShardRoutingTables == null) {
            continue;
        }
        final String customDataPath = IndexMetadata.INDEX_DATA_PATH_SETTING.get(state.metadata().index(index).getSettings());
        for (IndexShardRoutingTable routing : indexShardRoutingTables) {
            final int shardId = routing.shardId().id();
            ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, routing);
            if (request.shardStatuses().contains(shardHealth.getStatus())) {
                shardsToFetch.add(Tuple.tuple(routing.shardId(), customDataPath));
            }
        }
    }
    // async fetch store infos from all the nodes
    // NOTE: instead of fetching shard store info one by one from every node (nShards * nNodes requests)
    // we could fetch all shard store info from every node once (nNodes requests)
    // we have to implement a TransportNodesAction instead of using TransportNodesListGatewayStartedShards
    // for fetching shard stores info, that operates on a list of shards instead of a single shard
    new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener).start();
}

Example 5 with IndexRoutingTable

use of org.opensearch.cluster.routing.IndexRoutingTable in project OpenSearch by opensearch-project.

the class ClusterStateDiffIT method randomChangeToIndexRoutingTable.

/**
 * Randomly updates index routing table in the cluster state
 */
private IndexRoutingTable randomChangeToIndexRoutingTable(IndexRoutingTable original, String[] nodes) {
    IndexRoutingTable.Builder builder = IndexRoutingTable.builder(original.getIndex());
    for (ObjectCursor<IndexShardRoutingTable> indexShardRoutingTable : original.shards().values()) {
        Set<String> availableNodes = Sets.newHashSet(nodes);
        for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
            availableNodes.remove(shardRouting.currentNodeId());
            if (shardRouting.relocating()) {
                availableNodes.remove(shardRouting.relocatingNodeId());
            }
        }
        for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
            final ShardRouting updatedShardRouting = randomChange(shardRouting, availableNodes);
            availableNodes.remove(updatedShardRouting.currentNodeId());
            if (shardRouting.relocating()) {
                availableNodes.remove(updatedShardRouting.relocatingNodeId());
            }
            builder.addShard(updatedShardRouting);
        }
    }
    return builder.build();
}