Examples with IndexShardRoutingTable org.opensearch.cluster.routing.IndexShardRoutingTable used on opensource projects

Example 1 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class GatewayIndexStateIT method testRecoverBrokenIndexMetadata.

/**
 * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being
 * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
 * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
 */
public void testRecoverBrokenIndexMetadata() throws Exception {
    logger.info("--> starting one node");
    internalCluster().startNode();
    logger.info("--> indexing a simple document");
    client().prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
    logger.info("--> waiting for green status");
    if (usually()) {
        ensureYellow();
    } else {
        internalCluster().startNode();
        client().admin().cluster().health(Requests.clusterHealthRequest().waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true).waitForNodes("2")).actionGet();
    }
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    final IndexMetadata metadata = state.getMetadata().index("test");
    final IndexMetadata.Builder brokenMeta = IndexMetadata.builder(metadata).settings(Settings.builder().put(metadata.getSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.minimumIndexCompatibilityVersion().id).put("index.similarity.BM25.type", "classic").put("index.analysis.filter.myCollator.type", "icu_collation"));
    restartNodesOnBrokenClusterState(ClusterState.builder(state).metadata(Metadata.builder(state.getMetadata()).put(brokenMeta)));
    // check that the cluster does not keep reallocating shards
    assertBusy(() -> {
        final RoutingTable routingTable = client().admin().cluster().prepareState().get().getState().routingTable();
        final IndexRoutingTable indexRoutingTable = routingTable.index("test");
        assertNotNull(indexRoutingTable);
        for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
            assertTrue(shardRoutingTable.primaryShard().unassigned());
            assertEquals(UnassignedInfo.AllocationStatus.DECIDERS_NO, shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus());
            assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
        }
    }, 60, TimeUnit.SECONDS);
    client().admin().indices().prepareClose("test").get();
    state = client().admin().cluster().prepareState().get().getState();
    assertEquals(IndexMetadata.State.CLOSE, state.getMetadata().index(metadata.getIndex()).getState());
    assertEquals("classic", state.getMetadata().index(metadata.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
    // try to open it with the broken setting - fail again!
    OpenSearchException ex = expectThrows(OpenSearchException.class, () -> client().admin().indices().prepareOpen("test").get());
    assertEquals(ex.getMessage(), "Failed to verify index " + metadata.getIndex());
    assertNotNull(ex.getCause());
    assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
    assertEquals(ex.getCause().getMessage(), "Unknown filter type [icu_collation] for [myCollator]");
}

Example 2 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class AwarenessAllocationIT method testAwarenessZones.

public void testAwarenessZones() {
    Settings commonSettings = Settings.builder().put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a,b").put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone").build();
    logger.info("--> starting 4 nodes on different zones");
    List<String> nodes = internalCluster().startNodes(Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "a").build());
    String A_0 = nodes.get(0);
    String B_0 = nodes.get(1);
    String B_1 = nodes.get(2);
    String A_1 = nodes.get(3);
    logger.info("--> waiting for nodes to form a cluster");
    ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("4").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    createIndex("test", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
    if (randomBoolean()) {
        assertAcked(client().admin().indices().prepareClose("test"));
    }
    logger.info("--> waiting for shards to be allocated");
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    ObjectIntHashMap<String> counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_1), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(B_1), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(A_0), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(B_0), anyOf(equalTo(2), equalTo(3)));
}

Example 3 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class AwarenessAllocationIT method testAwarenessZonesIncrementalNodes.

public void testAwarenessZonesIncrementalNodes() {
    Settings commonSettings = Settings.builder().put("cluster.routing.allocation.awareness.force.zone.values", "a,b").put("cluster.routing.allocation.awareness.attributes", "zone").build();
    logger.info("--> starting 2 nodes on zones 'a' & 'b'");
    List<String> nodes = internalCluster().startNodes(Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build());
    String A_0 = nodes.get(0);
    String B_0 = nodes.get(1);
    createIndex("test", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
    if (randomBoolean()) {
        assertAcked(client().admin().indices().prepareClose("test"));
    }
    ClusterHealthResponse health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2").setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    ObjectIntHashMap<String> counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(5));
    assertThat(counts.get(B_0), equalTo(5));
    logger.info("--> starting another node in zone 'b'");
    String B_1 = internalCluster().startNode(Settings.builder().put(commonSettings).put("node.attr.zone", "b").build());
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("3").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    client().admin().cluster().prepareReroute().get();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("3").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(5));
    assertThat(counts.get(B_0), equalTo(3));
    assertThat(counts.get(B_1), equalTo(2));
    String noZoneNode = internalCluster().startNode();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    client().admin().cluster().prepareReroute().get();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(5));
    assertThat(counts.get(B_0), equalTo(3));
    assertThat(counts.get(B_1), equalTo(2));
    assertThat(counts.containsKey(noZoneNode), equalTo(false));
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.awareness.attributes", "").build()).get();
    health = client().admin().cluster().prepareHealth().setIndices("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("4").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_0), equalTo(3));
    assertThat(counts.get(B_0), equalTo(3));
    assertThat(counts.get(B_1), equalTo(2));
    assertThat(counts.get(noZoneNode), equalTo(2));
}

Example 4 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class IndexRecoveryIT method testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint.

public void testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    String indexName = "test-index";
    createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "12h").build());
    indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, between(0, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
    ensureGreen(indexName);
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final DiscoveryNodes discoveryNodes = clusterService().state().nodes();
    final IndexShardRoutingTable indexShardRoutingTable = clusterService().state().routingTable().shardRoutingTable(shardId);
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, discoveryNodes.get(indexShardRoutingTable.primaryShard().currentNodeId()).getName()).getShardOrNull(shardId);
    final ShardRouting replicaShardRouting = indexShardRoutingTable.replicaShards().get(0);
    internalCluster().restartNode(discoveryNodes.get(replicaShardRouting.currentNodeId()).getName(), new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(discoveryNodes.getSize() - 1)).setWaitForEvents(Priority.LANGUID).get().isTimedOut());
            indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, between(1, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
            // We do not guarantee that the replica can recover locally all the way to its own global checkpoint before starting
            // to recover from the primary, so we must be careful not to perform an operations-based recovery if this would require
            // some operations that are not being retained. Emulate this by advancing the lease ahead of the replica's GCP:
            primary.renewRetentionLease(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replicaShardRouting), primary.seqNoStats().getMaxSeqNo() + 1, ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE);
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen(indexName);
    // noinspection OptionalGetWithoutIsPresent because it fails the test if absent
    final RecoveryState recoveryState = client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName).stream().filter(rs -> rs.getPrimary() == false).findFirst().get();
    assertThat(recoveryState.getIndex().totalFileCount(), greaterThan(0));
}

Example 5 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class IndexRecoveryIT method testUsesFileBasedRecoveryIfOperationsBasedRecoveryWouldBeUnreasonable.

public void testUsesFileBasedRecoveryIfOperationsBasedRecoveryWouldBeUnreasonable() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    String indexName = "test-index";
    final Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "12h").put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms");
    final double reasonableOperationsBasedRecoveryProportion;
    if (randomBoolean()) {
        reasonableOperationsBasedRecoveryProportion = randomDoubleBetween(0.05, 0.99, true);
        settings.put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), reasonableOperationsBasedRecoveryProportion);
    } else {
        reasonableOperationsBasedRecoveryProportion = IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.get(Settings.EMPTY);
    }
    logger.info("--> performing ops-based recoveries up to [{}%] of docs", reasonableOperationsBasedRecoveryProportion * 100.0);
    createIndex(indexName, settings.build());
    indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(0, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
    ensureGreen(indexName);
    flush(indexName);
    // wait for all history to be discarded
    assertBusy(() -> {
        for (ShardStats shardStats : client().admin().indices().prepareStats(indexName).get().getShards()) {
            final long maxSeqNo = shardStats.getSeqNoStats().getMaxSeqNo();
            assertTrue(shardStats.getRetentionLeaseStats().retentionLeases() + " should discard history up to " + maxSeqNo, shardStats.getRetentionLeaseStats().retentionLeases().leases().stream().allMatch(l -> l.retainingSequenceNumber() == maxSeqNo + 1));
        }
    });
    // ensure that all operations are in the safe commit
    flush(indexName);
    final ShardStats shardStats = client().admin().indices().prepareStats(indexName).get().getShards()[0];
    final long docCount = shardStats.getStats().docs.getCount();
    assertThat(shardStats.getStats().docs.getDeleted(), equalTo(0L));
    assertThat(shardStats.getSeqNoStats().getMaxSeqNo() + 1, equalTo(docCount));
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final DiscoveryNodes discoveryNodes = clusterService().state().nodes();
    final IndexShardRoutingTable indexShardRoutingTable = clusterService().state().routingTable().shardRoutingTable(shardId);
    final ShardRouting replicaShardRouting = indexShardRoutingTable.replicaShards().get(0);
    assertTrue("should have lease for " + replicaShardRouting, client().admin().indices().prepareStats(indexName).get().getShards()[0].getRetentionLeaseStats().retentionLeases().contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replicaShardRouting)));
    internalCluster().restartNode(discoveryNodes.get(replicaShardRouting.currentNodeId()).getName(), new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(discoveryNodes.getSize() - 1)).setWaitForEvents(Priority.LANGUID).get().isTimedOut());
            final int newDocCount = Math.toIntExact(Math.round(Math.ceil((1 + Math.ceil(docCount * reasonableOperationsBasedRecoveryProportion)) / (1 - reasonableOperationsBasedRecoveryProportion))));
            /*
                     *     newDocCount >= (ceil(docCount * p) + 1) / (1-p)
                     *
                     * ==> 0 <= newDocCount * (1-p) - ceil(docCount * p) - 1
                     *       =  newDocCount - (newDocCount * p + ceil(docCount * p) + 1)
                     *       <  newDocCount - (ceil(newDocCount * p) + ceil(docCount * p))
                     *       <= newDocCount -  ceil(newDocCount * p + docCount * p)
                     *
                     * ==> docCount <  newDocCount + docCount - ceil((newDocCount + docCount) * p)
                     *              == localCheckpoint + 1    - ceil((newDocCount + docCount) * p)
                     *              == firstReasonableSeqNo
                     *
                     * The replica has docCount docs, i.e. has operations with seqnos [0..docCount-1], so a seqno-based recovery will start
                     * from docCount < firstReasonableSeqNo
                     *
                     * ==> it is unreasonable to recover the replica using a seqno-based recovery
                     */
            indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, newDocCount).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
            flush(indexName);
            assertBusy(() -> assertFalse("should no longer have lease for " + replicaShardRouting, client().admin().indices().prepareStats(indexName).get().getShards()[0].getRetentionLeaseStats().retentionLeases().contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replicaShardRouting))));
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen(indexName);
    // noinspection OptionalGetWithoutIsPresent because it fails the test if absent
    final RecoveryState recoveryState = client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName).stream().filter(rs -> rs.getPrimary() == false).findFirst().get();
    assertThat(recoveryState.getIndex().totalFileCount(), greaterThan(0));
}