Examples with RoutingTable org.opensearch.cluster.routing.RoutingTable used on opensource projects

Example 16 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class GatewayIndexStateIT method testRecoverBrokenIndexMetadata.

/**
 * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being
 * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
 * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
 */
public void testRecoverBrokenIndexMetadata() throws Exception {
    logger.info("--> starting one node");
    internalCluster().startNode();
    logger.info("--> indexing a simple document");
    client().prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
    logger.info("--> waiting for green status");
    if (usually()) {
        ensureYellow();
    } else {
        internalCluster().startNode();
        client().admin().cluster().health(Requests.clusterHealthRequest().waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true).waitForNodes("2")).actionGet();
    }
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    final IndexMetadata metadata = state.getMetadata().index("test");
    final IndexMetadata.Builder brokenMeta = IndexMetadata.builder(metadata).settings(Settings.builder().put(metadata.getSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.minimumIndexCompatibilityVersion().id).put("index.similarity.BM25.type", "classic").put("index.analysis.filter.myCollator.type", "icu_collation"));
    restartNodesOnBrokenClusterState(ClusterState.builder(state).metadata(Metadata.builder(state.getMetadata()).put(brokenMeta)));
    // check that the cluster does not keep reallocating shards
    assertBusy(() -> {
        final RoutingTable routingTable = client().admin().cluster().prepareState().get().getState().routingTable();
        final IndexRoutingTable indexRoutingTable = routingTable.index("test");
        assertNotNull(indexRoutingTable);
        for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
            assertTrue(shardRoutingTable.primaryShard().unassigned());
            assertEquals(UnassignedInfo.AllocationStatus.DECIDERS_NO, shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus());
            assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
        }
    }, 60, TimeUnit.SECONDS);
    client().admin().indices().prepareClose("test").get();
    state = client().admin().cluster().prepareState().get().getState();
    assertEquals(IndexMetadata.State.CLOSE, state.getMetadata().index(metadata.getIndex()).getState());
    assertEquals("classic", state.getMetadata().index(metadata.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
    // try to open it with the broken setting - fail again!
    OpenSearchException ex = expectThrows(OpenSearchException.class, () -> client().admin().indices().prepareOpen("test").get());
    assertEquals(ex.getMessage(), "Failed to verify index " + metadata.getIndex());
    assertNotNull(ex.getCause());
    assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
    assertEquals(ex.getCause().getMessage(), "Unknown filter type [icu_collation] for [myCollator]");
}

Example 17 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class AllocationBenchmark method setUp.

@Setup
public void setUp() throws Exception {
    final String[] params = indicesShardsReplicasSourceTargetRecoveries.split("\\|");
    numIndices = toInt(params[0]);
    numShards = toInt(params[1]);
    numReplicas = toInt(params[2]);
    sourceNodes = toInt(params[3]);
    targetNodes = toInt(params[4]);
    concurrentRecoveries = toInt(params[5]);
    int totalShardCount = (numReplicas + 1) * numShards * numIndices;
    initialClusterStrategy = Allocators.createAllocationService(Settings.builder().put("cluster.routing.allocation.awareness.attributes", "zone").put("cluster.routing.allocation.node_concurrent_recoveries", "20").put("cluster.routing.allocation.exclude.tag", "tag_0").build());
    // We'll try to move nodes from tag_1 to tag_0
    clusterConcurrentRecoveries = Math.min(sourceNodes, targetNodes) * concurrentRecoveries;
    Metadata.Builder mb = Metadata.builder();
    for (int i = 1; i <= numIndices; i++) {
        mb.put(IndexMetadata.builder("test_" + i).settings(Settings.builder().put("index.version.created", Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas));
    }
    Metadata metadata = mb.build();
    RoutingTable.Builder rb = RoutingTable.builder();
    for (int i = 1; i <= numIndices; i++) {
        rb.addAsNew(metadata.index("test_" + i));
    }
    RoutingTable routingTable = rb.build();
    initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).nodes(setUpClusterNodes(sourceNodes, targetNodes)).build();
    // Start all unassigned shards
    initialClusterState = initialClusterStrategy.reroute(initialClusterState, "reroute");
    while (initialClusterState.getRoutingNodes().hasUnassignedShards()) {
        initialClusterState = initialClusterStrategy.applyStartedShards(initialClusterState, initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING));
        initialClusterState = initialClusterStrategy.reroute(initialClusterState, "reroute");
    }
    // Ensure all shards are started
    while (initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size() > 0) {
        initialClusterState = initialClusterStrategy.applyStartedShards(initialClusterState, initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING));
    }
    assert (initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size() == totalShardCount);
    assert (initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size() == 0);
    assert (initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size() == 0);
    // make sure shards are only allocated on tag1
    for (ShardRouting startedShard : initialClusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED)) {
        assert (initialClusterState.getRoutingNodes().node(startedShard.currentNodeId()).node().getAttributes().get("tag")).equals("tag_1");
    }
}

Example 18 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class SimpleClusterStateIT method testFilteringByIndexWorks.

/**
 * Retrieves the cluster state for the given indices and then checks
 * that the cluster state returns coherent data for both routing table and metadata.
 */
private void testFilteringByIndexWorks(String[] indices, String[] expected) {
    ClusterStateResponse clusterState = client().admin().cluster().prepareState().clear().setMetadata(true).setRoutingTable(true).setIndices(indices).get();
    ImmutableOpenMap<String, IndexMetadata> metadata = clusterState.getState().getMetadata().indices();
    assertThat(metadata.size(), is(expected.length));
    RoutingTable routingTable = clusterState.getState().getRoutingTable();
    assertThat(routingTable.indicesRouting().size(), is(expected.length));
    for (String expectedIndex : expected) {
        assertThat(metadata, CollectionAssertions.hasKey(expectedIndex));
        assertThat(routingTable.hasIndex(expectedIndex), is(true));
    }
}

Example 19 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class MetadataCreateIndexServiceTests method runPrepareResizeIndexSettingsTest.

private void runPrepareResizeIndexSettingsTest(final Settings sourceSettings, final Settings requestSettings, final Collection<Setting<?>> additionalIndexScopedSettings, final boolean copySettings, final Consumer<Settings> consumer) {
    final String indexName = randomAlphaOfLength(10);
    final Settings indexSettings = Settings.builder().put("index.blocks.write", true).put("index.routing.allocation.require._name", "node1").put(sourceSettings).build();
    final ClusterState initialClusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0, indexSettings)).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    final AllocationService service = new AllocationService(new AllocationDeciders(singleton(new MaxRetryAllocationDecider())), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE, EmptySnapshotsInfoService.INSTANCE);
    final RoutingTable initialRoutingTable = service.reroute(initialClusterState, "reroute").routingTable();
    final ClusterState routingTableClusterState = ClusterState.builder(initialClusterState).routingTable(initialRoutingTable).build();
    // now we start the shard
    final RoutingTable routingTable = OpenSearchAllocationTestCase.startInitializingShardsAndReroute(service, routingTableClusterState, indexName).routingTable();
    final ClusterState clusterState = ClusterState.builder(routingTableClusterState).routingTable(routingTable).build();
    final Settings.Builder indexSettingsBuilder = Settings.builder().put("index.number_of_shards", 1).put(requestSettings);
    final Set<Setting<?>> settingsSet = Stream.concat(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS.stream(), additionalIndexScopedSettings.stream()).collect(Collectors.toSet());
    MetadataCreateIndexService.prepareResizeIndexSettings(clusterState, indexSettingsBuilder, clusterState.metadata().index(indexName).getIndex(), "target", ResizeType.SHRINK, copySettings, new IndexScopedSettings(Settings.EMPTY, settingsSet));
    consumer.accept(indexSettingsBuilder.build());
}

Example 20 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class MetadataCreateIndexServiceTests method testValidateShrinkIndex.

public void testValidateShrinkIndex() {
    int numShards = randomIntBetween(2, 42);
    ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build());
    assertEquals("index [source] already exists", expectThrows(ResourceAlreadyExistsException.class, () -> MetadataCreateIndexService.validateShrinkIndex(state, "target", "source", Settings.EMPTY)).getMessage());
    assertEquals("no such index [no_such_index]", expectThrows(IndexNotFoundException.class, () -> MetadataCreateIndexService.validateShrinkIndex(state, "no_such_index", "target", Settings.EMPTY)).getMessage());
    Settings targetSettings = Settings.builder().put("index.number_of_shards", 1).build();
    assertEquals("can't shrink an index with only one shard", expectThrows(IllegalArgumentException.class, () -> MetadataCreateIndexService.validateShrinkIndex(createClusterState("source", 1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", "target", targetSettings)).getMessage());
    assertEquals("the number of target shards [10] must be less that the number of source shards [5]", expectThrows(IllegalArgumentException.class, () -> MetadataCreateIndexService.validateShrinkIndex(createClusterState("source", 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", "target", Settings.builder().put("index.number_of_shards", 10).build())).getMessage());
    assertEquals("index source must be read-only to resize index. use \"index.blocks.write=true\"", expectThrows(IllegalStateException.class, () -> MetadataCreateIndexService.validateShrinkIndex(createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY), "source", "target", targetSettings)).getMessage());
    assertEquals("index source must have all shards allocated on the same node to shrink index", expectThrows(IllegalStateException.class, () -> MetadataCreateIndexService.validateShrinkIndex(state, "source", "target", targetSettings)).getMessage());
    assertEquals("the number of source shards [8] must be a multiple of [3]", expectThrows(IllegalArgumentException.class, () -> MetadataCreateIndexService.validateShrinkIndex(createClusterState("source", 8, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build()), "source", "target", Settings.builder().put("index.number_of_shards", 3).build())).getMessage());
    // create one that won't fail
    ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    AllocationService service = new AllocationService(new AllocationDeciders(singleton(new MaxRetryAllocationDecider())), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE, EmptySnapshotsInfoService.INSTANCE);
    RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    // now we start the shard
    routingTable = OpenSearchAllocationTestCase.startInitializingShardsAndReroute(service, clusterState, "source").routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    int targetShards;
    do {
        targetShards = randomIntBetween(1, numShards / 2);
    } while (isShrinkable(numShards, targetShards) == false);
    MetadataCreateIndexService.validateShrinkIndex(clusterState, "source", "target", Settings.builder().put("index.number_of_shards", targetShards).build());
}