Examples with RoutingTable org.opensearch.cluster.routing.RoutingTable used on opensource projects

Example 61 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class IndexBalanceTests method testBalanceAllNodesStartedAddIndex.

public void testBalanceAllNodesStartedAddIndex() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put("cluster.routing.allocation.node_initial_primaries_recoveries", 10).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(), 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    logger.info("Building initial routing table");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).currentNodeId(), nullValue());
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).currentNodeId(), nullValue());
    }
    logger.info("Adding three node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    newState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
        // backup shards are initializing as well, we make sure that they
        // recover from primary *started* shards in the
        // IndicesClusterStateService
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
    }
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    newState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
    }
    assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
    logger.info("Add new index 3 shards 1 replica");
    Metadata updatedMetadata = Metadata.builder(clusterState.metadata()).put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(updatedMetadata.index("test1")).build();
    clusterState = ClusterState.builder(clusterState).metadata(updatedMetadata).routingTable(updatedRoutingTable).build();
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    newState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
        // backup shards are initializing as well, we make sure that they
        // recover from primary *started* shards in the
        // IndicesClusterStateService
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
    }
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    newState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    routingNodes = clusterState.getRoutingNodes();
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
    }
    assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
}

Example 62 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class AllocationCommandsTests method testAllocateStalePrimaryCommand.

public void testAllocateStalePrimaryCommand() {
    AllocationService allocation = createAllocationService(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none").put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none").build());
    final String index = "test";
    logger.info("--> building initial routing table");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1).putInSyncAllocationIds(0, Collections.singleton("asdf")).putInSyncAllocationIds(1, Collections.singleton("qwertz"))).build();
    // shard routing is added as "from recovery" instead of "new index creation" so that we can test below that allocating an empty
    // primary with accept_data_loss flag set to false fails
    RoutingTable routingTable = RoutingTable.builder().addAsRecovery(metadata.index(index)).build();
    ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).build();
    final String node1 = "node1";
    final String node2 = "node2";
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode(node1)).add(newNode(node2))).build();
    clusterState = allocation.reroute(clusterState, "reroute");
    // mark all shards as stale
    final List<ShardRouting> shardRoutings = clusterState.getRoutingNodes().shardsWithState(UNASSIGNED);
    assertThat(shardRoutings, hasSize(2));
    logger.info("--> allocating empty primary with acceptDataLoss flag set to true");
    clusterState = allocation.reroute(clusterState, new AllocationCommands(new AllocateStalePrimaryAllocationCommand(index, 0, node1, true)), false, false).getClusterState();
    RoutingNode routingNode1 = clusterState.getRoutingNodes().node(node1);
    assertThat(routingNode1.size(), equalTo(1));
    assertThat(routingNode1.shardsWithState(INITIALIZING).size(), equalTo(1));
    Set<String> inSyncAllocationIds = clusterState.metadata().index(index).inSyncAllocationIds(0);
    assertThat(inSyncAllocationIds, equalTo(Collections.singleton(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID)));
    clusterState = startInitializingShardsAndReroute(allocation, clusterState);
    routingNode1 = clusterState.getRoutingNodes().node(node1);
    assertThat(routingNode1.size(), equalTo(1));
    assertThat(routingNode1.shardsWithState(STARTED).size(), equalTo(1));
    inSyncAllocationIds = clusterState.metadata().index(index).inSyncAllocationIds(0);
    assertThat(inSyncAllocationIds, hasSize(1));
    assertThat(inSyncAllocationIds, not(Collections.singleton(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID)));
}

Example 63 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class AllocationServiceTests method testExplainsNonAllocationOfShardWithUnknownAllocator.

public void testExplainsNonAllocationOfShardWithUnknownAllocator() {
    final AllocationService allocationService = new AllocationService(null, null, null, null);
    allocationService.setExistingShardsAllocators(Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, new TestGatewayAllocator()));
    final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
    nodesBuilder.add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT));
    nodesBuilder.add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT));
    final Metadata.Builder metadata = Metadata.builder().put(indexMetadata("index", Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "unknown")));
    final RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsRecovery(metadata.get("index"));
    final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(nodesBuilder).metadata(metadata).routingTable(routingTableBuilder.build()).build();
    final RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.emptyList()), clusterState.getRoutingNodes(), clusterState, ClusterInfo.EMPTY, null, 0L);
    allocation.setDebugMode(randomBoolean() ? RoutingAllocation.DebugMode.ON : RoutingAllocation.DebugMode.EXCLUDE_YES_DECISIONS);
    final ShardAllocationDecision shardAllocationDecision = allocationService.explainShardAllocation(clusterState.routingTable().index("index").shard(0).primaryShard(), allocation);
    assertTrue(shardAllocationDecision.isDecisionTaken());
    assertThat(shardAllocationDecision.getAllocateDecision().getAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.NO_VALID_SHARD_COPY));
    assertThat(shardAllocationDecision.getAllocateDecision().getAllocationDecision(), equalTo(AllocationDecision.NO_VALID_SHARD_COPY));
    assertThat(shardAllocationDecision.getAllocateDecision().getExplanation(), equalTo("cannot allocate because a previous copy of " + "the primary shard existed but can no longer be found on the nodes in the cluster"));
    for (NodeAllocationResult nodeAllocationResult : shardAllocationDecision.getAllocateDecision().nodeDecisions) {
        assertThat(nodeAllocationResult.getNodeDecision(), equalTo(AllocationDecision.NO));
        assertThat(nodeAllocationResult.getCanAllocateDecision().type(), equalTo(Decision.Type.NO));
        assertThat(nodeAllocationResult.getCanAllocateDecision().label(), equalTo("allocator_plugin"));
        assertThat(nodeAllocationResult.getCanAllocateDecision().getExplanation(), equalTo("finding the previous copies of this " + "shard requires an allocator called [unknown] but that allocator was not found; perhaps the corresponding plugin is " + "not installed"));
    }
}

Example 64 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class ShardFailedClusterStateTaskExecutorTests method createClusterStateWithStartedShards.

private ClusterState createClusterStateWithStartedShards(String reason) {
    int numberOfNodes = 1 + numberOfReplicas;
    DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
    IntStream.rangeClosed(1, numberOfNodes).mapToObj(node -> newNode("node" + node)).forEach(nodes::add);
    ClusterState stateAfterAddingNode = ClusterState.builder(clusterState).nodes(nodes).build();
    RoutingTable afterReroute = allocationService.reroute(stateAfterAddingNode, reason).routingTable();
    ClusterState stateAfterReroute = ClusterState.builder(stateAfterAddingNode).routingTable(afterReroute).build();
    return OpenSearchAllocationTestCase.startInitializingShardsAndReroute(allocationService, stateAfterReroute);
}

Example 65 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class MetadataCreateIndexServiceTests method runPrepareResizeIndexSettingsTest.

private void runPrepareResizeIndexSettingsTest(final Settings sourceSettings, final Settings requestSettings, final Collection<Setting<?>> additionalIndexScopedSettings, final boolean copySettings, final Consumer<Settings> consumer) {
    final String indexName = randomAlphaOfLength(10);
    final Settings indexSettings = Settings.builder().put("index.blocks.write", true).put("index.routing.allocation.require._name", "node1").put(sourceSettings).build();
    final ClusterState initialClusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0, indexSettings)).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    final AllocationService service = new AllocationService(new AllocationDeciders(singleton(new MaxRetryAllocationDecider())), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE, EmptySnapshotsInfoService.INSTANCE);
    final RoutingTable initialRoutingTable = service.reroute(initialClusterState, "reroute").routingTable();
    final ClusterState routingTableClusterState = ClusterState.builder(initialClusterState).routingTable(initialRoutingTable).build();
    // now we start the shard
    final RoutingTable routingTable = OpenSearchAllocationTestCase.startInitializingShardsAndReroute(service, routingTableClusterState, indexName).routingTable();
    final ClusterState clusterState = ClusterState.builder(routingTableClusterState).routingTable(routingTable).build();
    final Settings.Builder indexSettingsBuilder = Settings.builder().put("index.number_of_shards", 1).put(requestSettings);
    final Set<Setting<?>> settingsSet = Stream.concat(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS.stream(), additionalIndexScopedSettings.stream()).collect(Collectors.toSet());
    MetadataCreateIndexService.prepareResizeIndexSettings(clusterState, indexSettingsBuilder, clusterState.metadata().index(indexName).getIndex(), "target", ResizeType.SHRINK, copySettings, new IndexScopedSettings(Settings.EMPTY, settingsSet));
    consumer.accept(indexSettingsBuilder.build());
}