Examples with GatewayAllocator org.opensearch.gateway.GatewayAllocator used on opensource projects

Example 1 with GatewayAllocator

use of org.opensearch.gateway.GatewayAllocator in project OpenSearch by opensearch-project.

the class NodeLoadAwareAllocationTests method testSingleZoneOneReplicaLimitsShardAllocationOnOverload.

public void testSingleZoneOneReplicaLimitsShardAllocationOnOverload() {
    GatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    AllocationService strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 20, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), true), gatewayAllocator);
    logger.info("Building initial routing table for 'testSingleZoneOneReplicaLimitsShardAllocationOnOverload'");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(20).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding five nodes on same zone and do rerouting");
    clusterState = addNodes(clusterState, strategy, "zone_1", "node1", "node2", "node3", "node4", "node5");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(20));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(20));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(20));
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(40));
    logger.info("--> Remove node1 from zone");
    ClusterState newState = removeNodes(clusterState, strategy, "node1");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(40));
    for (RoutingNode node : newState.getRoutingNodes()) {
        assertThat(node.size(), equalTo(10));
    }
    logger.info("--> Remove node2 when the limit of overload is reached");
    newState = removeNodes(newState, strategy, "node2");
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(30));
    for (ShardRouting shard : newState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
        assertEquals(shard.unassignedInfo().getReason(), UnassignedInfo.Reason.NODE_LEFT);
        assertFalse(shard.primary());
    }
    logger.info("add another index with 20 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 20).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test1")).build();
    // increases avg shard per node to 80/5 = 16, overload factor 1.2, total allowed 20
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    newState = startInitializingShardsAndReroute(strategy, newState);
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(60));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(20));
    logger.info("add another index with 60 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 60).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test2")).build();
    // increases avg shard per node to 140/5 = 28, overload factor 1.2, total allowed 34 per node but still ALL primaries get assigned
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(120));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(20));
    logger.info("change settings to allow unassigned primaries");
    strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 20, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), false), gatewayAllocator);
    for (RoutingNode node : newState.getRoutingNodes()) {
        assertThat(node.size(), equalTo(40));
    }
    logger.info("add another index with 5 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test3").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test3")).build();
    // increases avg shard per node to 145/5 = 29, overload factor 1.2, total allowed 35 per node and NO primaries get assigned
    // since total owning shards are 40 per node already
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    newState = startInitializingShardsAndReroute(strategy, newState);
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(120));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(25));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().filter(ShardRouting::primary).count(), equalTo(5L));
}

Example 2 with GatewayAllocator

use of org.opensearch.gateway.GatewayAllocator in project OpenSearch by opensearch-project.

the class NodeLoadAwareAllocationTests method testExistingPrimariesAllocationOnOverload.

public void testExistingPrimariesAllocationOnOverload() {
    GatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    AllocationService strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 50, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), false), gatewayAllocator);
    logger.info("Building initial routing table for 'testExistingPrimariesAllocationOnOverload'");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(20).numberOfReplicas(0)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding five nodes on same zone and do rerouting");
    clusterState = addNodes(clusterState, strategy, "zone_1", "node1", "node2", "node3", "node4", "node5");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(20));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(20));
    logger.info("--> Remove nodes from zone holding primaries");
    ClusterState newState = removeNodes(clusterState, strategy, "node1", "node2", "node3");
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
    logger.info("add another index with 20 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 20).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test1")).build();
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    newState = startInitializingShardsAndReroute(strategy, newState);
    logger.info("limits should be applied on newly create primaries");
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(24));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(16));
    assertEquals(12L, newState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().filter(r -> r.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT).count());
    assertEquals(4L, newState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().filter(r -> r.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED).count());
    assertThat(newState.getRoutingNodes().node("node4").size(), equalTo(12));
    logger.info("--> Remove node4 from zone holding primaries");
    newState = removeNodes(newState, strategy, "node4");
    logger.info("--> change the overload load factor to zero and verify if unassigned primaries on disk get assigned despite overload");
    strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 0, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), false), gatewayAllocator);
    newState = strategy.reroute(newState, "reroute");
    logger.info("--> Add back node4 and ensure existing primaries are assigned");
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node4", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    logger.info("--> do another reroute, make sure nothing moves");
    assertThat(strategy.reroute(newState, "reroute").routingTable(), sameInstance(newState.routingTable()));
    assertThat(newState.getRoutingNodes().node("node4").size(), equalTo(12));
    assertThat(newState.getRoutingNodes().node("node5").size(), equalTo(12));
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(24));
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node1", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(32));
    // add back node2 when skewness is still breached
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node2", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(36));
    // add back node3
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node3", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(40));
}