Examples with INITIALIZING org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING used on opensource projects

Example 1 with INITIALIZING

use of org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING in project OpenSearch by opensearch-project.

the class DiskThresholdDeciderTests method testDiskThresholdWithSnapshotShardSizes.

public void testDiskThresholdWithSnapshotShardSizes() {
    final long shardSizeInBytes = randomBoolean() ? 10L : 50L;
    logger.info("--> using shard size [{}]", shardSizeInBytes);
    final Settings diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%").put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "95%").build();
    final ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
    // 79% used
    usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 21));
    // 99% used
    usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 1));
    final ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
    final ClusterInfoService clusterInfoService = () -> new DevNullClusterInfo(usages, usages, ImmutableOpenMap.of());
    final AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new RestoreInProgressAllocationDecider(), new SameShardAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), makeDecider(diskSettings))));
    final Snapshot snapshot = new Snapshot("_repository", new SnapshotId("_snapshot_name", UUIDs.randomBase64UUID(random())));
    final IndexId indexId = new IndexId("_indexid_name", UUIDs.randomBase64UUID(random()));
    final ShardId shardId = new ShardId(new Index("test", IndexMetadata.INDEX_UUID_NA_VALUE), 0);
    final Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).putInSyncAllocationIds(0, singleton(AllocationId.newInitializing().getId()))).build();
    final RoutingTable routingTable = RoutingTable.builder().addAsNewRestore(metadata.index("test"), new RecoverySource.SnapshotRecoverySource("_restore_uuid", snapshot, Version.CURRENT, indexId), new IntHashSet()).build();
    final ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shards = ImmutableOpenMap.builder();
    shards.put(shardId, new RestoreInProgress.ShardRestoreStatus("node1"));
    final RestoreInProgress.Builder restores = new RestoreInProgress.Builder().add(new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, singletonList("test"), shards.build()));
    ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName())).metadata(metadata).routingTable(routingTable).putCustom(RestoreInProgress.TYPE, restores.build()).nodes(// node2 is added because DiskThresholdDecider
    DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo).allMatch(unassignedInfo -> Reason.NEW_INDEX_RESTORED.equals(unassignedInfo.getReason())), is(true));
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo).allMatch(unassignedInfo -> AllocationStatus.NO_ATTEMPT.equals(unassignedInfo.getLastAllocationStatus())), is(true));
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1));
    final AtomicReference<SnapshotShardSizeInfo> snapshotShardSizeInfoRef = new AtomicReference<>(SnapshotShardSizeInfo.EMPTY);
    final AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), clusterInfoService, snapshotShardSizeInfoRef::get);
    // reroute triggers snapshot shard size fetching
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // shard cannot be assigned yet as the snapshot shard size is unknown
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo).allMatch(unassignedInfo -> AllocationStatus.FETCHING_SHARD_DATA.equals(unassignedInfo.getLastAllocationStatus())), is(true));
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1));
    final SnapshotShard snapshotShard = new SnapshotShard(snapshot, indexId, shardId);
    final ImmutableOpenMap.Builder<SnapshotShard, Long> snapshotShardSizes = ImmutableOpenMap.builder();
    final boolean shouldAllocate;
    if (randomBoolean()) {
        logger.info("--> simulating snapshot shards size retrieval success");
        snapshotShardSizes.put(snapshotShard, shardSizeInBytes);
        logger.info("--> shard allocation depends on its size");
        shouldAllocate = shardSizeInBytes < usages.get("node1").getFreeBytes();
    } else {
        logger.info("--> simulating snapshot shards size retrieval failure");
        snapshotShardSizes.put(snapshotShard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
        logger.info("--> shard is always allocated when its size could not be retrieved");
        shouldAllocate = true;
    }
    snapshotShardSizeInfoRef.set(new SnapshotShardSizeInfo(snapshotShardSizes.build()));
    // reroute uses the previous snapshot shard size
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logShardStates(clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(shouldAllocate ? 0 : 1));
    assertThat(clusterState.getRoutingNodes().shardsWithState("test", INITIALIZING, STARTED).size(), equalTo(shouldAllocate ? 1 : 0));
}

Example 2 with INITIALIZING

use of org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING in project OpenSearch by opensearch-project.

the class ThrottlingAllocationTests method testRecoveryCounts.

public void testRecoveryCounts() {
    TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    Settings settings = Settings.builder().put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey(), 1).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(), 20).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(), 2).build();
    AllocationService strategy = createAllocationService(settings, gatewayAllocator);
    Metadata metadata = Metadata.builder().persistentSettings(settings).build();
    ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    logger.info("create a new index and start all primaries");
    clusterState = createNewIndexAndStartAllPrimaries(2, 2, strategy, clusterState);
    logger.info("Add a new node for all replica assignment. 4 replicas move to INIT after reroute.");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(INITIALIZING).size(), equalTo(4));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node1"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node2"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node3"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node1"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node2"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node3"), equalTo(0));
    logger.info("Exclude node1 and add node4. After this, primary shard on node1 moves to RELOCATING state and only " + "non initial replica recoveries are impacted");
    settings = Settings.builder().put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey(), 1).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(), 20).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(), 2).put("cluster.routing.allocation.exclude._id", "node1").build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node4"))).build();
    strategy = createAllocationService(settings, gatewayAllocator);
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(INITIALIZING).size(), equalTo(5));
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(RELOCATING).size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node4"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node1"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node2"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node3"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node4"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node4"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node1"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node2"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node4"), equalTo(0));
    logger.info("Start primary on node4. Now all replicas for that primary start getting accounted for from node4.");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().index("new_index").shardsWithState(INITIALIZING).stream().filter(routing -> routing.primary() && routing.isRelocationTarget()).collect(Collectors.toList()));
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(INITIALIZING).size(), equalTo(4));
    assertThat(clusterState.routingTable().index("new_index").shardsWithState(RELOCATING).size(), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getIncomingRecoveries("node4"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node1"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node2"), equalTo(1));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node3"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialIncomingRecoveries("node4"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getOutgoingRecoveries("node4"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node1"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node2"), equalTo(2));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node3"), equalTo(0));
    assertThat(clusterState.getRoutingNodes().getInitialOutgoingRecoveries("node4"), equalTo(2));
}

Example 3 with INITIALIZING

use of org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING in project OpenSearch by opensearch-project.

the class AllocationPriorityTests method testPrioritizedIndicesAllocatedFirst.

/**
 * Tests that higher prioritized primaries and replicas are allocated first even on the balanced shard allocator
 * See https://github.com/elastic/elasticsearch/issues/13249 for details
 */
public void testPrioritizedIndicesAllocatedFirst() {
    AllocationService allocation = createAllocationService(Settings.builder().put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey(), 1).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 10).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(), 1).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(), 1).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 1).build());
    final String highPriorityName;
    final String lowPriorityName;
    final int priorityFirst;
    final int prioritySecond;
    if (randomBoolean()) {
        highPriorityName = "first";
        lowPriorityName = "second";
        prioritySecond = 1;
        priorityFirst = 100;
    } else {
        lowPriorityName = "first";
        highPriorityName = "second";
        prioritySecond = 100;
        priorityFirst = 1;
    }
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("first").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_PRIORITY, priorityFirst)).numberOfShards(2).numberOfReplicas(1)).put(IndexMetadata.builder("second").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_PRIORITY, prioritySecond)).numberOfShards(2).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("first")).addAsNew(metadata.index("second")).build();
    ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = allocation.reroute(clusterState, "reroute");
    clusterState = allocation.reroute(clusterState, "reroute");
    assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
    assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName());
    assertEquals(highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName());
    clusterState = startInitializingShardsAndReroute(allocation, clusterState);
    assertEquals(4, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
    List<String> indices = clusterState.getRoutingNodes().shardsWithState(INITIALIZING).stream().map(x -> x.getIndexName()).collect(Collectors.toList());
    assertTrue(indices.contains(lowPriorityName));
    assertTrue(indices.contains(highPriorityName));
    clusterState = startInitializingShardsAndReroute(allocation, clusterState);
    assertEquals(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).toString(), 2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size());
    assertEquals(lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName());
    assertEquals(lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName());
}