Search in sources :

Example 1 with AllocationDecider

use of org.opensearch.cluster.routing.allocation.decider.AllocationDecider in project OpenSearch by opensearch-project.

the class ClusterModule method createAllocationDeciders.

// TODO: this is public so allocation benchmark can access the default deciders...can we do that in another way?
/**
 * Return a new {@link AllocationDecider} instance with builtin deciders as well as those from plugins.
 */
public static Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings, List<ClusterPlugin> clusterPlugins) {
    // collect deciders by class so that we can detect duplicates
    Map<Class, AllocationDecider> deciders = new LinkedHashMap<>();
    addAllocationDecider(deciders, new MaxRetryAllocationDecider());
    addAllocationDecider(deciders, new ResizeAllocationDecider());
    addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider());
    addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider());
    addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new ConcurrentRebalanceAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new ConcurrentRecoveriesAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new EnableAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new NodeVersionAllocationDecider());
    addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider());
    addAllocationDecider(deciders, new RestoreInProgressAllocationDecider());
    addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new SameShardAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new ThrottlingAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new ShardsLimitAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new AwarenessAllocationDecider(settings, clusterSettings));
    addAllocationDecider(deciders, new NodeLoadAwareAllocationDecider(settings, clusterSettings));
    clusterPlugins.stream().flatMap(p -> p.createAllocationDeciders(settings, clusterSettings).stream()).forEach(d -> addAllocationDecider(deciders, d));
    return deciders.values();
}
Also used : IndexGraveyard(org.opensearch.cluster.metadata.IndexGraveyard) NamedWriteable(org.opensearch.common.io.stream.NamedWriteable) Metadata(org.opensearch.cluster.metadata.Metadata) EnableAllocationDecider(org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider) MetadataDeleteIndexService(org.opensearch.cluster.metadata.MetadataDeleteIndexService) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) ClusterRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider) SnapshotInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider) ThrottlingAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider) Entry(org.opensearch.common.io.stream.NamedWriteableRegistry.Entry) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) ComponentTemplateMetadata(org.opensearch.cluster.metadata.ComponentTemplateMetadata) DelayedAllocationService(org.opensearch.cluster.routing.DelayedAllocationService) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider) ComposableIndexTemplateMetadata(org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata) Property(org.opensearch.common.settings.Setting.Property) Map(java.util.Map) BalancedShardsAllocator(org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) ShardsAllocator(org.opensearch.cluster.routing.allocation.allocator.ShardsAllocator) TaskResultsService(org.opensearch.tasks.TaskResultsService) AllocationDeciders(org.opensearch.cluster.routing.allocation.decider.AllocationDeciders) DiskThresholdDecider(org.opensearch.cluster.routing.allocation.decider.DiskThresholdDecider) ResizeAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ResizeAllocationDecider) SameShardAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider) ClusterPlugin(org.opensearch.plugins.ClusterPlugin) AwarenessAllocationDecider(org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider) Collection(java.util.Collection) PersistentTasksCustomMetadata(org.opensearch.persistent.PersistentTasksCustomMetadata) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) Task(org.opensearch.tasks.Task) PersistentTasksNodeService(org.opensearch.persistent.PersistentTasksNodeService) Objects(java.util.Objects) List(java.util.List) DataStreamMetadata(org.opensearch.cluster.metadata.DataStreamMetadata) ExistingShardsAllocator(org.opensearch.cluster.routing.allocation.ExistingShardsAllocator) GatewayAllocator(org.opensearch.gateway.GatewayAllocator) Reader(org.opensearch.common.io.stream.Writeable.Reader) ScriptMetadata(org.opensearch.script.ScriptMetadata) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) SnapshotsInfoService(org.opensearch.snapshots.SnapshotsInfoService) RestoreInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider) ConcurrentRecoveriesAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRecoveriesAllocationDecider) MetadataIndexAliasesService(org.opensearch.cluster.metadata.MetadataIndexAliasesService) MetadataUpdateSettingsService(org.opensearch.cluster.metadata.MetadataUpdateSettingsService) HashMap(java.util.HashMap) ParseField(org.opensearch.common.ParseField) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) MappingUpdatedAction(org.opensearch.cluster.action.index.MappingUpdatedAction) NodeVersionAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider) LinkedHashMap(java.util.LinkedHashMap) ShardsLimitAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider) MetadataMappingService(org.opensearch.cluster.metadata.MetadataMappingService) AbstractModule(org.opensearch.common.inject.AbstractModule) NodeMappingRefreshAction(org.opensearch.cluster.action.index.NodeMappingRefreshAction) ClusterSettings(org.opensearch.common.settings.ClusterSettings) ReplicaAfterPrimaryActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider) MetadataIndexStateService(org.opensearch.cluster.metadata.MetadataIndexStateService) RebalanceOnlyWhenActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider) IngestMetadata(org.opensearch.ingest.IngestMetadata) FilterAllocationDecider(org.opensearch.cluster.routing.allocation.decider.FilterAllocationDecider) MetadataIndexTemplateService(org.opensearch.cluster.metadata.MetadataIndexTemplateService) Setting(org.opensearch.common.settings.Setting) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) MaxRetryAllocationDecider(org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider) Sets(org.opensearch.common.util.set.Sets) ConcurrentRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider) NodeLoadAwareAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) Collections(java.util.Collections) ResizeAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ResizeAllocationDecider) SnapshotInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider) EnableAllocationDecider(org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider) NodeLoadAwareAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider) RebalanceOnlyWhenActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider) ThrottlingAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider) MaxRetryAllocationDecider(org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider) ReplicaAfterPrimaryActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider) ConcurrentRecoveriesAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRecoveriesAllocationDecider) LinkedHashMap(java.util.LinkedHashMap) ClusterRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider) SameShardAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider) RestoreInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider) FilterAllocationDecider(org.opensearch.cluster.routing.allocation.decider.FilterAllocationDecider) DiskThresholdDecider(org.opensearch.cluster.routing.allocation.decider.DiskThresholdDecider) ShardsLimitAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider) AwarenessAllocationDecider(org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider) NodeVersionAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider) EnableAllocationDecider(org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider) ClusterRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider) SnapshotInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider) ThrottlingAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider) ResizeAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ResizeAllocationDecider) SameShardAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider) AwarenessAllocationDecider(org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider) RestoreInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider) ConcurrentRecoveriesAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRecoveriesAllocationDecider) NodeVersionAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider) ShardsLimitAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider) ReplicaAfterPrimaryActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider) RebalanceOnlyWhenActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider) FilterAllocationDecider(org.opensearch.cluster.routing.allocation.decider.FilterAllocationDecider) MaxRetryAllocationDecider(org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider) ConcurrentRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider) NodeLoadAwareAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider) ConcurrentRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider)

Example 2 with AllocationDecider

use of org.opensearch.cluster.routing.allocation.decider.AllocationDecider in project OpenSearch by opensearch-project.

the class DecisionsImpactOnClusterHealthTests method testPrimaryShardThrottleDecisionOnIndexCreation.

public void testPrimaryShardThrottleDecisionOnIndexCreation() throws IOException {
    final String indexName = "test-idx";
    Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()).build();
    AllocationDecider decider = new TestAllocateDecision(Decision.THROTTLE);
    // if deciders THROTTLE allocating a primary shard, stay in YELLOW state
    runAllocationTest(settings, indexName, Collections.singleton(decider), ClusterHealthStatus.YELLOW);
}
Also used : Settings(org.opensearch.common.settings.Settings) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider)

Example 3 with AllocationDecider

use of org.opensearch.cluster.routing.allocation.decider.AllocationDecider in project OpenSearch by opensearch-project.

the class DecisionsImpactOnClusterHealthTests method testPrimaryShardYesDecisionOnIndexCreation.

public void testPrimaryShardYesDecisionOnIndexCreation() throws IOException {
    final String indexName = "test-idx";
    Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()).build();
    AllocationDecider decider = new TestAllocateDecision(Decision.YES) {

        @Override
        public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
            if (node.getByShardId(shardRouting.shardId()) == null) {
                return Decision.YES;
            } else {
                return Decision.NO;
            }
        }
    };
    // if deciders say YES to allocating primary shards, stay in YELLOW state
    ClusterState clusterState = runAllocationTest(settings, indexName, Collections.singleton(decider), ClusterHealthStatus.YELLOW);
    // make sure primaries are initialized
    RoutingTable routingTable = clusterState.routingTable();
    for (IndexShardRoutingTable indexShardRoutingTable : routingTable.index(indexName)) {
        assertTrue(indexShardRoutingTable.primaryShard().initializing());
    }
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) RoutingNode(org.opensearch.cluster.routing.RoutingNode) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) RoutingTable(org.opensearch.cluster.routing.RoutingTable) ShardRouting(org.opensearch.cluster.routing.ShardRouting) Settings(org.opensearch.common.settings.Settings) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider)

Example 4 with AllocationDecider

use of org.opensearch.cluster.routing.allocation.decider.AllocationDecider in project OpenSearch by opensearch-project.

the class BalancedSingleShardTests method testDontBalanceShardWhenThresholdNotMet.

public void testDontBalanceShardWhenThresholdNotMet() {
    AllocationDecider canAllocateDecider = new AllocationDecider() {

        @Override
        public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
            return Decision.YES;
        }
    };
    // ridiculously high threshold setting so we won't rebalance
    Settings balancerSettings = Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), 1000f).build();
    Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, balancerSettings, false);
    ClusterState clusterState = rebalance.v1();
    MoveDecision rebalanceDecision = rebalance.v2();
    assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
    assertEquals(AllocationDecision.NO, rebalanceDecision.getAllocationDecision());
    assertNotNull(rebalanceDecision.getExplanation());
    assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
    assertNull(rebalanceDecision.getTargetNode());
    int prevRanking = 0;
    for (NodeAllocationResult result : rebalanceDecision.getNodeDecisions()) {
        assertThat(result.getWeightRanking(), greaterThanOrEqualTo(prevRanking));
        prevRanking = result.getWeightRanking();
    }
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) RoutingNode(org.opensearch.cluster.routing.RoutingNode) ShardRouting(org.opensearch.cluster.routing.ShardRouting) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider) Settings(org.opensearch.common.settings.Settings)

Example 5 with AllocationDecider

use of org.opensearch.cluster.routing.allocation.decider.AllocationDecider in project OpenSearch by opensearch-project.

the class ClusterModuleTests method testAllocationDeciderOrder.

// makes sure that the allocation deciders are setup in the correct order, such that the
// slower allocation deciders come last and we can exit early if there is a NO decision without
// running them. If the order of the deciders is changed for a valid reason, the order should be
// changed in the test too.
public void testAllocationDeciderOrder() {
    List<Class<? extends AllocationDecider>> expectedDeciders = Arrays.asList(MaxRetryAllocationDecider.class, ResizeAllocationDecider.class, ReplicaAfterPrimaryActiveAllocationDecider.class, RebalanceOnlyWhenActiveAllocationDecider.class, ClusterRebalanceAllocationDecider.class, ConcurrentRebalanceAllocationDecider.class, ConcurrentRecoveriesAllocationDecider.class, EnableAllocationDecider.class, NodeVersionAllocationDecider.class, SnapshotInProgressAllocationDecider.class, RestoreInProgressAllocationDecider.class, FilterAllocationDecider.class, SameShardAllocationDecider.class, DiskThresholdDecider.class, ThrottlingAllocationDecider.class, ShardsLimitAllocationDecider.class, AwarenessAllocationDecider.class, NodeLoadAwareAllocationDecider.class);
    Collection<AllocationDecider> deciders = ClusterModule.createAllocationDeciders(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), Collections.emptyList());
    Iterator<AllocationDecider> iter = deciders.iterator();
    int idx = 0;
    while (iter.hasNext()) {
        AllocationDecider decider = iter.next();
        assertSame(decider.getClass(), expectedDeciders.get(idx++));
    }
}
Also used : ClusterSettings(org.opensearch.common.settings.ClusterSettings) RestoreInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider) ConcurrentRecoveriesAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRecoveriesAllocationDecider) EnableAllocationDecider(org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider) ClusterRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider) SnapshotInProgressAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider) ThrottlingAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider) NodeVersionAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider) ShardsLimitAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider) ReplicaAfterPrimaryActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider) RebalanceOnlyWhenActiveAllocationDecider(org.opensearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider) FilterAllocationDecider(org.opensearch.cluster.routing.allocation.decider.FilterAllocationDecider) ResizeAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ResizeAllocationDecider) SameShardAllocationDecider(org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider) MaxRetryAllocationDecider(org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider) AwarenessAllocationDecider(org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider) ConcurrentRebalanceAllocationDecider(org.opensearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider) NodeLoadAwareAllocationDecider(org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider)

Aggregations

AllocationDecider (org.opensearch.cluster.routing.allocation.decider.AllocationDecider)13 ShardRouting (org.opensearch.cluster.routing.ShardRouting)9 ClusterState (org.opensearch.cluster.ClusterState)7 RoutingNode (org.opensearch.cluster.routing.RoutingNode)7 AllocationDeciders (org.opensearch.cluster.routing.allocation.decider.AllocationDeciders)6 Settings (org.opensearch.common.settings.Settings)6 BalancedShardsAllocator (org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator)5 DiscoveryNodes (org.opensearch.cluster.node.DiscoveryNodes)2 Decision (org.opensearch.cluster.routing.allocation.decider.Decision)2 SameShardAllocationDecider (org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider)2 ClusterSettings (org.opensearch.common.settings.ClusterSettings)2 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 Map (java.util.Map)1 Objects (java.util.Objects)1