Search in sources :

Example 1 with AllocationCommand

use of org.opensearch.cluster.routing.allocation.command.AllocationCommand in project OpenSearch by opensearch-project.

the class RerouteExplanation method readFrom.

public static RerouteExplanation readFrom(StreamInput in) throws IOException {
    AllocationCommand command = in.readNamedWriteable(AllocationCommand.class);
    Decision decisions = Decision.readFrom(in);
    return new RerouteExplanation(command, decisions);
}
Also used : AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) Decision(org.opensearch.cluster.routing.allocation.decider.Decision)

Example 2 with AllocationCommand

use of org.opensearch.cluster.routing.allocation.command.AllocationCommand in project OpenSearch by opensearch-project.

the class DiskThresholdDeciderTests method testShardRelocationsTakenIntoAccount.

public void testShardRelocationsTakenIntoAccount() {
    Settings diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8).build();
    ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
    // 60% used
    usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 40));
    // 60% used
    usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 40));
    // 60% used
    usagesBuilder.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 40));
    ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
    ImmutableOpenMap.Builder<String, Long> shardSizesBuilder = ImmutableOpenMap.builder();
    // 14 bytes
    shardSizesBuilder.put("[test][0][p]", 14L);
    shardSizesBuilder.put("[test][0][r]", 14L);
    // 1 bytes
    shardSizesBuilder.put("[test2][0][p]", 1L);
    shardSizesBuilder.put("[test2][0][r]", 1L);
    ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
    final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
    DiskThresholdDecider decider = makeDecider(diskSettings);
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), new EnableAllocationDecider(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none").build(), clusterSettings), decider)));
    final AtomicReference<ClusterInfo> clusterInfoReference = new AtomicReference<>(clusterInfo);
    final ClusterInfoService cis = clusterInfoReference::get;
    AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis, EmptySnapshotsInfoService.INSTANCE);
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).addAsNew(metadata.index("test2")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding two nodes");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // shards should be initializing
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4));
    logger.info("--> start the shards");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logShardStates(clusterState);
    // Assert that we're able to start the primary and replicas
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
    logger.info("--> adding node3");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    {
        AllocationCommand moveAllocationCommand = new MoveAllocationCommand("test", 0, "node2", "node3");
        AllocationCommands cmds = new AllocationCommands(moveAllocationCommand);
        clusterState = strategy.reroute(clusterState, cmds, false, false).getClusterState();
        logShardStates(clusterState);
    }
    final ImmutableOpenMap.Builder<String, DiskUsage> overfullUsagesBuilder = ImmutableOpenMap.builder();
    // 60% used
    overfullUsagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 40));
    // 60% used
    overfullUsagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 40));
    // 100% used
    overfullUsagesBuilder.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 0));
    final ImmutableOpenMap<String, DiskUsage> overfullUsages = overfullUsagesBuilder.build();
    final ImmutableOpenMap.Builder<String, Long> largerShardSizesBuilder = ImmutableOpenMap.builder();
    largerShardSizesBuilder.put("[test][0][p]", 14L);
    largerShardSizesBuilder.put("[test][0][r]", 14L);
    largerShardSizesBuilder.put("[test2][0][p]", 2L);
    largerShardSizesBuilder.put("[test2][0][r]", 2L);
    final ImmutableOpenMap<String, Long> largerShardSizes = largerShardSizesBuilder.build();
    final ClusterInfo overfullClusterInfo = new DevNullClusterInfo(overfullUsages, overfullUsages, largerShardSizes);
    {
        AllocationCommand moveAllocationCommand = new MoveAllocationCommand("test2", 0, "node2", "node3");
        AllocationCommands cmds = new AllocationCommands(moveAllocationCommand);
        final ClusterState clusterStateThatRejectsCommands = clusterState;
        assertThat(expectThrows(IllegalArgumentException.class, () -> strategy.reroute(clusterStateThatRejectsCommands, cmds, false, false)).getMessage(), containsString("the node is above the low watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.low=0.7], using more disk space than the maximum " + "allowed [70.0%], actual free: [26.0%]"));
        clusterInfoReference.set(overfullClusterInfo);
        assertThat(expectThrows(IllegalArgumentException.class, () -> strategy.reroute(clusterStateThatRejectsCommands, cmds, false, false)).getMessage(), containsString("the node has fewer free bytes remaining than the total size of all incoming shards"));
        clusterInfoReference.set(clusterInfo);
    }
    {
        AllocationCommand moveAllocationCommand = new MoveAllocationCommand("test2", 0, "node2", "node3");
        AllocationCommands cmds = new AllocationCommands(moveAllocationCommand);
        clusterState = startInitializingShardsAndReroute(strategy, clusterState);
        clusterState = strategy.reroute(clusterState, cmds, false, false).getClusterState();
        logShardStates(clusterState);
        clusterInfoReference.set(overfullClusterInfo);
        // ensure reroute doesn't fail even though there is negative free space
        strategy.reroute(clusterState, "foo");
    }
    {
        clusterInfoReference.set(overfullClusterInfo);
        clusterState = applyStartedShardsUntilNoChange(clusterState, strategy);
        final List<ShardRouting> startedShardsWithOverfullDisk = clusterState.getRoutingNodes().shardsWithState(STARTED);
        assertThat(startedShardsWithOverfullDisk.size(), equalTo(4));
        for (ShardRouting shardRouting : startedShardsWithOverfullDisk) {
            // no shards on node3 since it has no free space
            assertThat(shardRouting.toString(), shardRouting.currentNodeId(), oneOf("node1", "node2"));
        }
        // reset free space on node 3 and reserve space on node1
        clusterInfoReference.set(new DevNullClusterInfo(usages, usages, shardSizes, (new ImmutableOpenMap.Builder<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace>()).fPut(new ClusterInfo.NodeAndPath("node1", "/dev/null"), new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), between(51, 200)).build()).build()));
        clusterState = applyStartedShardsUntilNoChange(clusterState, strategy);
        final List<ShardRouting> startedShardsWithReservedSpace = clusterState.getRoutingNodes().shardsWithState(STARTED);
        assertThat(startedShardsWithReservedSpace.size(), equalTo(4));
        for (ShardRouting shardRouting : startedShardsWithReservedSpace) {
            // no shards on node1 since all its free space is reserved
            assertThat(shardRouting.toString(), shardRouting.currentNodeId(), oneOf("node2", "node3"));
        }
    }
}
Also used : ClusterSettings(org.opensearch.common.settings.ClusterSettings) AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) MoveAllocationCommand(org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand) Metadata(org.opensearch.cluster.metadata.Metadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) Matchers.containsString(org.hamcrest.Matchers.containsString) DiskUsage(org.opensearch.cluster.DiskUsage) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) ShardId(org.opensearch.index.shard.ShardId) Collections.singletonList(java.util.Collections.singletonList) List(java.util.List) Settings(org.opensearch.common.settings.Settings) DiskThresholdSettings(org.opensearch.cluster.routing.allocation.DiskThresholdSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) TestGatewayAllocator(org.opensearch.test.gateway.TestGatewayAllocator) ClusterState(org.opensearch.cluster.ClusterState) ClusterInfoService(org.opensearch.cluster.ClusterInfoService) BalancedShardsAllocator(org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator) MoveAllocationCommand(org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand) AtomicReference(java.util.concurrent.atomic.AtomicReference) AllocationCommands(org.opensearch.cluster.routing.allocation.command.AllocationCommands) ClusterInfo(org.opensearch.cluster.ClusterInfo) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) RoutingTable(org.opensearch.cluster.routing.RoutingTable) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting)

Example 3 with AllocationCommand

use of org.opensearch.cluster.routing.allocation.command.AllocationCommand in project OpenSearch by opensearch-project.

the class ClusterRerouteIT method testMessageLogging.

public void testMessageLogging() throws Exception {
    final Settings settings = Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.NONE.name()).put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE.name()).build();
    final String nodeName1 = internalCluster().startNode(settings);
    assertThat(cluster().size(), equalTo(1));
    ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("1").execute().actionGet();
    assertThat(healthResponse.isTimedOut(), equalTo(false));
    final String nodeName2 = internalCluster().startNode(settings);
    assertThat(cluster().size(), equalTo(2));
    healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
    assertThat(healthResponse.isTimedOut(), equalTo(false));
    final String indexName = "test_index";
    client().admin().indices().prepareCreate(indexName).setWaitForActiveShards(ActiveShardCount.NONE).setSettings(Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 2).put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)).execute().actionGet();
    Logger actionLogger = LogManager.getLogger(TransportClusterRerouteAction.class);
    try (MockLogAppender dryRunMockLog = MockLogAppender.createForLoggers(actionLogger)) {
        dryRunMockLog.addExpectation(new MockLogAppender.UnseenEventExpectation("no completed message logged on dry run", TransportClusterRerouteAction.class.getName(), Level.INFO, "allocated an empty primary*"));
        AllocationCommand dryRunAllocation = new AllocateEmptyPrimaryAllocationCommand(indexName, 0, nodeName1, true);
        ClusterRerouteResponse dryRunResponse = client().admin().cluster().prepareReroute().setExplain(randomBoolean()).setDryRun(true).add(dryRunAllocation).execute().actionGet();
        // during a dry run, messages exist but are not logged or exposed
        assertThat(dryRunResponse.getExplanations().getYesDecisionMessages(), hasSize(1));
        assertThat(dryRunResponse.getExplanations().getYesDecisionMessages().get(0), containsString("allocated an empty primary"));
        dryRunMockLog.assertAllExpectationsMatched();
    }
    try (MockLogAppender allocateMockLog = MockLogAppender.createForLoggers(actionLogger)) {
        allocateMockLog.addExpectation(new MockLogAppender.SeenEventExpectation("message for first allocate empty primary", TransportClusterRerouteAction.class.getName(), Level.INFO, "allocated an empty primary*" + nodeName1 + "*"));
        allocateMockLog.addExpectation(new MockLogAppender.UnseenEventExpectation("no message for second allocate empty primary", TransportClusterRerouteAction.class.getName(), Level.INFO, "allocated an empty primary*" + nodeName2 + "*"));
        AllocationCommand yesDecisionAllocation = new AllocateEmptyPrimaryAllocationCommand(indexName, 0, nodeName1, true);
        AllocationCommand noDecisionAllocation = new AllocateEmptyPrimaryAllocationCommand("noexist", 1, nodeName2, true);
        ClusterRerouteResponse response = client().admin().cluster().prepareReroute().setExplain(// so we get a NO decision back rather than an exception
        true).add(yesDecisionAllocation).add(noDecisionAllocation).execute().actionGet();
        assertThat(response.getExplanations().getYesDecisionMessages(), hasSize(1));
        assertThat(response.getExplanations().getYesDecisionMessages().get(0), containsString("allocated an empty primary"));
        assertThat(response.getExplanations().getYesDecisionMessages().get(0), containsString(nodeName1));
        allocateMockLog.assertAllExpectationsMatched();
    }
}
Also used : MockLogAppender(org.opensearch.test.MockLogAppender) ClusterHealthResponse(org.opensearch.action.admin.cluster.health.ClusterHealthResponse) AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) MoveAllocationCommand(org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand) AllocateEmptyPrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand) AllocateEmptyPrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand) Matchers.containsString(org.hamcrest.Matchers.containsString) Logger(org.apache.logging.log4j.Logger) Settings(org.opensearch.common.settings.Settings) ClusterRerouteResponse(org.opensearch.action.admin.cluster.reroute.ClusterRerouteResponse)

Example 4 with AllocationCommand

use of org.opensearch.cluster.routing.allocation.command.AllocationCommand in project OpenSearch by opensearch-project.

the class TransportClusterRerouteAction method masterOperation.

@Override
protected void masterOperation(final ClusterRerouteRequest request, final ClusterState state, final ActionListener<ClusterRerouteResponse> listener) {
    Map<String, List<AbstractAllocateAllocationCommand>> stalePrimaryAllocations = new HashMap<>();
    for (AllocationCommand command : request.getCommands().commands()) {
        if (command instanceof AllocateStalePrimaryAllocationCommand) {
            final AllocateStalePrimaryAllocationCommand cmd = (AllocateStalePrimaryAllocationCommand) command;
            stalePrimaryAllocations.computeIfAbsent(cmd.index(), k -> new ArrayList<>()).add(cmd);
        }
    }
    if (stalePrimaryAllocations.isEmpty()) {
        submitStateUpdate(request, listener);
    } else {
        verifyThenSubmitUpdate(request, listener, stalePrimaryAllocations);
    }
}
Also used : ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) ThreadPool(org.opensearch.threadpool.ThreadPool) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) Priority(org.opensearch.common.Priority) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) IndicesShardStoresAction(org.opensearch.action.admin.indices.shards.IndicesShardStoresAction) AllocateStalePrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand) Strings(org.opensearch.common.Strings) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Map(java.util.Map) AckedClusterStateUpdateTask(org.opensearch.cluster.AckedClusterStateUpdateTask) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) StreamInput(org.opensearch.common.io.stream.StreamInput) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) RoutingExplanations(org.opensearch.cluster.routing.allocation.RoutingExplanations) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) ExceptionsHelper(org.opensearch.ExceptionsHelper) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) AbstractAllocateAllocationCommand(org.opensearch.cluster.routing.allocation.command.AbstractAllocateAllocationCommand) TransportService(org.opensearch.transport.TransportService) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) ClusterService(org.opensearch.cluster.service.ClusterService) LogManager(org.apache.logging.log4j.LogManager) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) HashMap(java.util.HashMap) AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) AllocateStalePrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand) AbstractAllocateAllocationCommand(org.opensearch.cluster.routing.allocation.command.AbstractAllocateAllocationCommand) AllocateStalePrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List)

Aggregations

AllocationCommand (org.opensearch.cluster.routing.allocation.command.AllocationCommand)4 List (java.util.List)2 Logger (org.apache.logging.log4j.Logger)2 Matchers.containsString (org.hamcrest.Matchers.containsString)2 ClusterState (org.opensearch.cluster.ClusterState)2 AllocationService (org.opensearch.cluster.routing.allocation.AllocationService)2 MoveAllocationCommand (org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand)2 ImmutableOpenMap (org.opensearch.common.collect.ImmutableOpenMap)2 Settings (org.opensearch.common.settings.Settings)2 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 Collections.singletonList (java.util.Collections.singletonList)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 LogManager (org.apache.logging.log4j.LogManager)1 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)1 ExceptionsHelper (org.opensearch.ExceptionsHelper)1 ActionListener (org.opensearch.action.ActionListener)1 ActionListenerResponseHandler (org.opensearch.action.ActionListenerResponseHandler)1