Search in sources :

Example 1 with IndicesShardStoresResponse

use of org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse in project OpenSearch by opensearch-project.

the class TransportClusterRerouteAction method verifyThenSubmitUpdate.

private void verifyThenSubmitUpdate(ClusterRerouteRequest request, ActionListener<ClusterRerouteResponse> listener, Map<String, List<AbstractAllocateAllocationCommand>> stalePrimaryAllocations) {
    transportService.sendRequest(transportService.getLocalNode(), IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(stalePrimaryAllocations.keySet().toArray(Strings.EMPTY_ARRAY)), new ActionListenerResponseHandler<>(ActionListener.wrap(response -> {
        ImmutableOpenMap<String, ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>>> status = response.getStoreStatuses();
        Exception e = null;
        for (Map.Entry<String, List<AbstractAllocateAllocationCommand>> entry : stalePrimaryAllocations.entrySet()) {
            final String index = entry.getKey();
            final ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> indexStatus = status.get(index);
            if (indexStatus == null) {
                // request. We ignore it here since the relevant exception will be thrown by the reroute action later on.
                continue;
            }
            for (AbstractAllocateAllocationCommand command : entry.getValue()) {
                final List<IndicesShardStoresResponse.StoreStatus> shardStatus = indexStatus.get(command.shardId());
                if (shardStatus == null || shardStatus.isEmpty()) {
                    e = ExceptionsHelper.useOrSuppress(e, new IllegalArgumentException("No data for shard [" + command.shardId() + "] of index [" + index + "] found on any node"));
                } else if (shardStatus.stream().noneMatch(storeStatus -> {
                    final DiscoveryNode node = storeStatus.getNode();
                    final String nodeInCommand = command.node();
                    return nodeInCommand.equals(node.getName()) || nodeInCommand.equals(node.getId());
                })) {
                    e = ExceptionsHelper.useOrSuppress(e, new IllegalArgumentException("No data for shard [" + command.shardId() + "] of index [" + index + "] found on node [" + command.node() + ']'));
                }
            }
        }
        if (e == null) {
            submitStateUpdate(request, listener);
        } else {
            listener.onFailure(e);
        }
    }, listener::onFailure), IndicesShardStoresResponse::new));
}
Also used : ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) ThreadPool(org.opensearch.threadpool.ThreadPool) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) Priority(org.opensearch.common.Priority) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) IndicesShardStoresAction(org.opensearch.action.admin.indices.shards.IndicesShardStoresAction) AllocateStalePrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand) Strings(org.opensearch.common.Strings) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Map(java.util.Map) AckedClusterStateUpdateTask(org.opensearch.cluster.AckedClusterStateUpdateTask) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) StreamInput(org.opensearch.common.io.stream.StreamInput) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) RoutingExplanations(org.opensearch.cluster.routing.allocation.RoutingExplanations) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) ExceptionsHelper(org.opensearch.ExceptionsHelper) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) AbstractAllocateAllocationCommand(org.opensearch.cluster.routing.allocation.command.AbstractAllocateAllocationCommand) TransportService(org.opensearch.transport.TransportService) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) ClusterService(org.opensearch.cluster.service.ClusterService) LogManager(org.apache.logging.log4j.LogManager) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) AbstractAllocateAllocationCommand(org.opensearch.cluster.routing.allocation.command.AbstractAllocateAllocationCommand) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap)

Example 2 with IndicesShardStoresResponse

use of org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse in project OpenSearch by opensearch-project.

the class RestIndicesShardStoresAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    IndicesShardStoresRequest indicesShardStoresRequest = new IndicesShardStoresRequest(Strings.splitStringByCommaToArray(request.param("index")));
    if (request.hasParam("status")) {
        indicesShardStoresRequest.shardStatuses(Strings.splitStringByCommaToArray(request.param("status")));
    }
    indicesShardStoresRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesShardStoresRequest.indicesOptions()));
    return channel -> client.admin().indices().shardStores(indicesShardStoresRequest, new RestBuilderListener<IndicesShardStoresResponse>(channel) {

        @Override
        public RestResponse buildResponse(IndicesShardStoresResponse response, XContentBuilder builder) throws Exception {
            builder.startObject();
            response.toXContent(builder, request);
            builder.endObject();
            return new BytesRestResponse(OK, builder);
        }
    });
}
Also used : IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) OK(org.opensearch.rest.RestStatus.OK) NodeClient(org.opensearch.client.node.NodeClient) Collections.unmodifiableList(java.util.Collections.unmodifiableList) GET(org.opensearch.rest.RestRequest.Method.GET) RestRequest(org.opensearch.rest.RestRequest) IOException(java.io.IOException) IndicesOptions(org.opensearch.action.support.IndicesOptions) IndicesShardStoresAction(org.opensearch.action.admin.indices.shards.IndicesShardStoresAction) BytesRestResponse(org.opensearch.rest.BytesRestResponse) RestResponse(org.opensearch.rest.RestResponse) Strings(org.opensearch.common.Strings) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) RestBuilderListener(org.opensearch.rest.action.RestBuilderListener) List(java.util.List) Arrays.asList(java.util.Arrays.asList) BaseRestHandler(org.opensearch.rest.BaseRestHandler) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) BytesRestResponse(org.opensearch.rest.BytesRestResponse) RestResponse(org.opensearch.rest.RestResponse) BytesRestResponse(org.opensearch.rest.BytesRestResponse) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) IOException(java.io.IOException)

Example 3 with IndicesShardStoresResponse

use of org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse in project OpenSearch by opensearch-project.

the class CorruptedFileIT method testReplicaCorruption.

/**
 * This test verifies that if we corrupt a replica, we can still get to green, even though
 * listing its store fails. Note, we need to make sure that replicas are allocated on all data
 * nodes, so that replica won't be sneaky and allocated on a node that doesn't have a corrupted
 * replica.
 */
public void testReplicaCorruption() throws Exception {
    int numDocs = scaledRandomIntBetween(100, 1000);
    internalCluster().ensureAtLeastNumDataNodes(2);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1).put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), // no checkindex - we corrupt shards on
    false).put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), // no
    new ByteSizeValue(1, ByteSizeUnit.PB))));
    ensureGreen();
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);
    // disable allocations of replicas post restart (the restart will change replicas to primaries, so we have
    // to capture replicas post restart)
    assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries")));
    internalCluster().fullRestart();
    ensureYellow();
    final Index index = resolveIndex("test");
    final IndicesShardStoresResponse stores = client().admin().indices().prepareShardStores(index.getName()).get();
    for (IntObjectCursor<List<IndicesShardStoresResponse.StoreStatus>> shards : stores.getStoreStatuses().get(index.getName())) {
        for (IndicesShardStoresResponse.StoreStatus store : shards.value) {
            final ShardId shardId = new ShardId(index, shards.key);
            if (store.getAllocationStatus().equals(IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED)) {
                for (Path path : findFilesToCorruptOnNode(store.getNode().getName(), shardId)) {
                    try (OutputStream os = Files.newOutputStream(path)) {
                        os.write(0);
                    }
                    logger.info("corrupting file {} on node {}", path, store.getNode().getName());
                }
            }
        }
    }
    // enable allocation
    assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey())));
    ensureGreen(TimeValue.timeValueSeconds(60));
}
Also used : Path(java.nio.file.Path) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) OutputStream(java.io.OutputStream) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) CheckIndex(org.apache.lucene.index.CheckIndex) Index(org.opensearch.index.Index) SearchResponse(org.opensearch.action.search.SearchResponse) IndexRequestBuilder(org.opensearch.action.index.IndexRequestBuilder) ShardId(org.opensearch.index.shard.ShardId) CollectionUtils.iterableAsArrayList(org.opensearch.common.util.CollectionUtils.iterableAsArrayList) List(java.util.List) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList)

Aggregations

List (java.util.List)3 IndicesShardStoresResponse (org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 IndicesShardStoresAction (org.opensearch.action.admin.indices.shards.IndicesShardStoresAction)2 IndicesShardStoresRequest (org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest)2 Strings (org.opensearch.common.Strings)2 OutputStream (java.io.OutputStream)1 Path (java.nio.file.Path)1 Arrays.asList (java.util.Arrays.asList)1 Collections.unmodifiableList (java.util.Collections.unmodifiableList)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)1 LogManager (org.apache.logging.log4j.LogManager)1 Logger (org.apache.logging.log4j.Logger)1 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)1 CheckIndex (org.apache.lucene.index.CheckIndex)1 ExceptionsHelper (org.opensearch.ExceptionsHelper)1 ActionListener (org.opensearch.action.ActionListener)1