Search in sources :

Example 1 with AddBlockResult

use of org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockResult in project OpenSearch by opensearch-project.

the class MetadataIndexStateService method finalizeBlock.

/**
 * Finalizes the addition of blocks by turning the temporary UUID-based blocks into full blocks.
 * @param currentState the cluster state to update
 * @param blockedIndices the indices and their temporary UUID-based blocks to convert
 * @param verifyResult the index-level results for adding the block
 * @param block the full block to convert to
 * @return the updated cluster state, as well as the (failed and successful) index-level results for adding the block
 */
static Tuple<ClusterState, Collection<AddBlockResult>> finalizeBlock(final ClusterState currentState, final Map<Index, ClusterBlock> blockedIndices, final Map<Index, AddBlockResult> verifyResult, final APIBlock block) {
    final Metadata.Builder metadata = Metadata.builder(currentState.metadata());
    final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
    final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
    final Set<String> effectivelyBlockedIndices = new HashSet<>();
    Map<Index, AddBlockResult> blockingResults = new HashMap<>(verifyResult);
    for (Map.Entry<Index, AddBlockResult> result : verifyResult.entrySet()) {
        final Index index = result.getKey();
        final boolean acknowledged = result.getValue().hasFailures() == false;
        try {
            if (acknowledged == false) {
                logger.debug("verification of shards before blocking {} failed [{}]", index, result);
                continue;
            }
            final IndexMetadata indexMetadata = metadata.getSafe(index);
            final ClusterBlock tempBlock = blockedIndices.get(index);
            assert tempBlock != null;
            assert tempBlock.uuid() != null;
            final ClusterBlock currentBlock = currentState.blocks().getIndexBlockWithId(index.getName(), tempBlock.id());
            if (currentBlock != null && currentBlock.equals(block.block)) {
                logger.debug("verification of shards for {} succeeded, but block finalization already occurred" + " (possibly for another block) [{}]", index, result);
                continue;
            }
            if (currentBlock == null || currentBlock.equals(tempBlock) == false) {
                // we should report error in this case as the index can be left as open.
                blockingResults.put(result.getKey(), new AddBlockResult(result.getKey(), new IllegalStateException("verification of shards before blocking " + index + " succeeded but block has been removed in the meantime")));
                logger.debug("verification of shards before blocking {} succeeded but block has been removed in the meantime", index);
                continue;
            }
            assert currentBlock != null && currentBlock.equals(tempBlock) && currentBlock.id() == block.block.id();
            blocks.removeIndexBlockWithId(index.getName(), tempBlock.id());
            blocks.addIndexBlock(index.getName(), block.block);
            logger.debug("add block {} to index {} succeeded", block.block, index);
            effectivelyBlockedIndices.add(index.getName());
        } catch (final IndexNotFoundException e) {
            logger.debug("index {} has been deleted since blocking it started, ignoring", index);
        }
    }
    logger.info("completed adding block {} to indices {}", block.name, effectivelyBlockedIndices);
    return Tuple.tuple(ClusterState.builder(currentState).blocks(blocks).metadata(metadata).routingTable(routingTable.build()).build(), blockingResults.values());
}
Also used : ClusterBlocks(org.opensearch.cluster.block.ClusterBlocks) HashMap(java.util.HashMap) Index(org.opensearch.index.Index) AddBlockResult(org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockResult) ClusterBlock(org.opensearch.cluster.block.ClusterBlock) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) RoutingTable(org.opensearch.cluster.routing.RoutingTable) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) Map(java.util.Map) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) SortedMap(java.util.SortedMap) HashMap(java.util.HashMap) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap) HashSet(java.util.HashSet)

Example 2 with AddBlockResult

use of org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockResult in project OpenSearch by opensearch-project.

the class MetadataIndexStateService method addIndexBlock.

/**
 * Adds an index block based on the given request, and notifies the listener upon completion.
 * Adding blocks is done in three steps:
 * - First, a temporary UUID-based block is added to the index
 *   (see {@link #addIndexBlock(Index[], ClusterState, APIBlock)}.
 * - Second, shards are checked to have properly applied the UUID-based block.
 *   (see {@link WaitForBlocksApplied}).
 * - Third, the temporary UUID-based block is turned into a full block
 *   (see {@link #finalizeBlock(ClusterState, Map, Map, APIBlock)}.
 * Using this three-step process ensures non-interference by other operations in case where
 * we notify successful completion here.
 */
public void addIndexBlock(AddIndexBlockClusterStateUpdateRequest request, ActionListener<AddIndexBlockResponse> listener) {
    final Index[] concreteIndices = request.indices();
    if (concreteIndices == null || concreteIndices.length == 0) {
        throw new IllegalArgumentException("Index name is required");
    }
    List<String> writeIndices = new ArrayList<>();
    SortedMap<String, IndexAbstraction> lookup = clusterService.state().metadata().getIndicesLookup();
    for (Index index : concreteIndices) {
        IndexAbstraction ia = lookup.get(index.getName());
        if (ia != null && ia.getParentDataStream() != null && ia.getParentDataStream().getWriteIndex().getIndex().equals(index)) {
            writeIndices.add(index.getName());
        }
    }
    if (writeIndices.size() > 0) {
        throw new IllegalArgumentException("cannot add a block to the following data stream write indices [" + Strings.collectionToCommaDelimitedString(writeIndices) + "]");
    }
    clusterService.submitStateUpdateTask("add-index-block-[" + request.getBlock().name + "]-" + Arrays.toString(concreteIndices), new ClusterStateUpdateTask(Priority.URGENT) {

        private Map<Index, ClusterBlock> blockedIndices;

        @Override
        public ClusterState execute(final ClusterState currentState) {
            final Tuple<ClusterState, Map<Index, ClusterBlock>> tup = addIndexBlock(concreteIndices, currentState, request.getBlock());
            blockedIndices = tup.v2();
            return tup.v1();
        }

        @Override
        public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
            if (oldState == newState) {
                assert blockedIndices.isEmpty() : "List of blocked indices is not empty but cluster state wasn't changed";
                listener.onResponse(new AddIndexBlockResponse(true, false, Collections.emptyList()));
            } else {
                assert blockedIndices.isEmpty() == false : "List of blocked indices is empty but cluster state was changed";
                threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new WaitForBlocksApplied(blockedIndices, request, ActionListener.wrap(verifyResults -> clusterService.submitStateUpdateTask("finalize-index-block-[" + request.getBlock().name + "]-[" + blockedIndices.keySet().stream().map(Index::getName).collect(Collectors.joining(", ")) + "]", new ClusterStateUpdateTask(Priority.URGENT) {

                    private final List<AddBlockResult> indices = new ArrayList<>();

                    @Override
                    public ClusterState execute(final ClusterState currentState) throws Exception {
                        Tuple<ClusterState, Collection<AddBlockResult>> addBlockResult = finalizeBlock(currentState, blockedIndices, verifyResults, request.getBlock());
                        assert verifyResults.size() == addBlockResult.v2().size();
                        indices.addAll(addBlockResult.v2());
                        return addBlockResult.v1();
                    }

                    @Override
                    public void onFailure(final String source, final Exception e) {
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
                        final boolean acknowledged = indices.stream().noneMatch(AddBlockResult::hasFailures);
                        listener.onResponse(new AddIndexBlockResponse(acknowledged, acknowledged, indices));
                    }
                }), listener::onFailure)));
            }
        }

        @Override
        public void onFailure(final String source, final Exception e) {
            listener.onFailure(e);
        }

        @Override
        public TimeValue timeout() {
            return request.masterNodeTimeout();
        }
    });
}
Also used : Arrays(java.util.Arrays) CountDown(org.opensearch.common.util.concurrent.CountDown) NotifyOnceListener(org.opensearch.action.NotifyOnceListener) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) Version(org.opensearch.Version) OpenSearchException(org.opensearch.OpenSearchException) SnapshotsService(org.opensearch.snapshots.SnapshotsService) Strings(org.opensearch.common.Strings) CloseIndexResponse(org.opensearch.action.admin.indices.close.CloseIndexResponse) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) ClusterBlock(org.opensearch.cluster.block.ClusterBlock) Collections.singleton(java.util.Collections.singleton) TransportVerifyShardIndexBlockAction(org.opensearch.action.admin.indices.readonly.TransportVerifyShardIndexBlockAction) Map(java.util.Map) Inject(org.opensearch.common.inject.Inject) AddBlockShardResult(org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockShardResult) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TimeValue(org.opensearch.common.unit.TimeValue) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) Index(org.opensearch.index.Index) Collection(java.util.Collection) IndicesService(org.opensearch.indices.IndicesService) SnapshotInProgressException(org.opensearch.snapshots.SnapshotInProgressException) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) RestStatus(org.opensearch.rest.RestStatus) ShardResult(org.opensearch.action.admin.indices.close.CloseIndexResponse.ShardResult) Collectors(java.util.stream.Collectors) Tuple(org.opensearch.common.collect.Tuple) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) IndexResult(org.opensearch.action.admin.indices.close.CloseIndexResponse.IndexResult) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) AddIndexBlockClusterStateUpdateRequest(org.opensearch.action.admin.indices.readonly.AddIndexBlockClusterStateUpdateRequest) ClusterStateUpdateResponse(org.opensearch.cluster.ack.ClusterStateUpdateResponse) SortedMap(java.util.SortedMap) IntObjectCursor(com.carrotsearch.hppc.cursors.IntObjectCursor) APIBlock(org.opensearch.cluster.metadata.IndexMetadata.APIBlock) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) HashMap(java.util.HashMap) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) TransportVerifyShardBeforeCloseAction(org.opensearch.action.admin.indices.close.TransportVerifyShardBeforeCloseAction) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) CloseIndexClusterStateUpdateRequest(org.opensearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest) ClusterState(org.opensearch.cluster.ClusterState) AddIndexBlockResponse(org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse) LegacyESVersion(org.opensearch.LegacyESVersion) RestoreService(org.opensearch.snapshots.RestoreService) AckedClusterStateUpdateTask(org.opensearch.cluster.AckedClusterStateUpdateTask) UUIDs(org.opensearch.common.UUIDs) ClusterBlocks(org.opensearch.cluster.block.ClusterBlocks) Setting(org.opensearch.common.settings.Setting) ShardLimitValidator(org.opensearch.indices.ShardLimitValidator) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) TaskId(org.opensearch.tasks.TaskId) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) AddBlockResult(org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockResult) ActiveShardsObserver(org.opensearch.action.support.ActiveShardsObserver) ClusterService(org.opensearch.cluster.service.ClusterService) RoutingTable(org.opensearch.cluster.routing.RoutingTable) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap) OpenIndexClusterStateUpdateResponse(org.opensearch.cluster.ack.OpenIndexClusterStateUpdateResponse) OpenIndexClusterStateUpdateRequest(org.opensearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) ClusterState(org.opensearch.cluster.ClusterState) ArrayList(java.util.ArrayList) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) AckedClusterStateUpdateTask(org.opensearch.cluster.AckedClusterStateUpdateTask) Index(org.opensearch.index.Index) AddBlockResult(org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockResult) OpenSearchException(org.opensearch.OpenSearchException) SnapshotInProgressException(org.opensearch.snapshots.SnapshotInProgressException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ClusterBlock(org.opensearch.cluster.block.ClusterBlock) AddIndexBlockResponse(org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse) Tuple(org.opensearch.common.collect.Tuple) TimeValue(org.opensearch.common.unit.TimeValue)

Aggregations

Collections.unmodifiableMap (java.util.Collections.unmodifiableMap)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 SortedMap (java.util.SortedMap)2 AddBlockResult (org.opensearch.action.admin.indices.readonly.AddIndexBlockResponse.AddBlockResult)2 IntObjectCursor (com.carrotsearch.hppc.cursors.IntObjectCursor)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 Collections.singleton (java.util.Collections.singleton)1 EnumSet (java.util.EnumSet)1 List (java.util.List)1 Set (java.util.Set)1 Consumer (java.util.function.Consumer)1 Collectors (java.util.stream.Collectors)1 LogManager (org.apache.logging.log4j.LogManager)1 Logger (org.apache.logging.log4j.Logger)1 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)1