Examples with StepListener org.elasticsearch.action.StepListener used on opensource projects

Example 11 with StepListener

use of org.elasticsearch.action.StepListener in project crate by crate.

the class SnapshotsService method createSnapshot.

/**
 * Initializes the snapshotting process.
 * <p>
 * This method is used by clients to start snapshot. It makes sure that there is no snapshots are currently running and
 * creates a snapshot record in cluster state metadata.
 *
 * @param request  snapshot request
 * @param listener snapshot creation listener
 */
public void createSnapshot(final CreateSnapshotRequest request, final ActionListener<Snapshot> listener) {
    final String repositoryName = request.repository();
    final String snapshotName = request.snapshot();
    validate(repositoryName, snapshotName);
    // new UUID for the snapshot
    final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID());
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repositoriesService.repository(repositoryName).getRepositoryData(repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        clusterService.submitStateUpdateTask("create_snapshot [" + snapshotName + ']', new ClusterStateUpdateTask() {

            private SnapshotsInProgress.Entry newSnapshot = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                validate(repositoryName, snapshotName, currentState);
                SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
                if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "cannot snapshot while a snapshot deletion is in-progress in [" + deletionsInProgress + "]");
                }
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                if (snapshots == null || snapshots.entries().isEmpty()) {
                    // Store newSnapshot here to be processed in clusterStateProcessed
                    List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request.indicesOptions(), request.indices()));
                    LOGGER.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices);
                    List<IndexId> snapshotIndices = repositoryData.resolveNewIndices(indices);
                    newSnapshot = new SnapshotsInProgress.Entry(new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), State.INIT, snapshotIndices, List.of(request.templates()), threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), null, clusterService.state().nodes().getMinNodeVersion().onOrAfter(SHARD_GEN_IN_REPO_DATA_VERSION));
                    initializingSnapshots.add(newSnapshot.snapshot());
                    snapshots = new SnapshotsInProgress(newSnapshot);
                } else {
                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, " a snapshot is already running");
                }
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                LOGGER.warn(() -> new ParameterizedMessage("[{}][{}] failed to create snapshot", repositoryName, snapshotName), e);
                if (newSnapshot != null) {
                    initializingSnapshots.remove(newSnapshot.snapshot());
                }
                newSnapshot = null;
                listener.onFailure(e);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
                if (newSnapshot != null) {
                    final Snapshot current = newSnapshot.snapshot();
                    assert initializingSnapshots.contains(current);
                    beginSnapshot(newState, newSnapshot, request.partial(), new ActionListener<>() {

                        @Override
                        public void onResponse(final Snapshot snapshot) {
                            initializingSnapshots.remove(snapshot);
                            listener.onResponse(snapshot);
                        }

                        @Override
                        public void onFailure(final Exception e) {
                            initializingSnapshots.remove(current);
                            listener.onFailure(e);
                        }
                    });
                }
            }

            @Override
            public TimeValue timeout() {
                return request.masterNodeTimeout();
            }
        });
    }, listener::onFailure);
}

Example 12 with StepListener

use of org.elasticsearch.action.StepListener in project crate by crate.

the class SnapshotsService method snapshots.

/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @param snapshotIds       snapshots for which to fetch snapshot information
 * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
 *                          if false, they will throw an error
 * @return list of snapshots
 */
public void snapshots(final String repositoryName, final List<SnapshotId> snapshotIds, final boolean ignoreUnavailable, ActionListener<Collection<SnapshotInfo>> listener) {
    final Set<SnapshotInfo> snapshotSet = new HashSet<>();
    final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
    // first, look at the snapshots in progress
    final List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList()));
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(inProgressSnapshot(entry));
        snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId());
    }
    if (snapshotIdsToIterate.isEmpty()) {
        listener.onResponse(snapshotSet);
        return;
    }
    // then, look in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    GroupedActionListener<SnapshotInfo> snapshotInfosListener = new GroupedActionListener<>(ActionListener.wrap(snapshotInfos -> {
        final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
        snapshotList.addAll(snapshotInfos);
        CollectionUtil.timSort(snapshotList);
        listener.onResponse(unmodifiableList(snapshotList));
    }, listener::onFailure), snapshotIdsToIterate.size());
    for (SnapshotId snapshotId : snapshotIdsToIterate) {
        StepListener<SnapshotInfo> future = new StepListener<>();
        future.whenComplete(snapshotInfosListener::onResponse, (ex) -> {
            if (ignoreUnavailable) {
                LOGGER.warn(() -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);
                snapshotInfosListener.onResponse(null);
            } else {
                snapshotInfosListener.onFailure(ex);
            }
        });
        repository.getSnapshotInfo(snapshotId, future);
    }
}

Example 13 with StepListener

use of org.elasticsearch.action.StepListener in project crate by crate.

the class BlobStoreRepository method writeIndexGen.

/**
 * Writing a new index generation is a three step process.
 * First, the {@link RepositoryMetadata} entry for this repository is set into a pending state by incrementing its
 * pending generation {@code P} while its safe generation {@code N} remains unchanged.
 * Second, the updated {@link RepositoryData} is written to generation {@code P + 1}.
 * Lastly, the {@link RepositoryMetadata} entry for this repository is updated to the new generation {@code P + 1} and thus
 * pending and safe generation are set to the same value marking the end of the update of the repository data.
 *
 * @param repositoryData RepositoryData to write
 * @param expectedGen    expected repository generation at the start of the operation
 * @param writeShardGens whether to write {@link ShardGenerations} to the new {@link RepositoryData} blob
 * @param listener       completion listener
 */
protected void writeIndexGen(RepositoryData repositoryData, long expectedGen, boolean writeShardGens, ActionListener<Void> listener) {
    // can not write to a read only repository
    assert isReadOnly() == false;
    final long currentGen = repositoryData.getGenId();
    if (currentGen != expectedGen) {
        // the index file was updated by a concurrent operation, so we were operating on stale
        // repository data
        listener.onFailure(new RepositoryException(metadata.name(), "concurrent modification of the index-N file, expected current generation [" + expectedGen + "], actual current generation [" + currentGen + "]"));
        return;
    }
    // Step 1: Set repository generation state to the next possible pending generation
    final StepListener<Long> setPendingStep = new StepListener<>();
    clusterService.submitStateUpdateTask("set pending repository generation [" + metadata.name() + "][" + expectedGen + "]", new ClusterStateUpdateTask() {

        private long newGen;

        @Override
        public ClusterState execute(ClusterState currentState) {
            final RepositoryMetadata meta = getRepoMetadata(currentState);
            final String repoName = metadata.name();
            final long genInState = meta.generation();
            final boolean uninitializedMeta = meta.generation() == RepositoryData.UNKNOWN_REPO_GEN || bestEffortConsistency;
            if (uninitializedMeta == false && meta.pendingGeneration() != genInState) {
                LOGGER.info("Trying to write new repository data over unfinished write, repo [{}] is at " + "safe generation [{}] and pending generation [{}]", meta.name(), genInState, meta.pendingGeneration());
            }
            assert expectedGen == RepositoryData.EMPTY_REPO_GEN || uninitializedMeta || expectedGen == meta.generation() : "Expected non-empty generation [" + expectedGen + "] does not match generation tracked in [" + meta + "]";
            // If we run into the empty repo generation for the expected gen, the repo is assumed to have been cleared of
            // all contents by an external process so we reset the safe generation to the empty generation.
            final long safeGeneration = expectedGen == RepositoryData.EMPTY_REPO_GEN ? RepositoryData.EMPTY_REPO_GEN : (uninitializedMeta ? expectedGen : genInState);
            // Regardless of whether or not the safe generation has been reset, the pending generation always increments so that
            // even if a repository has been manually cleared of all contents we will never reuse the same repository generation.
            // This is motivated by the consistency behavior the S3 based blob repository implementation has to support which does
            // not offer any consistency guarantees when it comes to overwriting the same blob name with different content.
            final long nextPendingGen = metadata.pendingGeneration() + 1;
            newGen = uninitializedMeta ? Math.max(expectedGen + 1, nextPendingGen) : nextPendingGen;
            assert newGen > latestKnownRepoGen.get() : "Attempted new generation [" + newGen + "] must be larger than latest known generation [" + latestKnownRepoGen.get() + "]";
            return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, currentState.metadata().<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE).withUpdatedGeneration(repoName, safeGeneration, newGen)).build()).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            listener.onFailure(new RepositoryException(metadata.name(), "Failed to execute cluster state update [" + source + "]", e));
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            setPendingStep.onResponse(newGen);
        }
    });
    final StepListener<RepositoryData> filterRepositoryDataStep = new StepListener<>();
    // Step 2: Write new index-N blob to repository and update index.latest
    setPendingStep.whenComplete(newGen -> threadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> {
        // BwC logic: Load snapshot version information if any snapshot is missing a version in RepositoryData so that the new
        // RepositoryData contains a version for every snapshot
        final List<SnapshotId> snapshotIdsWithoutVersion = repositoryData.getSnapshotIds().stream().filter(snapshotId -> repositoryData.getVersion(snapshotId) == null).collect(Collectors.toList());
        if (snapshotIdsWithoutVersion.isEmpty() == false) {
            final Map<SnapshotId, Version> updatedVersionMap = new ConcurrentHashMap<>();
            final GroupedActionListener<Void> loadAllVersionsListener = new GroupedActionListener<>(ActionListener.runAfter(new ActionListener<Collection<Void>>() {

                @Override
                public void onResponse(Collection<Void> voids) {
                    LOGGER.info("Successfully loaded all snapshot's version information for {} from snapshot metadata", AllocationService.firstListElementsToCommaDelimitedString(snapshotIdsWithoutVersion, SnapshotId::toString, LOGGER.isDebugEnabled()));
                }

                @Override
                public void onFailure(Exception e) {
                    LOGGER.warn("Failure when trying to load missing version information from snapshot metadata", e);
                }
            }, () -> filterRepositoryDataStep.onResponse(repositoryData.withVersions(updatedVersionMap))), snapshotIdsWithoutVersion.size());
            for (SnapshotId snapshotId : snapshotIdsWithoutVersion) {
                threadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.run(loadAllVersionsListener, () -> {
                    ActionListener<SnapshotInfo> snapshotInfoListener = ActionListener.delegateFailure(loadAllVersionsListener, (delegate, snapshotInfo) -> {
                        updatedVersionMap.put(snapshotId, snapshotInfo.version());
                        delegate.onResponse(null);
                    });
                    getSnapshotInfo(snapshotId, snapshotInfoListener);
                }));
            }
        } else {
            filterRepositoryDataStep.onResponse(repositoryData);
        }
    })), listener::onFailure);
    filterRepositoryDataStep.whenComplete(filteredRepositoryData -> {
        final long newGen = setPendingStep.result();
        if (latestKnownRepoGen.get() >= newGen) {
            throw new IllegalArgumentException("Tried writing generation [" + newGen + "] but repository is at least at generation [" + latestKnownRepoGen.get() + "] already");
        }
        // write the index file
        final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen);
        LOGGER.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob);
        writeAtomic(indexBlob, BytesReference.bytes(filteredRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), writeShardGens)), true);
        // write the current generation to the index-latest file
        final BytesReference genBytes;
        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
            bStream.writeLong(newGen);
            genBytes = bStream.bytes();
        }
        LOGGER.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen);
        writeAtomic(INDEX_LATEST_BLOB, genBytes, false);
        // Step 3: Update CS to reflect new repository generation.
        clusterService.submitStateUpdateTask("set safe repository generation [" + metadata.name() + "][" + newGen + "]", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                final RepositoryMetadata meta = getRepoMetadata(currentState);
                if (meta.generation() != expectedGen) {
                    throw new IllegalStateException("Tried to update repo generation to [" + newGen + "] but saw unexpected generation in state [" + meta + "]");
                }
                if (meta.pendingGeneration() != newGen) {
                    throw new IllegalStateException("Tried to update from unexpected pending repo generation [" + meta.pendingGeneration() + "] after write to generation [" + newGen + "]");
                }
                return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, currentState.metadata().<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE).withUpdatedGeneration(metadata.name(), newGen, newGen)).build()).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                listener.onFailure(new RepositoryException(metadata.name(), "Failed to execute cluster state update [" + source + "]", e));
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.run(listener, () -> {
                    // Delete all now outdated index files up to 1000 blobs back from the new generation.
                    // If there are more than 1000 dangling index-N cleanup functionality on repo delete will take care of them.
                    // Deleting one older than the current expectedGen is done for BwC reasons as older versions used to keep
                    // two index-N blobs around.
                    final List<String> oldIndexN = LongStream.range(Math.max(Math.max(expectedGen - 1, 0), newGen - 1000), newGen).mapToObj(gen -> INDEX_FILE_PREFIX + gen).collect(Collectors.toList());
                    try {
                        blobContainer().deleteBlobsIgnoringIfNotExists(oldIndexN);
                    } catch (IOException e) {
                        LOGGER.warn("Failed to clean up old index blobs {}", oldIndexN);
                    }
                }));
            }
        });
    }, listener::onFailure);
}

Example 14 with StepListener

use of org.elasticsearch.action.StepListener in project crate by crate.

the class RestoreService method restoreSnapshot.

/**
 * Restores snapshot specified in the restore request.
 *
 * @param request  restore request
 * @param listener restore listener
 */
public void restoreSnapshot(final RestoreRequest request, final ActionListener<RestoreCompletionResponse> listener) {
    try {
        // Read snapshot info and metadata from the repository
        final String repositoryName = request.repositoryName;
        Repository repository = repositoriesService.repository(repositoryName);
        final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
        repository.getRepositoryData(repositoryDataListener);
        repositoryDataListener.whenComplete(repositoryData -> {
            final String snapshotName = request.snapshotName;
            final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
            if (matchingSnapshotId.isPresent() == false) {
                throw new SnapshotRestoreException(repositoryName, snapshotName, "snapshot does not exist");
            }
            final SnapshotId snapshotId = matchingSnapshotId.get();
            repository.getSnapshotInfo(snapshotId, ActionListener.delegateFailure(listener, (delegate, snapshotInfo) -> {
                final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
                // Make sure that we can restore from this snapshot
                validateSnapshotRestorable(repositoryName, snapshotInfo);
                // Resolve the indices from the snapshot that need to be restored
                final List<String> indicesInSnapshot = request.includeIndices() ? filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions()) : List.of();
                final StepListener<Metadata> globalMetadataListener = new StepListener<>();
                if (request.includeCustomMetadata() || request.includeGlobalSettings() || request.allTemplates() || (request.templates() != null && request.templates().length > 0)) {
                    repository.getSnapshotGlobalMetadata(snapshotId, globalMetadataListener);
                } else {
                    globalMetadataListener.onResponse(Metadata.EMPTY_METADATA);
                }
                globalMetadataListener.whenComplete(globalMetadata -> {
                    var metadataBuilder = Metadata.builder(globalMetadata);
                    var indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot);
                    var snapshotIndexMetadataListener = new StepListener<Collection<IndexMetadata>>();
                    repository.getSnapshotIndexMetadata(snapshotId, indexIdsInSnapshot, snapshotIndexMetadataListener);
                    snapshotIndexMetadataListener.whenComplete(snapshotIndexMetadata -> {
                        for (IndexMetadata indexMetadata : snapshotIndexMetadata) {
                            metadataBuilder.put(indexMetadata, false);
                        }
                        final Metadata metadata = metadataBuilder.build();
                        // Apply renaming on index names, returning a map of names where
                        // the key is the renamed index and the value is the original name
                        final Map<String, String> indices = renamedIndices(request, indicesInSnapshot);
                        // Now we can start the actual restore process by adding shards to be recovered in the cluster state
                        // and updating cluster metadata (global and index) as needed
                        clusterService.submitStateUpdateTask("restore_snapshot[" + snapshotName + ']', new ClusterStateUpdateTask() {

                            final String restoreUUID = UUIDs.randomBase64UUID();

                            RestoreInfo restoreInfo = null;

                            @Override
                            public ClusterState execute(ClusterState currentState) {
                                RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
                                // Check if the snapshot to restore is currently being deleted
                                SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
                                if (deletionsInProgress != null && deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.getSnapshot().equals(snapshot))) {
                                    throw new ConcurrentSnapshotExecutionException(snapshot, "cannot restore a snapshot while a snapshot deletion is in-progress [" + deletionsInProgress.getEntries().get(0).getSnapshot() + "]");
                                }
                                // Updating cluster state
                                ClusterState.Builder builder = ClusterState.builder(currentState);
                                Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
                                ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
                                RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
                                ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards;
                                Set<String> aliases = new HashSet<>();
                                if (indices.isEmpty() == false) {
                                    // We have some indices to restore
                                    ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder();
                                    final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion();
                                    for (Map.Entry<String, String> indexEntry : indices.entrySet()) {
                                        String index = indexEntry.getValue();
                                        boolean partial = checkPartial(index);
                                        SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(restoreUUID, snapshot, snapshotInfo.version(), index);
                                        String renamedIndexName = indexEntry.getKey();
                                        IndexMetadata snapshotIndexMetadata = metadata.index(index);
                                        snapshotIndexMetadata = updateIndexSettings(snapshotIndexMetadata, request.indexSettings(), request.ignoreIndexSettings());
                                        try {
                                            snapshotIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion);
                                        } catch (Exception ex) {
                                            throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
                                        }
                                        // Check that the index is closed or doesn't exist
                                        IndexMetadata currentIndexMetadata = currentState.metadata().index(renamedIndexName);
                                        IntSet ignoreShards = new IntHashSet();
                                        final Index renamedIndex;
                                        if (currentIndexMetadata == null) {
                                            // Index doesn't exist - create it and start recovery
                                            // Make sure that the index we are about to create has a validate name
                                            MetadataCreateIndexService.validateIndexName(renamedIndexName, currentState);
                                            createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetadata.getSettings(), currentState, false);
                                            IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata).state(IndexMetadata.State.OPEN).index(renamedIndexName);
                                            indexMdBuilder.settings(Settings.builder().put(snapshotIndexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()));
                                            shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState);
                                            if (!request.includeAliases() && !snapshotIndexMetadata.getAliases().isEmpty()) {
                                                // Remove all aliases - they shouldn't be restored
                                                indexMdBuilder.removeAllAliases();
                                            } else {
                                                for (ObjectCursor<String> alias : snapshotIndexMetadata.getAliases().keys()) {
                                                    aliases.add(alias.value);
                                                }
                                            }
                                            IndexMetadata updatedIndexMetadata = indexMdBuilder.build();
                                            if (partial) {
                                                populateIgnoredShards(index, ignoreShards);
                                            }
                                            rtBuilder.addAsNewRestore(updatedIndexMetadata, recoverySource, ignoreShards);
                                            blocks.addBlocks(updatedIndexMetadata);
                                            mdBuilder.put(updatedIndexMetadata, true);
                                            renamedIndex = updatedIndexMetadata.getIndex();
                                        } else {
                                            validateExistingIndex(currentIndexMetadata, snapshotIndexMetadata, renamedIndexName, partial);
                                            // Index exists and it's closed - open it in metadata and start recovery
                                            IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata).state(IndexMetadata.State.OPEN);
                                            indexMdBuilder.version(Math.max(snapshotIndexMetadata.getVersion(), 1 + currentIndexMetadata.getVersion()));
                                            indexMdBuilder.mappingVersion(Math.max(snapshotIndexMetadata.getMappingVersion(), 1 + currentIndexMetadata.getMappingVersion()));
                                            indexMdBuilder.settingsVersion(Math.max(snapshotIndexMetadata.getSettingsVersion(), 1 + currentIndexMetadata.getSettingsVersion()));
                                            for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
                                                indexMdBuilder.primaryTerm(shard, Math.max(snapshotIndexMetadata.primaryTerm(shard), currentIndexMetadata.primaryTerm(shard)));
                                            }
                                            if (!request.includeAliases()) {
                                                // Remove all snapshot aliases
                                                if (!snapshotIndexMetadata.getAliases().isEmpty()) {
                                                    indexMdBuilder.removeAllAliases();
                                                }
                                                // Add existing aliases
                                                for (ObjectCursor<AliasMetadata> alias : currentIndexMetadata.getAliases().values()) {
                                                    indexMdBuilder.putAlias(alias.value);
                                                }
                                            } else {
                                                for (ObjectCursor<String> alias : snapshotIndexMetadata.getAliases().keys()) {
                                                    aliases.add(alias.value);
                                                }
                                            }
                                            indexMdBuilder.settings(Settings.builder().put(snapshotIndexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_UUID, currentIndexMetadata.getIndexUUID()));
                                            IndexMetadata updatedIndexMetadata = indexMdBuilder.index(renamedIndexName).build();
                                            rtBuilder.addAsRestore(updatedIndexMetadata, recoverySource);
                                            blocks.updateBlocks(updatedIndexMetadata);
                                            mdBuilder.put(updatedIndexMetadata, true);
                                            renamedIndex = updatedIndexMetadata.getIndex();
                                        }
                                        for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
                                            if (!ignoreShards.contains(shard)) {
                                                shardsBuilder.put(new ShardId(renamedIndex, shard), new RestoreInProgress.ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId()));
                                            } else {
                                                shardsBuilder.put(new ShardId(renamedIndex, shard), new RestoreInProgress.ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.FAILURE));
                                            }
                                        }
                                    }
                                    shards = shardsBuilder.build();
                                    RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(restoreUUID, snapshot, overallState(RestoreInProgress.State.INIT, shards), List.copyOf(indices.keySet()), shards);
                                    RestoreInProgress.Builder restoreInProgressBuilder;
                                    if (restoreInProgress != null) {
                                        restoreInProgressBuilder = new RestoreInProgress.Builder(restoreInProgress);
                                    } else {
                                        restoreInProgressBuilder = new RestoreInProgress.Builder();
                                    }
                                    builder.putCustom(RestoreInProgress.TYPE, restoreInProgressBuilder.add(restoreEntry).build());
                                } else {
                                    shards = ImmutableOpenMap.of();
                                }
                                validateExistingTemplates();
                                checkAliasNameConflicts(indices, aliases);
                                // Restore templates (but do NOT overwrite existing templates)
                                restoreTemplates(mdBuilder, currentState);
                                // Restore global state if needed
                                if (request.includeGlobalSettings() && metadata.persistentSettings() != null) {
                                    Settings settings = metadata.persistentSettings();
                                    // CrateDB patch to only restore defined settings
                                    if (request.globalSettings().length > 0) {
                                        var filteredSettingBuilder = Settings.builder();
                                        for (String prefix : request.globalSettings()) {
                                            filteredSettingBuilder.put(settings.filter(s -> s.startsWith(prefix)));
                                        }
                                        settings = filteredSettingBuilder.build();
                                    }
                                    clusterSettings.validateUpdate(settings);
                                    mdBuilder.persistentSettings(settings);
                                }
                                if (request.includeCustomMetadata() && metadata.customs() != null) {
                                    // CrateDB patch to only restore defined custom metadata types
                                    List<String> customMetadataTypes = Arrays.asList(request.customMetadataTypes());
                                    boolean includeAll = customMetadataTypes.size() == 0;
                                    for (ObjectObjectCursor<String, Metadata.Custom> cursor : metadata.customs()) {
                                        if (!RepositoriesMetadata.TYPE.equals(cursor.key)) {
                                            if (includeAll || customMetadataTypes.contains(cursor.key)) {
                                                mdBuilder.putCustom(cursor.key, cursor.value);
                                            }
                                        }
                                    }
                                }
                                if (completed(shards)) {
                                    // We don't have any indices to restore - we are done
                                    restoreInfo = new RestoreInfo(snapshotId.getName(), Collections.unmodifiableList(new ArrayList<>(indices.keySet())), shards.size(), shards.size() - failedShards(shards));
                                }
                                RoutingTable rt = rtBuilder.build();
                                ClusterState updatedState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rt).build();
                                return allocationService.reroute(updatedState, "restored snapshot [" + snapshot + "]");
                            }

                            private void checkAliasNameConflicts(Map<String, String> renamedIndices, Set<String> aliases) {
                                for (Map.Entry<String, String> renamedIndex : renamedIndices.entrySet()) {
                                    if (aliases.contains(renamedIndex.getKey())) {
                                        throw new SnapshotRestoreException(snapshot, "cannot rename index [" + renamedIndex.getValue() + "] into [" + renamedIndex.getKey() + "] because of conflict with an alias with the same name");
                                    }
                                }
                            }

                            private void populateIgnoredShards(String index, IntSet ignoreShards) {
                                for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) {
                                    if (index.equals(failure.index())) {
                                        ignoreShards.add(failure.shardId());
                                    }
                                }
                            }

                            private boolean checkPartial(String index) {
                                // Make sure that index was fully snapshotted
                                if (failed(snapshotInfo, index)) {
                                    if (request.partial()) {
                                        return true;
                                    } else {
                                        throw new SnapshotRestoreException(snapshot, "index [" + index + "] wasn't fully snapshotted - cannot " + "restore");
                                    }
                                } else {
                                    return false;
                                }
                            }

                            private void validateExistingIndex(IndexMetadata currentIndexMetadata, IndexMetadata snapshotIndexMetadata, String renamedIndex, boolean partial) {
                                // Index exist - checking that it's closed
                                if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) {
                                    // TODO: Enable restore for open indices
                                    throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because an open index " + "with same name already exists in the cluster. Either close or delete the existing index or restore the " + "index under a different name by providing a rename pattern and replacement name");
                                }
                                // Index exist - checking if it's partial restore
                                if (partial) {
                                    throw new SnapshotRestoreException(snapshot, "cannot restore partial index [" + renamedIndex + "] because such index already exists");
                                }
                                // Make sure that the number of shards is the same. That's the only thing that we cannot change
                                if (currentIndexMetadata.getNumberOfShards() != snapshotIndexMetadata.getNumberOfShards()) {
                                    throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetadata.getNumberOfShards() + "] shards from a snapshot of index [" + snapshotIndexMetadata.getIndex().getName() + "] with [" + snapshotIndexMetadata.getNumberOfShards() + "] shards");
                                }
                            }

                            /**
                             * Optionally updates index settings in indexMetadata by removing settings listed in ignoreSettings and
                             * merging them with settings in changeSettings.
                             */
                            private IndexMetadata updateIndexSettings(IndexMetadata indexMetadata, Settings changeSettings, String[] ignoreSettings) {
                                if (changeSettings.names().isEmpty() && ignoreSettings.length == 0) {
                                    return indexMetadata;
                                }
                                Settings normalizedChangeSettings = Settings.builder().put(changeSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
                                IndexMetadata.Builder builder = IndexMetadata.builder(indexMetadata);
                                Settings settings = indexMetadata.getSettings();
                                Set<String> keyFilters = new HashSet<>();
                                List<String> simpleMatchPatterns = new ArrayList<>();
                                for (String ignoredSetting : ignoreSettings) {
                                    if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
                                        if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
                                            throw new SnapshotRestoreException(snapshot, "cannot remove setting [" + ignoredSetting + "] on restore");
                                        } else {
                                            keyFilters.add(ignoredSetting);
                                        }
                                    } else {
                                        simpleMatchPatterns.add(ignoredSetting);
                                    }
                                }
                                Predicate<String> settingsFilter = k -> {
                                    if (UNREMOVABLE_SETTINGS.contains(k) == false) {
                                        for (String filterKey : keyFilters) {
                                            if (k.equals(filterKey)) {
                                                return false;
                                            }
                                        }
                                        for (String pattern : simpleMatchPatterns) {
                                            if (Regex.simpleMatch(pattern, k)) {
                                                return false;
                                            }
                                        }
                                    }
                                    return true;
                                };
                                Settings.Builder settingsBuilder = Settings.builder().put(settings.filter(settingsFilter)).put(normalizedChangeSettings.filter(k -> {
                                    if (UNMODIFIABLE_SETTINGS.contains(k)) {
                                        throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + k + "] on restore");
                                    } else {
                                        return true;
                                    }
                                }));
                                settingsBuilder.remove(IndexMetadata.VERIFIED_BEFORE_CLOSE_SETTING.getKey());
                                return builder.settings(settingsBuilder).build();
                            }

                            private void restoreTemplates(Metadata.Builder mdBuilder, ClusterState currentState) {
                                List<String> toRestore = Arrays.asList(request.templates());
                                if (metadata.templates() != null) {
                                    for (ObjectCursor<IndexTemplateMetadata> cursor : metadata.templates().values()) {
                                        if (currentState.metadata().templates().get(cursor.value.name()) == null && (request.allTemplates() || toRestore.contains(cursor.value.name()))) {
                                            mdBuilder.put(cursor.value);
                                        }
                                    }
                                }
                            }

                            private void validateExistingTemplates() {
                                if (request.indicesOptions().ignoreUnavailable() || request.allTemplates()) {
                                    return;
                                }
                                for (String template : request.templates()) {
                                    if (!metadata.templates().containsKey(template)) {
                                        throw new ResourceNotFoundException("[{}] template not found", template);
                                    }
                                }
                            }

                            @Override
                            public void onFailure(String source, Exception e) {
                                LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e);
                                listener.onFailure(e);
                            }

                            @Override
                            public TimeValue timeout() {
                                return request.masterNodeTimeout();
                            }

                            @Override
                            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                                listener.onResponse(new RestoreCompletionResponse(restoreUUID, snapshot, restoreInfo));
                            }
                        });
                    }, listener::onFailure);
                }, listener::onFailure);
            }));
        }, listener::onFailure);
    } catch (Exception e) {
        LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", request.repositoryName + ":" + request.snapshotName), e);
        listener.onFailure(e);
    }
}