Examples with State org.opensearch.cluster.SnapshotsInProgress.State used on opensource projects

Example 1 with State

use of org.opensearch.cluster.SnapshotsInProgress.State in project OpenSearch by opensearch-project.

the class SharedClusterSnapshotRestoreIT method testRestoreSnapshotWithCorruptedIndexMetadata.

/**
 * Tests that a snapshot of multiple indices including one with a corrupted index metadata
 * file can still be used to restore the non corrupted indices
 */
public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception {
    final Client client = client();
    final Path repo = randomRepoPath();
    final int nbIndices = randomIntBetween(2, 3);
    final Map<String, Integer> nbDocsPerIndex = new HashMap<>();
    for (int i = 0; i < nbIndices; i++) {
        String indexName = "test-idx-" + i;
        assertAcked(prepareCreate(indexName).setSettings(indexSettingsNoReplicas(Math.min(2, numberOfShards()))));
        int nbDocs = randomIntBetween(1, 10);
        nbDocsPerIndex.put(indexName, nbDocs);
        IndexRequestBuilder[] documents = new IndexRequestBuilder[nbDocs];
        for (int j = 0; j < nbDocs; j++) {
            documents[j] = client.prepareIndex(indexName).setSource("foo", "bar");
        }
        indexRandom(true, documents);
    }
    flushAndRefresh();
    createRepository("test-repo", "fs", repo);
    final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap");
    assertThat(snapshotInfo.indices(), hasSize(nbIndices));
    final RepositoryData repositoryData = getRepositoryData("test-repo");
    final Map<String, IndexId> indexIds = repositoryData.getIndices();
    assertThat(indexIds.size(), equalTo(nbIndices));
    // Choose a random index from the snapshot
    final IndexId corruptedIndex = randomFrom(indexIds.values());
    final Path indexMetadataPath = repo.resolve("indices").resolve(corruptedIndex.getId()).resolve("meta-" + repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotInfo.snapshotId(), corruptedIndex) + ".dat");
    // Truncate the index metadata file
    try (SeekableByteChannel outChan = Files.newByteChannel(indexMetadataPath, StandardOpenOption.WRITE)) {
        outChan.truncate(randomInt(10));
    }
    List<SnapshotInfo> snapshotInfos = clusterAdmin().prepareGetSnapshots("test-repo").get().getSnapshots();
    assertThat(snapshotInfos.size(), equalTo(1));
    assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
    assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap"));
    assertAcked(client().admin().indices().prepareDelete(nbDocsPerIndex.keySet().toArray(new String[nbDocsPerIndex.size()])));
    Predicate<String> isRestorableIndex = index -> corruptedIndex.getName().equals(index) == false;
    clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setIndices(nbDocsPerIndex.keySet().stream().filter(isRestorableIndex).toArray(String[]::new)).setRestoreGlobalState(randomBoolean()).setWaitForCompletion(true).get();
    ensureGreen();
    for (Map.Entry<String, Integer> entry : nbDocsPerIndex.entrySet()) {
        if (isRestorableIndex.test(entry.getKey())) {
            assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().longValue());
        }
    }
    assertAcked(startDeleteSnapshot("test-repo", snapshotInfo.snapshotId().getName()).get());
}

Example 2 with State

use of org.opensearch.cluster.SnapshotsInProgress.State in project OpenSearch by opensearch-project.

the class SnapshotsService method shards.

/**
 * Calculates the assignment of shards to data nodes for a new snapshot based on the given cluster state and the
 * indices that should be included in the snapshot.
 *
 * @param indices             Indices to snapshot
 * @param useShardGenerations whether to write {@link ShardGenerations} during the snapshot
 * @return list of shard to be included into current snapshot
 */
private static ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards(SnapshotsInProgress snapshotsInProgress, @Nullable SnapshotDeletionsInProgress deletionsInProgress, Metadata metadata, RoutingTable routingTable, List<IndexId> indices, boolean useShardGenerations, RepositoryData repositoryData, String repoName) {
    ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
    final ShardGenerations shardGenerations = repositoryData.shardGenerations();
    final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo(repoName, snapshotsInProgress.entries());
    final boolean readyToExecute = deletionsInProgress == null || deletionsInProgress.getEntries().stream().noneMatch(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED);
    for (IndexId index : indices) {
        final String indexName = index.getName();
        final boolean isNewIndex = repositoryData.getIndices().containsKey(indexName) == false;
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            // The index was deleted before we managed to start the snapshot - mark it as missing.
            builder.put(new ShardId(indexName, IndexMetadata.INDEX_UUID_NA_VALUE, 0), ShardSnapshotStatus.MISSING);
        } else {
            final IndexRoutingTable indexRoutingTable = routingTable.index(indexName);
            for (int i = 0; i < indexMetadata.getNumberOfShards(); i++) {
                final ShardId shardId = indexRoutingTable.shard(i).shardId();
                final String shardRepoGeneration;
                if (useShardGenerations) {
                    final String inFlightGeneration = inFlightShardStates.generationForShard(index, shardId.id(), shardGenerations);
                    if (inFlightGeneration == null && isNewIndex) {
                        assert shardGenerations.getShardGen(index, shardId.getId()) == null : "Found shard generation for new index [" + index + "]";
                        shardRepoGeneration = ShardGenerations.NEW_SHARD_GEN;
                    } else {
                        shardRepoGeneration = inFlightGeneration;
                    }
                } else {
                    shardRepoGeneration = null;
                }
                final ShardSnapshotStatus shardSnapshotStatus;
                if (indexRoutingTable == null) {
                    shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus(null, ShardState.MISSING, "missing routing table", shardRepoGeneration);
                } else {
                    ShardRouting primary = indexRoutingTable.shard(i).primaryShard();
                    if (readyToExecute == false || inFlightShardStates.isActive(indexName, i)) {
                        shardSnapshotStatus = ShardSnapshotStatus.UNASSIGNED_QUEUED;
                    } else if (primary == null || !primary.assignedToNode()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(null, ShardState.MISSING, "primary shard is not allocated", shardRepoGeneration);
                    } else if (primary.relocating() || primary.initializing()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), ShardState.WAITING, shardRepoGeneration);
                    } else if (!primary.started()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), ShardState.MISSING, "primary shard hasn't been started yet", shardRepoGeneration);
                    } else {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), shardRepoGeneration);
                    }
                }
                builder.put(shardId, shardSnapshotStatus);
            }
        }
    }
    return builder.build();
}

Example 3 with State

use of org.opensearch.cluster.SnapshotsInProgress.State in project OpenSearch by opensearch-project.

the class SnapshotsService method createDeleteStateUpdate.

private ClusterStateUpdateTask createDeleteStateUpdate(List<SnapshotId> snapshotIds, String repoName, RepositoryData repositoryData, Priority priority, ActionListener<Void> listener) {
    // Short circuit to noop state update if there isn't anything to delete
    if (snapshotIds.isEmpty()) {
        return new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                listener.onFailure(e);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                listener.onResponse(null);
            }
        };
    }
    return new ClusterStateUpdateTask(priority) {

        private SnapshotDeletionsInProgress.Entry newDelete;

        private boolean reusedExistingDelete = false;

        // Snapshots that had all of their shard snapshots in queued state and thus were removed from the
        // cluster state right away
        private final Collection<Snapshot> completedNoCleanup = new ArrayList<>();

        // Snapshots that were aborted and that already wrote data to the repository and now have to be deleted
        // from the repository after the cluster state update
        private final Collection<SnapshotsInProgress.Entry> completedWithCleanup = new ArrayList<>();

        @Override
        public ClusterState execute(ClusterState currentState) {
            final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
            final Version minNodeVersion = currentState.nodes().getMinNodeVersion();
            if (minNodeVersion.before(FULL_CONCURRENCY_VERSION)) {
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete - another snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
            }
            final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
            if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete snapshots while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]");
            }
            final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY);
            for (RestoreInProgress.Entry entry : restoreInProgress) {
                if (repoName.equals(entry.snapshot().getRepository()) && snapshotIds.contains(entry.snapshot().getSnapshotId())) {
                    throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete snapshot during a restore in progress in [" + restoreInProgress + "]");
                }
            }
            final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
            final Set<SnapshotId> activeCloneSources = snapshots.entries().stream().filter(SnapshotsInProgress.Entry::isClone).map(SnapshotsInProgress.Entry::source).collect(Collectors.toSet());
            for (SnapshotId snapshotId : snapshotIds) {
                if (activeCloneSources.contains(snapshotId)) {
                    throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotId), "cannot delete snapshot while it is being cloned");
                }
            }
            // Snapshot ids that will have to be physically deleted from the repository
            final Set<SnapshotId> snapshotIdsRequiringCleanup = new HashSet<>(snapshotIds);
            final SnapshotsInProgress updatedSnapshots;
            if (minNodeVersion.onOrAfter(FULL_CONCURRENCY_VERSION)) {
                updatedSnapshots = SnapshotsInProgress.of(snapshots.entries().stream().map(existing -> {
                    if (existing.state() == State.STARTED && snapshotIdsRequiringCleanup.contains(existing.snapshot().getSnapshotId())) {
                        // snapshot is started - mark every non completed shard as aborted
                        final SnapshotsInProgress.Entry abortedEntry = existing.abort();
                        if (abortedEntry == null) {
                            // No work has been done for this snapshot yet so we remove it from the cluster state directly
                            final Snapshot existingNotYetStartedSnapshot = existing.snapshot();
                            // any leaked listener assertions
                            if (endingSnapshots.add(existingNotYetStartedSnapshot)) {
                                completedNoCleanup.add(existingNotYetStartedSnapshot);
                            }
                            snapshotIdsRequiringCleanup.remove(existingNotYetStartedSnapshot.getSnapshotId());
                        } else if (abortedEntry.state().completed()) {
                            completedWithCleanup.add(abortedEntry);
                        }
                        return abortedEntry;
                    }
                    return existing;
                }).filter(Objects::nonNull).collect(Collectors.toList()));
                if (snapshotIdsRequiringCleanup.isEmpty()) {
                    // We only saw snapshots that could be removed from the cluster state right away, no need to update the deletions
                    return updateWithSnapshots(currentState, updatedSnapshots, null);
                }
            } else {
                if (snapshots.entries().isEmpty() == false) {
                    // However other snapshots are running - cannot continue
                    throw new ConcurrentSnapshotExecutionException(repoName, snapshotIds.toString(), "another snapshot is currently running cannot delete");
                }
                updatedSnapshots = snapshots;
            }
            // add the snapshot deletion to the cluster state
            final SnapshotDeletionsInProgress.Entry replacedEntry = deletionsInProgress.getEntries().stream().filter(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.WAITING).findFirst().orElse(null);
            if (replacedEntry == null) {
                final Optional<SnapshotDeletionsInProgress.Entry> foundDuplicate = deletionsInProgress.getEntries().stream().filter(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED && entry.getSnapshots().containsAll(snapshotIds)).findFirst();
                if (foundDuplicate.isPresent()) {
                    newDelete = foundDuplicate.get();
                    reusedExistingDelete = true;
                    return currentState;
                }
                final List<SnapshotId> toDelete = Collections.unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup));
                ensureBelowConcurrencyLimit(repoName, toDelete.get(0).getName(), snapshots, deletionsInProgress);
                newDelete = new SnapshotDeletionsInProgress.Entry(toDelete, repoName, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), updatedSnapshots.entries().stream().filter(entry -> repoName.equals(entry.repository())).noneMatch(SnapshotsService::isWritingToRepository) && deletionsInProgress.getEntries().stream().noneMatch(entry -> repoName.equals(entry.repository()) && entry.state() == SnapshotDeletionsInProgress.State.STARTED) ? SnapshotDeletionsInProgress.State.STARTED : SnapshotDeletionsInProgress.State.WAITING);
            } else {
                newDelete = replacedEntry.withAddedSnapshots(snapshotIdsRequiringCleanup);
            }
            return updateWithSnapshots(currentState, updatedSnapshots, (replacedEntry == null ? deletionsInProgress : deletionsInProgress.withRemovedEntry(replacedEntry.uuid())).withAddedEntry(newDelete));
        }

        @Override
        public void onFailure(String source, Exception e) {
            endingSnapshots.removeAll(completedNoCleanup);
            listener.onFailure(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            if (completedNoCleanup.isEmpty() == false) {
                logger.info("snapshots {} aborted", completedNoCleanup);
            }
            for (Snapshot snapshot : completedNoCleanup) {
                failSnapshotCompletionListeners(snapshot, new SnapshotException(snapshot, SnapshotsInProgress.ABORTED_FAILURE_TEXT));
            }
            if (newDelete == null) {
                listener.onResponse(null);
            } else {
                addDeleteListener(newDelete.uuid(), listener);
                if (reusedExistingDelete) {
                    return;
                }
                if (newDelete.state() == SnapshotDeletionsInProgress.State.STARTED) {
                    if (tryEnterRepoLoop(repoName)) {
                        deleteSnapshotsFromRepository(newDelete, repositoryData, newState.nodes().getMinNodeVersion());
                    } else {
                        logger.trace("Delete [{}] could not execute directly and was queued", newDelete);
                    }
                } else {
                    for (SnapshotsInProgress.Entry completedSnapshot : completedWithCleanup) {
                        endSnapshot(completedSnapshot, newState.metadata(), repositoryData);
                    }
                }
            }
        }
    };
}

Example 4 with State

use of org.opensearch.cluster.SnapshotsInProgress.State in project OpenSearch by opensearch-project.

the class SnapshotsService method beginSnapshot.

/**
 * Starts snapshot.
 * <p>
 * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
 * Note: This method is only used in clusters that contain a node older than {@link #NO_REPO_INITIALIZE_VERSION} to ensure a backwards
 * compatible path for initializing the snapshot in the repository is executed.
 *
 * @param clusterState               cluster state
 * @param snapshot                   snapshot meta data
 * @param partial                    allow partial snapshots
 * @param userCreateSnapshotListener listener
 */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final List<String> indices, final Repository repository, final ActionListener<Snapshot> userCreateSnapshotListener) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() {

        boolean hadAbortedInitializations;

        @Override
        protected void doRun() {
            assert initializingSnapshots.contains(snapshot.snapshot());
            if (repository.isReadOnly()) {
                throw new RepositoryException(repository.getMetadata().name(), "cannot create snapshot in a readonly repository");
            }
            final String snapshotName = snapshot.snapshot().getSnapshotId().getName();
            final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
            repository.getRepositoryData(repositoryDataListener);
            repositoryDataListener.whenComplete(repositoryData -> {
                // check if the snapshot name already exists in the repository
                if (repositoryData.getSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
                    throw new InvalidSnapshotNameException(repository.getMetadata().name(), snapshotName, "snapshot with the same name already exists");
                }
                if (clusterState.nodes().getMinNodeVersion().onOrAfter(NO_REPO_INITIALIZE_VERSION) == false) {
                    // In mixed version clusters we initialize the snapshot in the repository so that in case of a master failover to an
                    // older version master node snapshot finalization (that assumes initializeSnapshot was called) produces a valid
                    // snapshot.
                    repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metadataForSnapshot(snapshot, clusterState.metadata()));
                }
                logger.info("snapshot [{}] started", snapshot.snapshot());
                final Version version = minCompatibleVersion(clusterState.nodes().getMinNodeVersion(), repositoryData, null);
                if (indices.isEmpty()) {
                    // No indices in this snapshot - we are done
                    userCreateSnapshotListener.onResponse(snapshot.snapshot());
                    endSnapshot(SnapshotsInProgress.startedEntry(snapshot.snapshot(), snapshot.includeGlobalState(), snapshot.partial(), Collections.emptyList(), Collections.emptyList(), threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), ImmutableOpenMap.of(), snapshot.userMetadata(), version), clusterState.metadata(), repositoryData);
                    return;
                }
                clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                        List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                        for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                            if (entry.snapshot().equals(snapshot.snapshot()) == false) {
                                entries.add(entry);
                                continue;
                            }
                            if (entry.state() == State.ABORTED) {
                                entries.add(entry);
                                assert entry.shards().isEmpty();
                                hadAbortedInitializations = true;
                            } else {
                                final List<IndexId> indexIds = repositoryData.resolveNewIndices(indices, Collections.emptyMap());
                                // Replace the snapshot that was just initialized
                                ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards = shards(snapshots, currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY), currentState.metadata(), currentState.routingTable(), indexIds, useShardGenerations(version), repositoryData, entry.repository());
                                if (!partial) {
                                    Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metadata());
                                    Set<String> missing = indicesWithMissingShards.v1();
                                    Set<String> closed = indicesWithMissingShards.v2();
                                    if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                        final StringBuilder failureMessage = new StringBuilder();
                                        if (missing.isEmpty() == false) {
                                            failureMessage.append("Indices don't have primary shards ");
                                            failureMessage.append(missing);
                                        }
                                        if (closed.isEmpty() == false) {
                                            if (failureMessage.length() > 0) {
                                                failureMessage.append("; ");
                                            }
                                            failureMessage.append("Indices are closed ");
                                            failureMessage.append(closed);
                                        }
                                        entries.add(new SnapshotsInProgress.Entry(entry, State.FAILED, indexIds, repositoryData.getGenId(), shards, version, failureMessage.toString()));
                                        continue;
                                    }
                                }
                                entries.add(new SnapshotsInProgress.Entry(entry, State.STARTED, indexIds, repositoryData.getGenId(), shards, version, null));
                            }
                        }
                        return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(unmodifiableList(entries))).build();
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        logger.warn(() -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                        removeFailedSnapshotFromClusterState(snapshot.snapshot(), e, null, new CleanupAfterErrorListener(userCreateSnapshotListener, e));
                    }

                    @Override
                    public void onNoLongerMaster(String source) {
                        // We are not longer a master - we shouldn't try to do any cleanup
                        // The new master will take care of it
                        logger.warn("[{}] failed to create snapshot - no longer a master", snapshot.snapshot().getSnapshotId());
                        userCreateSnapshotListener.onFailure(new SnapshotException(snapshot.snapshot(), "master changed during snapshot initialization"));
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                        // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                        // completion listener in this method. For the snapshot completion to work properly, the snapshot
                        // should still exist when listener is registered.
                        userCreateSnapshotListener.onResponse(snapshot.snapshot());
                        if (hadAbortedInitializations) {
                            final SnapshotsInProgress snapshotsInProgress = newState.custom(SnapshotsInProgress.TYPE);
                            assert snapshotsInProgress != null;
                            final SnapshotsInProgress.Entry entry = snapshotsInProgress.snapshot(snapshot.snapshot());
                            assert entry != null;
                            endSnapshot(entry, newState.metadata(), repositoryData);
                        } else {
                            endCompletedSnapshots(newState);
                        }
                    }
                });
            }, this::onFailure);
        }

        @Override
        public void onFailure(Exception e) {
            logger.warn(() -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
            removeFailedSnapshotFromClusterState(snapshot.snapshot(), e, null, new CleanupAfterErrorListener(userCreateSnapshotListener, e));
        }
    });
}

Example 5 with State

use of org.opensearch.cluster.SnapshotsInProgress.State in project OpenSearch by opensearch-project.

the class SnapshotsService method deleteSnapshots.

/**
 * Deletes snapshots from the repository. In-progress snapshots matched by the delete will be aborted before deleting them.
 *
 * @param request         delete snapshot request
 * @param listener        listener
 */
public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionListener<Void> listener) {
    final String[] snapshotNames = request.snapshots();
    final String repoName = request.repository();
    logger.info(() -> new ParameterizedMessage("deleting snapshots [{}] from repository [{}]", Strings.arrayToCommaDelimitedString(snapshotNames), repoName));
    final Repository repository = repositoriesService.repository(repoName);
    repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.NORMAL) {

        private Snapshot runningSnapshot;

        private ClusterStateUpdateTask deleteFromRepoTask;

        private boolean abortedDuringInit = false;

        private List<SnapshotId> outstandingDeletes;

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            final Version minNodeVersion = currentState.nodes().getMinNodeVersion();
            if (snapshotNames.length > 1 && minNodeVersion.before(MULTI_DELETE_VERSION)) {
                throw new IllegalArgumentException("Deleting multiple snapshots in a single request is only supported in version [ " + MULTI_DELETE_VERSION + "] but cluster contained node of version [" + currentState.nodes().getMinNodeVersion() + "]");
            }
            final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
            final List<SnapshotsInProgress.Entry> snapshotEntries = findInProgressSnapshots(snapshots, snapshotNames, repoName);
            final List<SnapshotId> snapshotIds = matchingSnapshotIds(snapshotEntries.stream().map(e -> e.snapshot().getSnapshotId()).collect(Collectors.toList()), repositoryData, snapshotNames, repoName);
            if (snapshotEntries.isEmpty() || minNodeVersion.onOrAfter(SnapshotsService.FULL_CONCURRENCY_VERSION)) {
                deleteFromRepoTask = createDeleteStateUpdate(snapshotIds, repoName, repositoryData, Priority.NORMAL, listener);
                return deleteFromRepoTask.execute(currentState);
            }
            assert snapshotEntries.size() == 1 : "Expected just a single running snapshot but saw " + snapshotEntries;
            final SnapshotsInProgress.Entry snapshotEntry = snapshotEntries.get(0);
            runningSnapshot = snapshotEntry.snapshot();
            final ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards;
            final State state = snapshotEntry.state();
            final String failure;
            outstandingDeletes = new ArrayList<>(snapshotIds);
            if (state != State.INIT) {
                // INIT state snapshots won't ever be physically written to the repository but all other states will end up in the repo
                outstandingDeletes.add(runningSnapshot.getSnapshotId());
            }
            if (state == State.INIT) {
                // snapshot is still initializing, mark it as aborted
                shards = snapshotEntry.shards();
                assert shards.isEmpty();
                failure = "Snapshot was aborted during initialization";
                abortedDuringInit = true;
            } else if (state == State.STARTED) {
                // snapshot is started - mark every non completed shard as aborted
                final SnapshotsInProgress.Entry abortedEntry = snapshotEntry.abort();
                shards = abortedEntry.shards();
                failure = abortedEntry.failure();
            } else {
                boolean hasUncompletedShards = false;
                // Cleanup in case a node gone missing and snapshot wasn't updated for some reason
                for (ObjectCursor<ShardSnapshotStatus> shardStatus : snapshotEntry.shards().values()) {
                    // Check if we still have shard running on existing nodes
                    if (shardStatus.value.state().completed() == false && shardStatus.value.nodeId() != null && currentState.nodes().get(shardStatus.value.nodeId()) != null) {
                        hasUncompletedShards = true;
                        break;
                    }
                }
                if (hasUncompletedShards) {
                    // snapshot is being finalized - wait for shards to complete finalization process
                    logger.debug("trying to delete completed snapshot - should wait for shards to finalize on all nodes");
                    return currentState;
                } else {
                    // no shards to wait for but a node is gone - this is the only case
                    // where we force to finish the snapshot
                    logger.debug("trying to delete completed snapshot with no finalizing shards - can delete immediately");
                    shards = snapshotEntry.shards();
                }
                failure = snapshotEntry.failure();
            }
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(snapshots.entries().stream().filter(existing -> abortedDuringInit == false || existing.equals(snapshotEntry) == false).map(existing -> {
                if (existing.equals(snapshotEntry)) {
                    return snapshotEntry.fail(shards, State.ABORTED, failure);
                }
                return existing;
            }).collect(Collectors.toList()))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            listener.onFailure(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            if (deleteFromRepoTask != null) {
                assert outstandingDeletes == null : "Shouldn't have outstanding deletes after already starting delete task";
                deleteFromRepoTask.clusterStateProcessed(source, oldState, newState);
                return;
            }
            if (abortedDuringInit) {
                // BwC Path where we removed an outdated INIT state snapshot from the cluster state
                logger.info("Successfully aborted snapshot [{}]", runningSnapshot);
                if (outstandingDeletes.isEmpty()) {
                    listener.onResponse(null);
                } else {
                    clusterService.submitStateUpdateTask("delete snapshot", createDeleteStateUpdate(outstandingDeletes, repoName, repositoryData, Priority.IMMEDIATE, listener));
                }
                return;
            }
            logger.trace("adding snapshot completion listener to wait for deleted snapshot to finish");
            addListener(runningSnapshot, ActionListener.wrap(result -> {
                logger.debug("deleted snapshot completed - deleting files");
                clusterService.submitStateUpdateTask("delete snapshot", createDeleteStateUpdate(outstandingDeletes, repoName, result.v1(), Priority.IMMEDIATE, listener));
            }, e -> {
                if (ExceptionsHelper.unwrap(e, NotMasterException.class, FailedToCommitClusterStateException.class) != null) {
                    logger.warn("master failover before deleted snapshot could complete", e);
                    // Just pass the exception to the transport handler as is so it is retried on the new master
                    listener.onFailure(e);
                } else {
                    logger.warn("deleted snapshot failed", e);
                    listener.onFailure(new SnapshotMissingException(runningSnapshot.getRepository(), runningSnapshot.getSnapshotId(), e));
                }
            }));
        }

        @Override
        public TimeValue timeout() {
            return request.masterNodeTimeout();
        }
    }, "delete snapshot", listener::onFailure);
}