Examples with SnapshotsService org.opensearch.snapshots.SnapshotsService used on opensource projects

Example 1 with SnapshotsService

use of org.opensearch.snapshots.SnapshotsService in project OpenSearch by opensearch-project.

the class BlobStoreRepository method finalizeSnapshot.

@Override
public void finalizeSnapshot(final ShardGenerations shardGenerations, final long repositoryStateId, final Metadata clusterMetadata, SnapshotInfo snapshotInfo, Version repositoryMetaVersion, Function<ClusterState, ClusterState> stateTransformer, final ActionListener<RepositoryData> listener) {
    assert repositoryStateId > RepositoryData.UNKNOWN_REPO_GEN : "Must finalize based on a valid repository generation but received [" + repositoryStateId + "]";
    final Collection<IndexId> indices = shardGenerations.indices();
    final SnapshotId snapshotId = snapshotInfo.snapshotId();
    // Once we are done writing the updated index-N blob we remove the now unreferenced index-${uuid} blobs in each shard
    // directory if all nodes are at least at version SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION
    // If there are older version nodes in the cluster, we don't need to run this cleanup as it will have already happened
    // when writing the index-${N} to each shard directory.
    final boolean writeShardGens = SnapshotsService.useShardGenerations(repositoryMetaVersion);
    final Consumer<Exception> onUpdateFailure = e -> listener.onFailure(new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e));
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final boolean writeIndexGens = SnapshotsService.useIndexGenerations(repositoryMetaVersion);
    final StepListener<RepositoryData> repoDataListener = new StepListener<>();
    getRepositoryData(repoDataListener);
    repoDataListener.whenComplete(existingRepositoryData -> {
        final Map<IndexId, String> indexMetas;
        final Map<String, String> indexMetaIdentifiers;
        if (writeIndexGens) {
            indexMetaIdentifiers = ConcurrentCollections.newConcurrentMap();
            indexMetas = ConcurrentCollections.newConcurrentMap();
        } else {
            indexMetas = null;
            indexMetaIdentifiers = null;
        }
        final ActionListener<Void> allMetaListener = new GroupedActionListener<>(ActionListener.wrap(v -> {
            final RepositoryData updatedRepositoryData = existingRepositoryData.addSnapshot(snapshotId, snapshotInfo.state(), Version.CURRENT, shardGenerations, indexMetas, indexMetaIdentifiers);
            writeIndexGen(updatedRepositoryData, repositoryStateId, repositoryMetaVersion, stateTransformer, ActionListener.wrap(newRepoData -> {
                if (writeShardGens) {
                    cleanupOldShardGens(existingRepositoryData, updatedRepositoryData);
                }
                listener.onResponse(newRepoData);
            }, onUpdateFailure));
        }, onUpdateFailure), 2 + indices.size());
        // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method will
        // mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version of the
        // index or global metadata will be compatible with the segments written in this snapshot as well.
        // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
        // that decrements the generation it points at
        // Write Global MetaData
        executor.execute(ActionRunnable.run(allMetaListener, () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)));
        // write the index metadata for each index in the snapshot
        for (IndexId index : indices) {
            executor.execute(ActionRunnable.run(allMetaListener, () -> {
                final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
                if (writeIndexGens) {
                    final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
                    String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
                    if (metaUUID == null) {
                        // We don't yet have this version of the metadata so we write it
                        metaUUID = UUIDs.base64UUID();
                        INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
                        indexMetaIdentifiers.put(identifiers, metaUUID);
                    }
                    indexMetas.put(index, identifiers);
                } else {
                    INDEX_METADATA_FORMAT.write(clusterMetadata.index(index.getName()), indexContainer(index), snapshotId.getUUID(), compress);
                }
            }));
        }
        executor.execute(ActionRunnable.run(allMetaListener, () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)));
    }, onUpdateFailure);
}