Search in sources :

Example 1 with IndexShardSnapshotFailedException

use of org.opensearch.index.snapshots.IndexShardSnapshotFailedException in project OpenSearch by opensearch-project.

the class SnapshotShardsService method snapshot.

/**
 * Creates shard snapshot
 *
 * @param snapshot       snapshot
 * @param snapshotStatus snapshot status
 */
private void snapshot(final ShardId shardId, final Snapshot snapshot, final IndexId indexId, final Map<String, Object> userMetadata, final IndexShardSnapshotStatus snapshotStatus, Version version, ActionListener<String> listener) {
    try {
        final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id());
        if (indexShard.routingEntry().primary() == false) {
            throw new IndexShardSnapshotFailedException(shardId, "snapshot should be performed only on primary");
        }
        if (indexShard.routingEntry().relocating()) {
            // do not snapshot when in the process of relocation of primaries so we won't get conflicts
            throw new IndexShardSnapshotFailedException(shardId, "cannot snapshot while relocating");
        }
        final IndexShardState indexShardState = indexShard.state();
        if (indexShardState == IndexShardState.CREATED || indexShardState == IndexShardState.RECOVERING) {
            // shard has just been created, or still recovering
            throw new IndexShardSnapshotFailedException(shardId, "shard didn't fully recover yet");
        }
        final Repository repository = repositoriesService.repository(snapshot.getRepository());
        Engine.IndexCommitRef snapshotRef = null;
        try {
            // we flush first to make sure we get the latest writes snapshotted
            snapshotRef = indexShard.acquireLastIndexCommit(true);
            final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit();
            repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, snapshotRef.getIndexCommit(), getShardStateId(indexShard, snapshotIndexCommit), snapshotStatus, version, userMetadata, ActionListener.runBefore(listener, snapshotRef::close));
        } catch (Exception e) {
            IOUtils.close(snapshotRef);
            throw e;
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
Also used : Repository(org.opensearch.repositories.Repository) IndexShard(org.opensearch.index.shard.IndexShard) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) IndexShardState(org.opensearch.index.shard.IndexShardState) Engine(org.opensearch.index.engine.Engine) IndexCommit(org.apache.lucene.index.IndexCommit) IOException(java.io.IOException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) TransportException(org.opensearch.transport.TransportException)

Example 2 with IndexShardSnapshotFailedException

use of org.opensearch.index.snapshots.IndexShardSnapshotFailedException in project OpenSearch by opensearch-project.

the class BlobStoreRepository method snapshotShard.

@Override
public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
    if (isReadOnly()) {
        listener.onFailure(new RepositoryException(metadata.name(), "cannot snapshot shard on a readonly repository"));
        return;
    }
    final ShardId shardId = store.shardId();
    final long startTime = threadPool.absoluteTimeInMillis();
    try {
        final String generation = snapshotStatus.generation();
        logger.debug("[{}] [{}] snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation);
        final BlobContainer shardContainer = shardContainer(indexId, shardId);
        final Set<String> blobs;
        if (generation == null) {
            try {
                blobs = shardContainer.listBlobsByPrefix(INDEX_FILE_PREFIX).keySet();
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
            }
        } else {
            blobs = Collections.singleton(INDEX_FILE_PREFIX + generation);
        }
        Tuple<BlobStoreIndexShardSnapshots, String> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, generation);
        BlobStoreIndexShardSnapshots snapshots = tuple.v1();
        String fileListGeneration = tuple.v2();
        if (snapshots.snapshots().stream().anyMatch(sf -> sf.snapshot().equals(snapshotId.getName()))) {
            throw new IndexShardSnapshotFailedException(shardId, "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting");
        }
        // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository
        final List<BlobStoreIndexShardSnapshot.FileInfo> filesFromSegmentInfos = Optional.ofNullable(shardStateIdentifier).map(id -> {
            for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) {
                if (id.equals(snapshotFileSet.shardStateIdentifier())) {
                    return snapshotFileSet.indexFiles();
                }
            }
            return null;
        }).orElse(null);
        final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles;
        int indexIncrementalFileCount = 0;
        int indexTotalNumberOfFiles = 0;
        long indexIncrementalSize = 0;
        long indexTotalFileSize = 0;
        final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
        // in the commit with files already in the repository
        if (filesFromSegmentInfos == null) {
            indexCommitPointFiles = new ArrayList<>();
            final Collection<String> fileNames;
            final Store.MetadataSnapshot metadataFromStore;
            try (Releasable ignored = incrementStoreRef(store, snapshotStatus, shardId)) {
                // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
                try {
                    logger.trace("[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
                    metadataFromStore = store.getMetadata(snapshotIndexCommit);
                    fileNames = snapshotIndexCommit.getFileNames();
                } catch (IOException e) {
                    throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
                }
            }
            for (String fileName : fileNames) {
                if (snapshotStatus.isAborted()) {
                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
                    throw new AbortedSnapshotException();
                }
                logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
                final StoreFileMetadata md = metadataFromStore.get(fileName);
                BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
                List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
                if (filesInfo != null) {
                    for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
                        if (fileInfo.isSame(md)) {
                            // a commit point file with the same name, size and checksum was already copied to repository
                            // we will reuse it for this snapshot
                            existingFileInfo = fileInfo;
                            break;
                        }
                    }
                }
                // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
                // directly in the shard level metadata in this case
                final boolean needsWrite = md.hashEqualsContents() == false;
                indexTotalFileSize += md.length();
                indexTotalNumberOfFiles++;
                if (existingFileInfo == null) {
                    indexIncrementalFileCount++;
                    indexIncrementalSize += md.length();
                    // create a new FileInfo
                    BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo((needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(), md, chunkSize());
                    indexCommitPointFiles.add(snapshotFileInfo);
                    if (needsWrite) {
                        filesToSnapshot.add(snapshotFileInfo);
                    }
                    assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store);
                } else {
                    indexCommitPointFiles.add(existingFileInfo);
                }
            }
        } else {
            for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesFromSegmentInfos) {
                indexTotalNumberOfFiles++;
                indexTotalFileSize += fileInfo.length();
            }
            indexCommitPointFiles = filesFromSegmentInfos;
        }
        snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileSize);
        final String indexGeneration;
        final boolean writeShardGens = SnapshotsService.useShardGenerations(repositoryMetaVersion);
        // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
        List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
        newSnapshotsList.add(new SnapshotFiles(snapshotId.getName(), indexCommitPointFiles, shardStateIdentifier));
        for (SnapshotFiles point : snapshots) {
            newSnapshotsList.add(point);
        }
        final BlobStoreIndexShardSnapshots updatedBlobStoreIndexShardSnapshots = new BlobStoreIndexShardSnapshots(newSnapshotsList);
        final Runnable afterWriteSnapBlob;
        if (writeShardGens) {
            // When using shard generations we can safely write the index-${uuid} blob before writing out any of the actual data
            // for this shard since the uuid named blob will simply not be referenced in case of error and thus we will never
            // reference a generation that has not had all its files fully upload.
            indexGeneration = UUIDs.randomBase64UUID();
            try {
                INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedBlobStoreIndexShardSnapshots, shardContainer, indexGeneration, compress);
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to write shard level snapshot metadata for [" + snapshotId + "] to [" + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration) + "]", e);
            }
            afterWriteSnapBlob = () -> {
            };
        } else {
            // When not using shard generations we can only write the index-${N} blob after all other work for this shard has
            // completed.
            // Also, in case of numeric shard generations the data node has to take care of deleting old shard generations.
            final long newGen = Long.parseLong(fileListGeneration) + 1;
            indexGeneration = Long.toString(newGen);
            // Delete all previous index-N blobs
            final List<String> blobsToDelete = blobs.stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).collect(Collectors.toList());
            assert blobsToDelete.stream().mapToLong(b -> Long.parseLong(b.replaceFirst(SNAPSHOT_INDEX_PREFIX, ""))).max().orElse(-1L) < Long.parseLong(indexGeneration) : "Tried to delete an index-N blob newer than the current generation [" + indexGeneration + "] when deleting index-N blobs " + blobsToDelete;
            afterWriteSnapBlob = () -> {
                try {
                    writeShardIndexBlobAtomic(shardContainer, newGen, updatedBlobStoreIndexShardSnapshots);
                } catch (IOException e) {
                    throw new IndexShardSnapshotFailedException(shardId, "Failed to finalize snapshot creation [" + snapshotId + "] with shard index [" + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration) + "]", e);
                }
                try {
                    deleteFromContainer(shardContainer, blobsToDelete);
                } catch (IOException e) {
                    logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete old index-N blobs during finalization", snapshotId, shardId), e);
                }
            };
        }
        final StepListener<Collection<Void>> allFilesUploadedListener = new StepListener<>();
        allFilesUploadedListener.whenComplete(v -> {
            final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.moveToFinalize(snapshotIndexCommit.getGeneration());
            // now create and write the commit point
            logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
            try {
                INDEX_SHARD_SNAPSHOT_FORMAT.write(new BlobStoreIndexShardSnapshot(snapshotId.getName(), lastSnapshotStatus.getIndexVersion(), indexCommitPointFiles, lastSnapshotStatus.getStartTime(), threadPool.absoluteTimeInMillis() - lastSnapshotStatus.getStartTime(), lastSnapshotStatus.getIncrementalFileCount(), lastSnapshotStatus.getIncrementalSize()), shardContainer, snapshotId.getUUID(), compress);
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
            }
            afterWriteSnapBlob.run();
            snapshotStatus.moveToDone(threadPool.absoluteTimeInMillis(), indexGeneration);
            listener.onResponse(indexGeneration);
        }, listener::onFailure);
        if (indexIncrementalFileCount == 0) {
            allFilesUploadedListener.onResponse(Collections.emptyList());
            return;
        }
        final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        // Start as many workers as fit into the snapshot pool at once at the most
        final int workers = Math.min(threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(), indexIncrementalFileCount);
        final ActionListener<Void> filesListener = fileQueueListener(filesToSnapshot, workers, allFilesUploadedListener);
        for (int i = 0; i < workers; ++i) {
            executeOneFileSnapshot(store, snapshotId, indexId, snapshotStatus, filesToSnapshot, executor, filesListener);
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
Also used : Metadata(org.opensearch.cluster.metadata.Metadata) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) Strings(org.opensearch.common.Strings) AbortedSnapshotException(org.opensearch.snapshots.AbortedSnapshotException) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) RecoveryState(org.opensearch.indices.recovery.RecoveryState) Map(java.util.Map) Lucene(org.opensearch.common.lucene.Lucene) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) BlobStoreIndexShardSnapshot(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot) BlockingQueue(java.util.concurrent.BlockingQueue) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) Logger(org.apache.logging.log4j.Logger) RepositoryOperation(org.opensearch.repositories.RepositoryOperation) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) BytesArray(org.opensearch.common.bytes.BytesArray) BlobStoreIndexShardSnapshots(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots) FsBlobContainer(org.opensearch.common.blobstore.fs.FsBlobContainer) StepListener(org.opensearch.action.StepListener) XContentType(org.opensearch.common.xcontent.XContentType) IndexCommit(org.apache.lucene.index.IndexCommit) ThreadPool(org.opensearch.threadpool.ThreadPool) BlobContainer(org.opensearch.common.blobstore.BlobContainer) Releasable(org.opensearch.common.lease.Releasable) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) Numbers(org.opensearch.common.Numbers) SlicedInputStream(org.opensearch.index.snapshots.blobstore.SlicedInputStream) SnapshotException(org.opensearch.snapshots.SnapshotException) Streams(org.opensearch.common.io.Streams) CompressorFactory(org.opensearch.common.compress.CompressorFactory) RepositoryVerificationException(org.opensearch.repositories.RepositoryVerificationException) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) InputStreamIndexInput(org.opensearch.common.lucene.store.InputStreamIndexInput) LongStream(java.util.stream.LongStream) IndexInput(org.apache.lucene.store.IndexInput) SetOnce(org.apache.lucene.util.SetOnce) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) IOException(java.io.IOException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) NotXContentException(org.opensearch.common.compress.NotXContentException) AtomicLong(java.util.concurrent.atomic.AtomicLong) RepositoryCleanupResult(org.opensearch.repositories.RepositoryCleanupResult) BlobPath(org.opensearch.common.blobstore.BlobPath) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) CounterMetric(org.opensearch.common.metrics.CounterMetric) ShardGenerations(org.opensearch.repositories.ShardGenerations) NoSuchFileException(java.nio.file.NoSuchFileException) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) SnapshotCreationException(org.opensearch.snapshots.SnapshotCreationException) ByteSizeUnit(org.opensearch.common.unit.ByteSizeUnit) SnapshotFiles(org.opensearch.index.snapshots.blobstore.SnapshotFiles) SnapshotsService(org.opensearch.snapshots.SnapshotsService) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) MapperService(org.opensearch.index.mapper.MapperService) IndexId(org.opensearch.repositories.IndexId) XContentFactory(org.opensearch.common.xcontent.XContentFactory) RepositoryStats(org.opensearch.repositories.RepositoryStats) BlobMetadata(org.opensearch.common.blobstore.BlobMetadata) RecoverySettings(org.opensearch.indices.recovery.RecoverySettings) RepositoryException(org.opensearch.repositories.RepositoryException) FileInfo.canonicalName(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName) BytesRef(org.apache.lucene.util.BytesRef) SnapshotId(org.opensearch.snapshots.SnapshotId) Collection(java.util.Collection) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Store(org.opensearch.index.store.Store) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) BlobStore(org.opensearch.common.blobstore.BlobStore) List(java.util.List) Optional(java.util.Optional) BytesReference(org.opensearch.common.bytes.BytesReference) RateLimitingInputStream(org.opensearch.index.snapshots.blobstore.RateLimitingInputStream) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FilterInputStream(java.io.FilterInputStream) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) IndexMetaDataGenerations(org.opensearch.repositories.IndexMetaDataGenerations) UUIDs(org.opensearch.common.UUIDs) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) IndexOutput(org.apache.lucene.store.IndexOutput) IndexShardRestoreFailedException(org.opensearch.index.snapshots.IndexShardRestoreFailedException) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) DeleteResult(org.opensearch.common.blobstore.DeleteResult) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) RateLimiter(org.apache.lucene.store.RateLimiter) InputStream(java.io.InputStream) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) BlobStoreIndexShardSnapshot(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot) ArrayList(java.util.ArrayList) Store(org.opensearch.index.store.Store) BlobStore(org.opensearch.common.blobstore.BlobStore) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardId(org.opensearch.index.shard.ShardId) BlobStoreIndexShardSnapshots(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots) SnapshotFiles(org.opensearch.index.snapshots.blobstore.SnapshotFiles) Executor(java.util.concurrent.Executor) AbortedSnapshotException(org.opensearch.snapshots.AbortedSnapshotException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) RepositoryException(org.opensearch.repositories.RepositoryException) IOException(java.io.IOException) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) AbortedSnapshotException(org.opensearch.snapshots.AbortedSnapshotException) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) RepositoryVerificationException(org.opensearch.repositories.RepositoryVerificationException) IOException(java.io.IOException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) NotXContentException(org.opensearch.common.compress.NotXContentException) NoSuchFileException(java.nio.file.NoSuchFileException) SnapshotCreationException(org.opensearch.snapshots.SnapshotCreationException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRestoreFailedException(org.opensearch.index.snapshots.IndexShardRestoreFailedException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) FsBlobContainer(org.opensearch.common.blobstore.fs.FsBlobContainer) BlobContainer(org.opensearch.common.blobstore.BlobContainer) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ActionRunnable(org.opensearch.action.ActionRunnable) Collection(java.util.Collection) Releasable(org.opensearch.common.lease.Releasable) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) StepListener(org.opensearch.action.StepListener)

Example 3 with IndexShardSnapshotFailedException

use of org.opensearch.index.snapshots.IndexShardSnapshotFailedException in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryRestoreTests method testSnapshotWithConflictingName.

public void testSnapshotWithConflictingName() throws Exception {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, "_doc", Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        final String shardGen = snapshotShard(shard, snapshot, repository);
        assertNotNull(shardGen);
        final Snapshot snapshotWithSameName = new Snapshot(repository.getMetadata().name(), new SnapshotId(snapshot.getSnapshotId().getName(), "_uuid2"));
        final ShardGenerations shardGenerations = ShardGenerations.builder().put(indexId, 0, shardGen).build();
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(shardGenerations, RepositoryData.EMPTY_REPO_GEN, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository));
        assertThat(isfe.getMessage(), containsString("Duplicate snapshot name"));
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}
Also used : IndexId(org.opensearch.repositories.IndexId) ShardGenerations(org.opensearch.repositories.ShardGenerations) IndexShard(org.opensearch.index.shard.IndexShard) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) Matchers.containsString(org.hamcrest.Matchers.containsString) IOException(java.io.IOException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) RepositoryData(org.opensearch.repositories.RepositoryData) ShardId(org.opensearch.index.shard.ShardId) Snapshot(org.opensearch.snapshots.Snapshot) SnapshotId(org.opensearch.snapshots.SnapshotId) Repository(org.opensearch.repositories.Repository) FsRepository(org.opensearch.repositories.fs.FsRepository) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo)

Aggregations

IOException (java.io.IOException)3 IndexShardSnapshotFailedException (org.opensearch.index.snapshots.IndexShardSnapshotFailedException)3 Repository (org.opensearch.repositories.Repository)3 IndexCommit (org.apache.lucene.index.IndexCommit)2 ShardId (org.opensearch.index.shard.ShardId)2 IndexId (org.opensearch.repositories.IndexId)2 RepositoryData (org.opensearch.repositories.RepositoryData)2 ShardGenerations (org.opensearch.repositories.ShardGenerations)2 SnapshotId (org.opensearch.snapshots.SnapshotId)2 SnapshotInfo (org.opensearch.snapshots.SnapshotInfo)2 FilterInputStream (java.io.FilterInputStream)1 InputStream (java.io.InputStream)1 NoSuchFileException (java.nio.file.NoSuchFileException)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Set (java.util.Set)1