Examples with AbortedSnapshotException org.opensearch.snapshots.AbortedSnapshotException used on opensource projects

Example 1 with AbortedSnapshotException

use of org.opensearch.snapshots.AbortedSnapshotException in project OpenSearch by opensearch-project.

the class BlobStoreRepository method snapshotFile.

/**
 * Snapshot individual file
 * @param fileInfo file to be snapshotted
 */
private void snapshotFile(BlobStoreIndexShardSnapshot.FileInfo fileInfo, IndexId indexId, ShardId shardId, SnapshotId snapshotId, IndexShardSnapshotStatus snapshotStatus, Store store) throws IOException {
    final BlobContainer shardContainer = shardContainer(indexId, shardId);
    final String file = fileInfo.physicalName();
    try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
        for (int i = 0; i < fileInfo.numberOfParts(); i++) {
            final long partBytes = fileInfo.partBytes(i);
            // Make reads abortable by mutating the snapshotStatus object
            final InputStream inputStream = new FilterInputStream(maybeRateLimitSnapshots(new InputStreamIndexInput(indexInput, partBytes))) {

                @Override
                public int read() throws IOException {
                    checkAborted();
                    return super.read();
                }

                @Override
                public int read(byte[] b, int off, int len) throws IOException {
                    checkAborted();
                    return super.read(b, off, len);
                }

                private void checkAborted() {
                    if (snapshotStatus.isAborted()) {
                        logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileInfo.physicalName());
                        throw new AbortedSnapshotException();
                    }
                }
            };
            final String partName = fileInfo.partName(i);
            logger.trace(() -> new ParameterizedMessage("[{}] Writing [{}] to [{}]", metadata.name(), partName, shardContainer.path()));
            shardContainer.writeBlob(partName, inputStream, partBytes, false);
        }
        Store.verify(indexInput);
        snapshotStatus.addProcessedFile(fileInfo.length());
    } catch (Exception t) {
        failStoreIfCorrupted(store, t);
        snapshotStatus.addProcessedFile(0);
        throw t;
    }
}

Example 2 with AbortedSnapshotException

use of org.opensearch.snapshots.AbortedSnapshotException in project OpenSearch by opensearch-project.

the class BlobStoreRepository method snapshotShard.

@Override
public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
    if (isReadOnly()) {
        listener.onFailure(new RepositoryException(metadata.name(), "cannot snapshot shard on a readonly repository"));
        return;
    }
    final ShardId shardId = store.shardId();
    final long startTime = threadPool.absoluteTimeInMillis();
    try {
        final String generation = snapshotStatus.generation();
        logger.debug("[{}] [{}] snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation);
        final BlobContainer shardContainer = shardContainer(indexId, shardId);
        final Set<String> blobs;
        if (generation == null) {
            try {
                blobs = shardContainer.listBlobsByPrefix(INDEX_FILE_PREFIX).keySet();
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
            }
        } else {
            blobs = Collections.singleton(INDEX_FILE_PREFIX + generation);
        }
        Tuple<BlobStoreIndexShardSnapshots, String> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, generation);
        BlobStoreIndexShardSnapshots snapshots = tuple.v1();
        String fileListGeneration = tuple.v2();
        if (snapshots.snapshots().stream().anyMatch(sf -> sf.snapshot().equals(snapshotId.getName()))) {
            throw new IndexShardSnapshotFailedException(shardId, "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting");
        }
        // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository
        final List<BlobStoreIndexShardSnapshot.FileInfo> filesFromSegmentInfos = Optional.ofNullable(shardStateIdentifier).map(id -> {
            for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) {
                if (id.equals(snapshotFileSet.shardStateIdentifier())) {
                    return snapshotFileSet.indexFiles();
                }
            }
            return null;
        }).orElse(null);
        final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles;
        int indexIncrementalFileCount = 0;
        int indexTotalNumberOfFiles = 0;
        long indexIncrementalSize = 0;
        long indexTotalFileSize = 0;
        final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
        // in the commit with files already in the repository
        if (filesFromSegmentInfos == null) {
            indexCommitPointFiles = new ArrayList<>();
            final Collection<String> fileNames;
            final Store.MetadataSnapshot metadataFromStore;
            try (Releasable ignored = incrementStoreRef(store, snapshotStatus, shardId)) {
                // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
                try {
                    logger.trace("[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
                    metadataFromStore = store.getMetadata(snapshotIndexCommit);
                    fileNames = snapshotIndexCommit.getFileNames();
                } catch (IOException e) {
                    throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
                }
            }
            for (String fileName : fileNames) {
                if (snapshotStatus.isAborted()) {
                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
                    throw new AbortedSnapshotException();
                }
                logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
                final StoreFileMetadata md = metadataFromStore.get(fileName);
                BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
                List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
                if (filesInfo != null) {
                    for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
                        if (fileInfo.isSame(md)) {
                            // a commit point file with the same name, size and checksum was already copied to repository
                            // we will reuse it for this snapshot
                            existingFileInfo = fileInfo;
                            break;
                        }
                    }
                }
                // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
                // directly in the shard level metadata in this case
                final boolean needsWrite = md.hashEqualsContents() == false;
                indexTotalFileSize += md.length();
                indexTotalNumberOfFiles++;
                if (existingFileInfo == null) {
                    indexIncrementalFileCount++;
                    indexIncrementalSize += md.length();
                    // create a new FileInfo
                    BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo((needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(), md, chunkSize());
                    indexCommitPointFiles.add(snapshotFileInfo);
                    if (needsWrite) {
                        filesToSnapshot.add(snapshotFileInfo);
                    }
                    assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store);
                } else {
                    indexCommitPointFiles.add(existingFileInfo);
                }
            }
        } else {
            for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesFromSegmentInfos) {
                indexTotalNumberOfFiles++;
                indexTotalFileSize += fileInfo.length();
            }
            indexCommitPointFiles = filesFromSegmentInfos;
        }
        snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileSize);
        final String indexGeneration;
        final boolean writeShardGens = SnapshotsService.useShardGenerations(repositoryMetaVersion);
        // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
        List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
        newSnapshotsList.add(new SnapshotFiles(snapshotId.getName(), indexCommitPointFiles, shardStateIdentifier));
        for (SnapshotFiles point : snapshots) {
            newSnapshotsList.add(point);
        }
        final BlobStoreIndexShardSnapshots updatedBlobStoreIndexShardSnapshots = new BlobStoreIndexShardSnapshots(newSnapshotsList);
        final Runnable afterWriteSnapBlob;
        if (writeShardGens) {
            // When using shard generations we can safely write the index-${uuid} blob before writing out any of the actual data
            // for this shard since the uuid named blob will simply not be referenced in case of error and thus we will never
            // reference a generation that has not had all its files fully upload.
            indexGeneration = UUIDs.randomBase64UUID();
            try {
                INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedBlobStoreIndexShardSnapshots, shardContainer, indexGeneration, compress);
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to write shard level snapshot metadata for [" + snapshotId + "] to [" + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration) + "]", e);
            }
            afterWriteSnapBlob = () -> {
            };
        } else {
            // When not using shard generations we can only write the index-${N} blob after all other work for this shard has
            // completed.
            // Also, in case of numeric shard generations the data node has to take care of deleting old shard generations.
            final long newGen = Long.parseLong(fileListGeneration) + 1;
            indexGeneration = Long.toString(newGen);
            // Delete all previous index-N blobs
            final List<String> blobsToDelete = blobs.stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).collect(Collectors.toList());
            assert blobsToDelete.stream().mapToLong(b -> Long.parseLong(b.replaceFirst(SNAPSHOT_INDEX_PREFIX, ""))).max().orElse(-1L) < Long.parseLong(indexGeneration) : "Tried to delete an index-N blob newer than the current generation [" + indexGeneration + "] when deleting index-N blobs " + blobsToDelete;
            afterWriteSnapBlob = () -> {
                try {
                    writeShardIndexBlobAtomic(shardContainer, newGen, updatedBlobStoreIndexShardSnapshots);
                } catch (IOException e) {
                    throw new IndexShardSnapshotFailedException(shardId, "Failed to finalize snapshot creation [" + snapshotId + "] with shard index [" + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration) + "]", e);
                }
                try {
                    deleteFromContainer(shardContainer, blobsToDelete);
                } catch (IOException e) {
                    logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete old index-N blobs during finalization", snapshotId, shardId), e);
                }
            };
        }
        final StepListener<Collection<Void>> allFilesUploadedListener = new StepListener<>();
        allFilesUploadedListener.whenComplete(v -> {
            final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.moveToFinalize(snapshotIndexCommit.getGeneration());
            // now create and write the commit point
            logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
            try {
                INDEX_SHARD_SNAPSHOT_FORMAT.write(new BlobStoreIndexShardSnapshot(snapshotId.getName(), lastSnapshotStatus.getIndexVersion(), indexCommitPointFiles, lastSnapshotStatus.getStartTime(), threadPool.absoluteTimeInMillis() - lastSnapshotStatus.getStartTime(), lastSnapshotStatus.getIncrementalFileCount(), lastSnapshotStatus.getIncrementalSize()), shardContainer, snapshotId.getUUID(), compress);
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
            }
            afterWriteSnapBlob.run();
            snapshotStatus.moveToDone(threadPool.absoluteTimeInMillis(), indexGeneration);
            listener.onResponse(indexGeneration);
        }, listener::onFailure);
        if (indexIncrementalFileCount == 0) {
            allFilesUploadedListener.onResponse(Collections.emptyList());
            return;
        }
        final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        // Start as many workers as fit into the snapshot pool at once at the most
        final int workers = Math.min(threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(), indexIncrementalFileCount);
        final ActionListener<Void> filesListener = fileQueueListener(filesToSnapshot, workers, allFilesUploadedListener);
        for (int i = 0; i < workers; ++i) {
            executeOneFileSnapshot(store, snapshotId, indexId, snapshotStatus, filesToSnapshot, executor, filesListener);
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}