Examples with SnapshotFiles org.elasticsearch.index.snapshots.blobstore.SnapshotFiles used on opensource projects

Example 1 with SnapshotFiles

use of org.elasticsearch.index.snapshots.blobstore.SnapshotFiles in project crate by crate.

the class FileRestoreContext method restore.

/**
 * Performs restore operation
 */
public void restore(SnapshotFiles snapshotFiles, Store store, ActionListener<Void> listener) {
    store.incRef();
    try {
        LOGGER.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId);
        Store.MetadataSnapshot recoveryTargetMetadata;
        try {
            // this will throw an IOException if the store has no segments infos file. The
            // store can still have existing files but they will be deleted just before being
            // restored.
            recoveryTargetMetadata = store.getMetadata(null, true);
        } catch (org.apache.lucene.index.IndexNotFoundException e) {
            // happens when restore to an empty shard, not a big deal
            LOGGER.trace("[{}] [{}] restoring from to an empty shard", shardId, snapshotId);
            recoveryTargetMetadata = Store.MetadataSnapshot.EMPTY;
        } catch (IOException e) {
            LOGGER.warn(new ParameterizedMessage("[{}] [{}] Can't read metadata from store, will not reuse local files during restore", shardId, snapshotId), e);
            recoveryTargetMetadata = Store.MetadataSnapshot.EMPTY;
        }
        final List<BlobStoreIndexShardSnapshot.FileInfo> filesToRecover = new ArrayList<>();
        final Map<String, StoreFileMetadata> snapshotMetadata = new HashMap<>();
        final Map<String, BlobStoreIndexShardSnapshot.FileInfo> fileInfos = new HashMap<>();
        for (final BlobStoreIndexShardSnapshot.FileInfo fileInfo : snapshotFiles.indexFiles()) {
            snapshotMetadata.put(fileInfo.metadata().name(), fileInfo.metadata());
            fileInfos.put(fileInfo.metadata().name(), fileInfo);
        }
        final Store.MetadataSnapshot sourceMetadata = new Store.MetadataSnapshot(unmodifiableMap(snapshotMetadata), emptyMap(), 0);
        final StoreFileMetadata restoredSegmentsFile = sourceMetadata.getSegmentsFile();
        if (restoredSegmentsFile == null) {
            throw new IndexShardRestoreFailedException(shardId, "Snapshot has no segments file");
        }
        final Store.RecoveryDiff diff = sourceMetadata.recoveryDiff(recoveryTargetMetadata);
        for (StoreFileMetadata md : diff.identical) {
            BlobStoreIndexShardSnapshot.FileInfo fileInfo = fileInfos.get(md.name());
            recoveryState.getIndex().addFileDetail(fileInfo.physicalName(), fileInfo.length(), true);
            if (LOGGER.isTraceEnabled()) {
                LOGGER.trace("[{}] [{}] not_recovering file [{}] from [{}], exists in local store and is same", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
            }
        }
        for (StoreFileMetadata md : concat(diff)) {
            BlobStoreIndexShardSnapshot.FileInfo fileInfo = fileInfos.get(md.name());
            filesToRecover.add(fileInfo);
            recoveryState.getIndex().addFileDetail(fileInfo.physicalName(), fileInfo.length(), false);
            if (LOGGER.isTraceEnabled()) {
                LOGGER.trace("[{}] [{}] recovering [{}] from [{}]", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
            }
        }
        if (filesToRecover.isEmpty()) {
            LOGGER.trace("[{}] [{}] no files to recover, all exist within the local store", shardId, snapshotId);
        }
        try {
            // list of all existing store files
            final List<String> deleteIfExistFiles = Arrays.asList(store.directory().listAll());
            for (final BlobStoreIndexShardSnapshot.FileInfo fileToRecover : filesToRecover) {
                // if a file with a same physical name already exist in the store we need to delete it
                // before restoring it from the snapshot. We could be lenient and try to reuse the existing
                // store files (and compare their names/length/checksum again with the snapshot files) but to
                // avoid extra complexity we simply delete them and restore them again like StoreRecovery
                // does with dangling indices. Any existing store file that is not restored from the snapshot
                // will be clean up by RecoveryTarget.cleanFiles().
                final String physicalName = fileToRecover.physicalName();
                if (deleteIfExistFiles.contains(physicalName)) {
                    LOGGER.trace("[{}] [{}] deleting pre-existing file [{}]", shardId, snapshotId, physicalName);
                    store.directory().deleteFile(physicalName);
                }
            }
            restoreFiles(filesToRecover, store, ActionListener.wrap(v -> {
                store.incRef();
                try {
                    afterRestore(snapshotFiles, store, restoredSegmentsFile);
                    listener.onResponse(null);
                } finally {
                    store.decRef();
                }
            }, listener::onFailure));
        } catch (IOException ex) {
            throw new IndexShardRestoreFailedException(shardId, "Failed to recover index", ex);
        }
    } catch (Exception e) {
        listener.onFailure(e);
    } finally {
        store.decRef();
    }
}

Example 2 with SnapshotFiles

use of org.elasticsearch.index.snapshots.blobstore.SnapshotFiles in project crate by crate.

the class BlobStoreRepository method snapshotShard.

@Override
public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, ActionListener<String> listener) {
    final ShardId shardId = store.shardId();
    final long startTime = threadPool.absoluteTimeInMillis();
    try {
        final String generation = snapshotStatus.generation();
        LOGGER.debug("[{}] [{}] snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation);
        final BlobContainer shardContainer = shardContainer(indexId, shardId);
        final Set<String> blobs;
        if (generation == null) {
            try {
                blobs = shardContainer.listBlobsByPrefix(INDEX_FILE_PREFIX).keySet();
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
            }
        } else {
            blobs = Collections.singleton(INDEX_FILE_PREFIX + generation);
        }
        Tuple<BlobStoreIndexShardSnapshots, String> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, generation);
        BlobStoreIndexShardSnapshots snapshots = tuple.v1();
        final String fileListGeneration = tuple.v2();
        if (snapshots.snapshots().stream().anyMatch(sf -> sf.snapshot().equals(snapshotId.getName()))) {
            throw new IndexShardSnapshotFailedException(shardId, "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting");
        }
        final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
        final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
        store.incRef();
        final Collection<String> fileNames;
        final Store.MetadataSnapshot metadataFromStore;
        try {
            // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
            try {
                LOGGER.trace("[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
                metadataFromStore = store.getMetadata(snapshotIndexCommit);
                fileNames = snapshotIndexCommit.getFileNames();
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
            }
        } finally {
            store.decRef();
        }
        int indexIncrementalFileCount = 0;
        int indexTotalNumberOfFiles = 0;
        long indexIncrementalSize = 0;
        long indexTotalFileCount = 0;
        for (String fileName : fileNames) {
            if (snapshotStatus.isAborted()) {
                LOGGER.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
                throw new IndexShardSnapshotFailedException(shardId, "Aborted");
            }
            LOGGER.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
            final StoreFileMetadata md = metadataFromStore.get(fileName);
            BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
            List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
            if (filesInfo != null) {
                for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
                    if (fileInfo.isSame(md)) {
                        // a commit point file with the same name, size and checksum was already copied to repository
                        // we will reuse it for this snapshot
                        existingFileInfo = fileInfo;
                        break;
                    }
                }
            }
            indexTotalFileCount += md.length();
            indexTotalNumberOfFiles++;
            if (existingFileInfo == null) {
                indexIncrementalFileCount++;
                indexIncrementalSize += md.length();
                // create a new FileInfo
                BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo(DATA_BLOB_PREFIX + UUIDs.randomBase64UUID(), md, chunkSize());
                indexCommitPointFiles.add(snapshotFileInfo);
                filesToSnapshot.add(snapshotFileInfo);
            } else {
                indexCommitPointFiles.add(existingFileInfo);
            }
        }
        snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileCount);
        assert indexIncrementalFileCount == filesToSnapshot.size();
        final StepListener<Collection<Void>> allFilesUploadedListener = new StepListener<>();
        allFilesUploadedListener.whenComplete(v -> {
            final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.moveToFinalize(snapshotIndexCommit.getGeneration());
            // now create and write the commit point
            final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), lastSnapshotStatus.getIndexVersion(), indexCommitPointFiles, lastSnapshotStatus.getStartTime(), threadPool.absoluteTimeInMillis() - lastSnapshotStatus.getStartTime(), lastSnapshotStatus.getIncrementalFileCount(), lastSnapshotStatus.getIncrementalSize());
            LOGGER.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
            try {
                indexShardSnapshotFormat.write(snapshot, shardContainer, snapshotId.getUUID(), false);
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
            }
            // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
            List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
            newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
            for (SnapshotFiles point : snapshots) {
                newSnapshotsList.add(point);
            }
            final List<String> blobsToDelete;
            final String indexGeneration;
            if (writeShardGens) {
                indexGeneration = UUIDs.randomBase64UUID();
                blobsToDelete = Collections.emptyList();
            } else {
                indexGeneration = Long.toString(Long.parseLong(fileListGeneration) + 1);
                // Delete all previous index-N blobs
                blobsToDelete = blobs.stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).collect(Collectors.toList());
                assert blobsToDelete.stream().mapToLong(b -> Long.parseLong(b.replaceFirst(SNAPSHOT_INDEX_PREFIX, ""))).max().orElse(-1L) < Long.parseLong(indexGeneration) : "Tried to delete an index-N blob newer than the current generation [" + indexGeneration + "] when deleting index-N blobs " + blobsToDelete;
            }
            try {
                writeShardIndexBlob(shardContainer, indexGeneration, new BlobStoreIndexShardSnapshots(newSnapshotsList));
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to finalize snapshot creation [" + snapshotId + "] with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]", e);
            }
            if (writeShardGens == false) {
                try {
                    shardContainer.deleteBlobsIgnoringIfNotExists(blobsToDelete);
                } catch (IOException e) {
                    LOGGER.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete old index-N blobs during finalization", snapshotId, shardId), e);
                }
            }
            snapshotStatus.moveToDone(threadPool.absoluteTimeInMillis(), indexGeneration);
            listener.onResponse(indexGeneration);
        }, listener::onFailure);
        if (indexIncrementalFileCount == 0) {
            allFilesUploadedListener.onResponse(Collections.emptyList());
            return;
        }
        final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        int maximumPoolSize = executor instanceof ThreadPoolExecutor ? ((ThreadPoolExecutor) executor).getMaximumPoolSize() : 1;
        // Start as many workers as fit into the snapshot pool at once at the most
        final int workers = Math.min(maximumPoolSize, indexIncrementalFileCount);
        final ActionListener<Void> filesListener = fileQueueListener(filesToSnapshot, workers, allFilesUploadedListener);
        for (int i = 0; i < workers; ++i) {
            executor.execute(ActionRunnable.run(filesListener, () -> {
                BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = filesToSnapshot.poll(0L, TimeUnit.MILLISECONDS);
                if (snapshotFileInfo != null) {
                    store.incRef();
                    try {
                        do {
                            snapshotFile(snapshotFileInfo, indexId, shardId, snapshotId, snapshotStatus, store);
                            snapshotFileInfo = filesToSnapshot.poll(0L, TimeUnit.MILLISECONDS);
                        } while (snapshotFileInfo != null);
                    } finally {
                        store.decRef();
                    }
                }
            }));
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}

Example 3 with SnapshotFiles

use of org.elasticsearch.index.snapshots.blobstore.SnapshotFiles in project crate by crate.

the class BlobStoreRepository method deleteFromShardSnapshotMeta.

/**
 * Delete snapshot from shard level metadata.
 */
private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta(Set<SnapshotId> survivingSnapshots, IndexId indexId, ShardId snapshotShardId, SnapshotId snapshotId, BlobContainer shardContainer, Set<String> blobs, BlobStoreIndexShardSnapshots snapshots, String indexGeneration) {
    // Build a list of snapshots that should be preserved
    List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
    final Set<String> survivingSnapshotNames = survivingSnapshots.stream().map(SnapshotId::getName).collect(Collectors.toSet());
    for (SnapshotFiles point : snapshots) {
        if (survivingSnapshotNames.contains(point.snapshot())) {
            newSnapshotsList.add(point);
        }
    }
    try {
        if (newSnapshotsList.isEmpty()) {
            return new ShardSnapshotMetaDeleteResult(indexId, snapshotShardId.id(), ShardGenerations.DELETED_SHARD_GEN, blobs);
        } else {
            final BlobStoreIndexShardSnapshots updatedSnapshots = new BlobStoreIndexShardSnapshots(newSnapshotsList);
            writeShardIndexBlob(shardContainer, indexGeneration, updatedSnapshots);
            final Set<String> survivingSnapshotUUIDs = survivingSnapshots.stream().map(SnapshotId::getUUID).collect(Collectors.toSet());
            return new ShardSnapshotMetaDeleteResult(indexId, snapshotShardId.id(), indexGeneration, unusedBlobs(blobs, survivingSnapshotUUIDs, updatedSnapshots));
        }
    } catch (IOException e) {
        throw new IndexShardSnapshotFailedException(snapshotShardId, "Failed to finalize snapshot deletion [" + snapshotId + "] with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]", e);
    }
}

Example 4 with SnapshotFiles

use of org.elasticsearch.index.snapshots.blobstore.SnapshotFiles in project crate by crate.

the class BlobStoreRepository method restoreShard.

@Override
public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState, ActionListener<Void> listener) {
    final ShardId shardId = store.shardId();
    final ActionListener<Void> restoreListener = ActionListener.delegateResponse(listener, (l, e) -> l.onFailure(new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e)));
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final BlobContainer container = shardContainer(indexId, snapshotShardId);
    executor.execute(ActionRunnable.wrap(restoreListener, l -> {
        final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId);
        final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
        new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) {

            @Override
            protected void restoreFiles(List<BlobStoreIndexShardSnapshot.FileInfo> filesToRecover, Store store, ActionListener<Void> listener) {
                if (filesToRecover.isEmpty()) {
                    listener.onResponse(null);
                } else {
                    // Start as many workers as fit into the snapshot pool at once at the most
                    int maxPoolSize = executor instanceof ThreadPoolExecutor ? ((ThreadPoolExecutor) executor).getMaximumPoolSize() : 1;
                    final int workers = Math.min(maxPoolSize, snapshotFiles.indexFiles().size());
                    final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> files = new LinkedBlockingQueue<>(filesToRecover);
                    final ActionListener<Void> allFilesListener = fileQueueListener(files, workers, ActionListener.map(listener, v -> null));
                    // restore the files from the snapshot to the Lucene store
                    for (int i = 0; i < workers; ++i) {
                        executor.execute(ActionRunnable.run(allFilesListener, () -> {
                            store.incRef();
                            try {
                                BlobStoreIndexShardSnapshot.FileInfo fileToRecover;
                                while ((fileToRecover = files.poll(0L, TimeUnit.MILLISECONDS)) != null) {
                                    restoreFile(fileToRecover, store);
                                }
                            } finally {
                                store.decRef();
                            }
                        }));
                    }
                }
            }

            private void restoreFile(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store store) throws IOException {
                boolean success = false;
                try (InputStream stream = maybeRateLimit(new SlicedInputStream(fileInfo.numberOfParts()) {

                    @Override
                    protected InputStream openSlice(long slice) throws IOException {
                        return container.readBlob(fileInfo.partName(slice));
                    }
                }, restoreRateLimiter, restoreRateLimitingTimeInNanos)) {
                    try (IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
                        final byte[] buffer = new byte[BUFFER_SIZE];
                        int length;
                        while ((length = stream.read(buffer)) > 0) {
                            indexOutput.writeBytes(buffer, 0, length);
                            recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.physicalName(), length);
                        }
                        Store.verify(indexOutput);
                        indexOutput.close();
                        store.directory().sync(Collections.singleton(fileInfo.physicalName()));
                        success = true;
                    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
                        try {
                            store.markStoreCorrupted(ex);
                        } catch (IOException e) {
                            LOGGER.warn("store cannot be marked as corrupted", e);
                        }
                        throw ex;
                    } finally {
                        if (success == false) {
                            store.deleteQuiet(fileInfo.physicalName());
                        }
                    }
                }
            }
        }.restore(snapshotFiles, store, l);
    }));
}