Examples with Repository org.elasticsearch.repositories.Repository used on opensource projects

Example 11 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class BlobStoreRepository method snapshotShard.

@Override
public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, ActionListener<String> listener) {
    final ShardId shardId = store.shardId();
    final long startTime = threadPool.absoluteTimeInMillis();
    try {
        final String generation = snapshotStatus.generation();
        LOGGER.debug("[{}] [{}] snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation);
        final BlobContainer shardContainer = shardContainer(indexId, shardId);
        final Set<String> blobs;
        if (generation == null) {
            try {
                blobs = shardContainer.listBlobsByPrefix(INDEX_FILE_PREFIX).keySet();
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
            }
        } else {
            blobs = Collections.singleton(INDEX_FILE_PREFIX + generation);
        }
        Tuple<BlobStoreIndexShardSnapshots, String> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, generation);
        BlobStoreIndexShardSnapshots snapshots = tuple.v1();
        final String fileListGeneration = tuple.v2();
        if (snapshots.snapshots().stream().anyMatch(sf -> sf.snapshot().equals(snapshotId.getName()))) {
            throw new IndexShardSnapshotFailedException(shardId, "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting");
        }
        final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
        final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
        store.incRef();
        final Collection<String> fileNames;
        final Store.MetadataSnapshot metadataFromStore;
        try {
            // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
            try {
                LOGGER.trace("[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
                metadataFromStore = store.getMetadata(snapshotIndexCommit);
                fileNames = snapshotIndexCommit.getFileNames();
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
            }
        } finally {
            store.decRef();
        }
        int indexIncrementalFileCount = 0;
        int indexTotalNumberOfFiles = 0;
        long indexIncrementalSize = 0;
        long indexTotalFileCount = 0;
        for (String fileName : fileNames) {
            if (snapshotStatus.isAborted()) {
                LOGGER.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
                throw new IndexShardSnapshotFailedException(shardId, "Aborted");
            }
            LOGGER.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
            final StoreFileMetadata md = metadataFromStore.get(fileName);
            BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
            List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
            if (filesInfo != null) {
                for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
                    if (fileInfo.isSame(md)) {
                        // a commit point file with the same name, size and checksum was already copied to repository
                        // we will reuse it for this snapshot
                        existingFileInfo = fileInfo;
                        break;
                    }
                }
            }
            indexTotalFileCount += md.length();
            indexTotalNumberOfFiles++;
            if (existingFileInfo == null) {
                indexIncrementalFileCount++;
                indexIncrementalSize += md.length();
                // create a new FileInfo
                BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo(DATA_BLOB_PREFIX + UUIDs.randomBase64UUID(), md, chunkSize());
                indexCommitPointFiles.add(snapshotFileInfo);
                filesToSnapshot.add(snapshotFileInfo);
            } else {
                indexCommitPointFiles.add(existingFileInfo);
            }
        }
        snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileCount);
        assert indexIncrementalFileCount == filesToSnapshot.size();
        final StepListener<Collection<Void>> allFilesUploadedListener = new StepListener<>();
        allFilesUploadedListener.whenComplete(v -> {
            final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.moveToFinalize(snapshotIndexCommit.getGeneration());
            // now create and write the commit point
            final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), lastSnapshotStatus.getIndexVersion(), indexCommitPointFiles, lastSnapshotStatus.getStartTime(), threadPool.absoluteTimeInMillis() - lastSnapshotStatus.getStartTime(), lastSnapshotStatus.getIncrementalFileCount(), lastSnapshotStatus.getIncrementalSize());
            LOGGER.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
            try {
                indexShardSnapshotFormat.write(snapshot, shardContainer, snapshotId.getUUID(), false);
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
            }
            // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
            List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
            newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
            for (SnapshotFiles point : snapshots) {
                newSnapshotsList.add(point);
            }
            final List<String> blobsToDelete;
            final String indexGeneration;
            if (writeShardGens) {
                indexGeneration = UUIDs.randomBase64UUID();
                blobsToDelete = Collections.emptyList();
            } else {
                indexGeneration = Long.toString(Long.parseLong(fileListGeneration) + 1);
                // Delete all previous index-N blobs
                blobsToDelete = blobs.stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).collect(Collectors.toList());
                assert blobsToDelete.stream().mapToLong(b -> Long.parseLong(b.replaceFirst(SNAPSHOT_INDEX_PREFIX, ""))).max().orElse(-1L) < Long.parseLong(indexGeneration) : "Tried to delete an index-N blob newer than the current generation [" + indexGeneration + "] when deleting index-N blobs " + blobsToDelete;
            }
            try {
                writeShardIndexBlob(shardContainer, indexGeneration, new BlobStoreIndexShardSnapshots(newSnapshotsList));
            } catch (IOException e) {
                throw new IndexShardSnapshotFailedException(shardId, "Failed to finalize snapshot creation [" + snapshotId + "] with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]", e);
            }
            if (writeShardGens == false) {
                try {
                    shardContainer.deleteBlobsIgnoringIfNotExists(blobsToDelete);
                } catch (IOException e) {
                    LOGGER.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete old index-N blobs during finalization", snapshotId, shardId), e);
                }
            }
            snapshotStatus.moveToDone(threadPool.absoluteTimeInMillis(), indexGeneration);
            listener.onResponse(indexGeneration);
        }, listener::onFailure);
        if (indexIncrementalFileCount == 0) {
            allFilesUploadedListener.onResponse(Collections.emptyList());
            return;
        }
        final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        int maximumPoolSize = executor instanceof ThreadPoolExecutor ? ((ThreadPoolExecutor) executor).getMaximumPoolSize() : 1;
        // Start as many workers as fit into the snapshot pool at once at the most
        final int workers = Math.min(maximumPoolSize, indexIncrementalFileCount);
        final ActionListener<Void> filesListener = fileQueueListener(filesToSnapshot, workers, allFilesUploadedListener);
        for (int i = 0; i < workers; ++i) {
            executor.execute(ActionRunnable.run(filesListener, () -> {
                BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = filesToSnapshot.poll(0L, TimeUnit.MILLISECONDS);
                if (snapshotFileInfo != null) {
                    store.incRef();
                    try {
                        do {
                            snapshotFile(snapshotFileInfo, indexId, shardId, snapshotId, snapshotStatus, store);
                            snapshotFileInfo = filesToSnapshot.poll(0L, TimeUnit.MILLISECONDS);
                        } while (snapshotFileInfo != null);
                    } finally {
                        store.decRef();
                    }
                }
            }));
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}

Example 12 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class BlobStoreRepository method finalizeSnapshot.

/**
 * {@inheritDoc}
 */
@Override
public void finalizeSnapshot(final SnapshotId snapshotId, final ShardGenerations shardGenerations, final long startTime, final String failure, final int totalShards, final List<SnapshotShardFailure> shardFailures, final long repositoryStateId, final boolean includeGlobalState, final Metadata clusterMetadata, boolean writeShardGens, final ActionListener<SnapshotInfo> listener) {
    final Collection<IndexId> indices = shardGenerations.indices();
    // Once we are done writing the updated index-N blob we remove the now unreferenced index-${uuid} blobs in each shard
    // directory if all nodes are at least at version SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION
    // If there are older version nodes in the cluster, we don't need to run this cleanup as it will have already happened
    // when writing the index-${N} to each shard directory.
    final Consumer<Exception> onUpdateFailure = e -> listener.onFailure(new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e));
    final ActionListener<SnapshotInfo> allMetaListener = new GroupedActionListener<>(ActionListener.wrap(snapshotInfos -> {
        assert snapshotInfos.size() == 1 : "Should have only received a single SnapshotInfo but received " + snapshotInfos;
        final SnapshotInfo snapshotInfo = snapshotInfos.iterator().next();
        getRepositoryData(ActionListener.wrap(existingRepositoryData -> {
            final RepositoryData updatedRepositoryData = existingRepositoryData.addSnapshot(snapshotId, snapshotInfo.state(), Version.CURRENT, shardGenerations);
            writeIndexGen(updatedRepositoryData, repositoryStateId, writeShardGens, ActionListener.wrap(v -> {
                if (writeShardGens) {
                    cleanupOldShardGens(existingRepositoryData, updatedRepositoryData);
                }
                listener.onResponse(snapshotInfo);
            }, onUpdateFailure));
        }, onUpdateFailure));
    }, onUpdateFailure), 2 + indices.size());
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method will
    // mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version of the
    // index or global metadata will be compatible with the segments written in this snapshot as well.
    // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
    // that decrements the generation it points at
    // Write Global Metadata
    executor.execute(ActionRunnable.run(allMetaListener, () -> globalMetadataFormat.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), false)));
    // write the index metadata for each index in the snapshot
    for (IndexId index : indices) {
        executor.execute(ActionRunnable.run(allMetaListener, () -> indexMetadataFormat.write(clusterMetadata.index(index.getName()), indexContainer(index), snapshotId.getUUID(), false)));
    }
    executor.execute(ActionRunnable.supply(allMetaListener, () -> {
        final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshotId, indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, threadPool.absoluteTimeInMillis(), totalShards, shardFailures, includeGlobalState);
        snapshotFormat.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), false);
        return snapshotInfo;
    }));
}

Example 13 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class SnapshotsService method deleteSnapshot.

/**
 * Deletes a snapshot from the repository, looking up the {@link Snapshot} reference before deleting.
 * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
 *
 * @param repositoryName  repositoryName
 * @param snapshotName    snapshotName
 * @param listener        listener
 */
public void deleteSnapshot(final String repositoryName, final String snapshotName, final ActionListener<Void> listener, final boolean immediatePriority) {
    // First, look for the snapshot in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    repository.getRepositoryData(ActionListener.wrap(repositoryData -> {
        Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds().stream().filter(s -> s.getName().equals(snapshotName)).findFirst();
        // if nothing found by the same name, then look in the cluster state for current in progress snapshots
        long repoGenId = repositoryData.getGenId();
        if (matchedEntry.isPresent() == false) {
            Optional<SnapshotsInProgress.Entry> matchedInProgress = currentSnapshots(repositoryName, Collections.emptyList()).stream().filter(s -> s.snapshot().getSnapshotId().getName().equals(snapshotName)).findFirst();
            if (matchedInProgress.isPresent()) {
                matchedEntry = matchedInProgress.map(s -> s.snapshot().getSnapshotId());
                // Derive repository generation if a snapshot is in progress because it will increment the generation when it finishes
                repoGenId = matchedInProgress.get().repositoryStateId() + 1L;
            }
        }
        if (matchedEntry.isPresent() == false) {
            throw new SnapshotMissingException(repositoryName, snapshotName);
        }
        deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener, repoGenId, immediatePriority);
    }, listener::onFailure));
}

Example 14 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class SnapshotsService method beginSnapshot.

/**
 * Starts snapshot.
 * <p>
 * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
 *
 * @param clusterState               cluster state
 * @param snapshot                   snapshot meta data
 * @param partial                    allow partial snapshots
 * @param userCreateSnapshotListener listener
 */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final ActionListener<Snapshot> userCreateSnapshotListener) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() {

        boolean snapshotCreated;

        boolean hadAbortedInitializations;

        @Override
        protected void doRun() {
            assert initializingSnapshots.contains(snapshot.snapshot());
            Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
            if (repository.isReadOnly()) {
                throw new RepositoryException(repository.getMetadata().name(), "cannot create snapshot in a readonly repository");
            }
            final String snapshotName = snapshot.snapshot().getSnapshotId().getName();
            final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
            repository.getRepositoryData(repositoryDataListener);
            repositoryDataListener.whenComplete(repositoryData -> {
                // check if the snapshot name already exists in the repository
                if (repositoryData.getSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
                    throw new InvalidSnapshotNameException(repository.getMetadata().name(), snapshotName, "snapshot with the same name already exists");
                }
                snapshotCreated = true;
                LOGGER.info("snapshot [{}] started", snapshot.snapshot());
                if (snapshot.indices().isEmpty()) {
                    // No indices in this snapshot - we are done
                    userCreateSnapshotListener.onResponse(snapshot.snapshot());
                    endSnapshot(snapshot, clusterState.metadata());
                    return;
                }
                clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                        List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                        for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                            if (entry.snapshot().equals(snapshot.snapshot()) == false) {
                                entries.add(entry);
                                continue;
                            }
                            if (entry.state() == State.ABORTED) {
                                entries.add(entry);
                                assert entry.shards().isEmpty();
                                hadAbortedInitializations = true;
                            } else {
                                // Replace the snapshot that was just initialized
                                ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards = shards(currentState, entry, repositoryData);
                                if (!partial) {
                                    Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metadata());
                                    Set<String> missing = indicesWithMissingShards.v1();
                                    Set<String> closed = indicesWithMissingShards.v2();
                                    if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                        final StringBuilder failureMessage = new StringBuilder();
                                        if (missing.isEmpty() == false) {
                                            failureMessage.append("Indices don't have primary shards ");
                                            failureMessage.append(missing);
                                        }
                                        if (closed.isEmpty() == false) {
                                            if (failureMessage.length() > 0) {
                                                failureMessage.append("; ");
                                            }
                                            failureMessage.append("Indices are closed ");
                                            failureMessage.append(closed);
                                        }
                                        entries.add(new SnapshotsInProgress.Entry(entry, State.FAILED, shards, failureMessage.toString()));
                                        continue;
                                    }
                                }
                                entries.add(new SnapshotsInProgress.Entry(entry, State.STARTED, shards));
                            }
                        }
                        return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(unmodifiableList(entries))).build();
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                        removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
                    }

                    @Override
                    public void onNoLongerMaster(String source) {
                        // We are not longer a master - we shouldn't try to do any cleanup
                        // The new master will take care of it
                        LOGGER.warn("[{}] failed to create snapshot - no longer a master", snapshot.snapshot().getSnapshotId());
                        userCreateSnapshotListener.onFailure(new SnapshotException(snapshot.snapshot(), "master changed during snapshot initialization"));
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                        // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                        // completion listener in this method. For the snapshot completion to work properly, the snapshot
                        // should still exist when listener is registered.
                        userCreateSnapshotListener.onResponse(snapshot.snapshot());
                        if (hadAbortedInitializations) {
                            final SnapshotsInProgress snapshotsInProgress = newState.custom(SnapshotsInProgress.TYPE);
                            assert snapshotsInProgress != null;
                            final SnapshotsInProgress.Entry entry = snapshotsInProgress.snapshot(snapshot.snapshot());
                            assert entry != null;
                            endSnapshot(entry, newState.metadata());
                        }
                    }
                });
            }, this::onFailure);
        }

        @Override
        public void onFailure(Exception e) {
            LOGGER.warn(() -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
            removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
        }
    });
}

Example 15 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class BlobStoreRepositoryRestoreTests method testRestoreSnapshotWithExistingFiles.

/**
 * Restoring a snapshot that contains multiple files must succeed even when
 * some files already exist in the shard's store.
 */
public void testRestoreSnapshotWithExistingFiles() throws IOException {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        snapshotShard(shard, snapshot, repository);
        // capture current store files
        final Store.MetadataSnapshot storeFiles = shard.snapshotStoreMetadata();
        assertFalse(storeFiles.asMap().isEmpty());
        // close the shard
        closeShards(shard);
        // delete some random files in the store
        List<String> deletedFiles = randomSubsetOf(randomIntBetween(1, storeFiles.size() - 1), storeFiles.asMap().keySet());
        for (String deletedFile : deletedFiles) {
            Files.delete(shard.shardPath().resolveIndex().resolve(deletedFile));
        }
        // build a new shard using the same store directory as the closed shard
        ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE);
        shard = newShard(shardRouting, shard.shardPath(), shard.indexSettings().getIndexMetadata(), null, new InternalEngineFactory(), () -> {
        }, RetentionLeaseSyncer.EMPTY, EMPTY_EVENT_LISTENER);
        // restore the shard
        recoverShardFromSnapshot(shard, snapshot, repository);
        // check that the shard is not corrupted
        TestUtil.checkIndex(shard.store().directory());
        // check that all files have been restored
        final Directory directory = shard.store().directory();
        final List<String> directoryFiles = Arrays.asList(directory.listAll());
        for (StoreFileMetadata storeFile : storeFiles) {
            String fileName = storeFile.name();
            assertTrue("File [" + fileName + "] does not exist in store directory", directoryFiles.contains(fileName));
            assertEquals(storeFile.length(), shard.store().directory().fileLength(fileName));
        }
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}