Examples with RepositoryData org.opensearch.repositories.RepositoryData used on opensource projects

Example 21 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class SnapshotsService method finalizeSnapshotEntry.

private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata metadata, RepositoryData repositoryData) {
    assert currentlyFinalizing.contains(entry.repository());
    try {
        final String failure = entry.failure();
        final Snapshot snapshot = entry.snapshot();
        logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
        ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
        for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) {
            ShardId shardId = shardStatus.key;
            ShardSnapshotStatus status = shardStatus.value;
            final ShardState state = status.state();
            if (state.failed()) {
                shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
            } else if (state.completed() == false) {
                shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, "skipped"));
            } else {
                assert state == ShardState.SUCCESS;
            }
        }
        final ShardGenerations shardGenerations = buildGenerations(entry, metadata);
        final String repository = snapshot.getRepository();
        final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), entry.startTime(), failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata());
        final StepListener<Metadata> metadataListener = new StepListener<>();
        final Repository repo = repositoriesService.repository(snapshot.getRepository());
        if (entry.isClone()) {
            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(metadataListener, () -> {
                final Metadata.Builder metaBuilder = Metadata.builder(repo.getSnapshotGlobalMetadata(entry.source()));
                for (IndexId index : entry.indices()) {
                    metaBuilder.put(repo.getSnapshotIndexMetaData(repositoryData, entry.source(), index), false);
                }
                return metaBuilder.build();
            }));
        } else {
            metadataListener.onResponse(metadata);
        }
        metadataListener.whenComplete(meta -> repo.finalizeSnapshot(shardGenerations, repositoryData.getGenId(), metadataForSnapshot(entry, meta), snapshotInfo, entry.version(), state -> stateWithoutSnapshot(state, snapshot), ActionListener.wrap(newRepoData -> {
            completeListenersIgnoringException(endAndGetListenersToResolve(snapshot), Tuple.tuple(newRepoData, snapshotInfo));
            logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state());
            runNextQueuedOperation(newRepoData, repository, true);
        }, e -> handleFinalizationFailure(e, entry, repositoryData))), e -> handleFinalizationFailure(e, entry, repositoryData));
    } catch (Exception e) {
        assert false : new AssertionError(e);
        handleFinalizationFailure(e, entry, repositoryData);
    }
}

Example 22 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreRepository method writeUpdatedShardMetaDataAndComputeDeletes.

// updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up.
private void writeUpdatedShardMetaDataAndComputeDeletes(Collection<SnapshotId> snapshotIds, RepositoryData oldRepositoryData, boolean useUUIDs, ActionListener<Collection<ShardSnapshotMetaDeleteResult>> onAllShardsCompleted) {
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final List<IndexId> indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds);
    if (indices.isEmpty()) {
        onAllShardsCompleted.onResponse(Collections.emptyList());
        return;
    }
    // Listener that flattens out the delete results for each index
    final ActionListener<Collection<ShardSnapshotMetaDeleteResult>> deleteIndexMetadataListener = new GroupedActionListener<>(ActionListener.map(onAllShardsCompleted, res -> res.stream().flatMap(Collection::stream).collect(Collectors.toList())), indices.size());
    for (IndexId indexId : indices) {
        final Set<SnapshotId> survivingSnapshots = oldRepositoryData.getSnapshots(indexId).stream().filter(id -> snapshotIds.contains(id) == false).collect(Collectors.toSet());
        final StepListener<Collection<Integer>> shardCountListener = new StepListener<>();
        final Collection<String> indexMetaGenerations = snapshotIds.stream().map(id -> oldRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId)).collect(Collectors.toSet());
        final ActionListener<Integer> allShardCountsListener = new GroupedActionListener<>(shardCountListener, indexMetaGenerations.size());
        final BlobContainer indexContainer = indexContainer(indexId);
        for (String indexMetaGeneration : indexMetaGenerations) {
            executor.execute(ActionRunnable.supply(allShardCountsListener, () -> {
                try {
                    return INDEX_METADATA_FORMAT.read(indexContainer, indexMetaGeneration, namedXContentRegistry).getNumberOfShards();
                } catch (Exception ex) {
                    logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", indexMetaGeneration, indexId.getName()), ex);
                    // ignoring it and letting the cleanup deal with it.
                    return null;
                }
            }));
        }
        shardCountListener.whenComplete(counts -> {
            final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0);
            if (shardCount == 0) {
                deleteIndexMetadataListener.onResponse(null);
                return;
            }
            // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index
            final ActionListener<ShardSnapshotMetaDeleteResult> allShardsListener = new GroupedActionListener<>(deleteIndexMetadataListener, shardCount);
            for (int shardId = 0; shardId < shardCount; shardId++) {
                final int finalShardId = shardId;
                executor.execute(new AbstractRunnable() {

                    @Override
                    protected void doRun() throws Exception {
                        final BlobContainer shardContainer = shardContainer(indexId, finalShardId);
                        final Set<String> blobs = shardContainer.listBlobs().keySet();
                        final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots;
                        final long newGen;
                        if (useUUIDs) {
                            newGen = -1L;
                            blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, oldRepositoryData.shardGenerations().getShardGen(indexId, finalShardId)).v1();
                        } else {
                            Tuple<BlobStoreIndexShardSnapshots, Long> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer);
                            newGen = tuple.v2() + 1;
                            blobStoreIndexShardSnapshots = tuple.v1();
                        }
                        allShardsListener.onResponse(deleteFromShardSnapshotMeta(survivingSnapshots, indexId, finalShardId, snapshotIds, shardContainer, blobs, blobStoreIndexShardSnapshots, newGen));
                    }

                    @Override
                    public void onFailure(Exception ex) {
                        logger.warn(() -> new ParameterizedMessage("{} failed to delete shard data for shard [{}][{}]", snapshotIds, indexId.getName(), finalShardId), ex);
                        // Just passing null here to count down the listener instead of failing it, the stale data left behind
                        // here will be retried in the next delete or repository cleanup
                        allShardsListener.onResponse(null);
                    }
                });
            }
        }, deleteIndexMetadataListener::onFailure);
    }
}

Example 23 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreRepository method doGetRepositoryData.

private void doGetRepositoryData(ActionListener<RepositoryData> listener) {
    // Retry loading RepositoryData in a loop in case we run into concurrent modifications of the repository.
    // Keep track of the most recent generation we failed to load so we can break out of the loop if we fail to load the same
    // generation repeatedly.
    long lastFailedGeneration = RepositoryData.UNKNOWN_REPO_GEN;
    while (true) {
        final long genToLoad;
        if (bestEffortConsistency) {
            // We're only using #latestKnownRepoGen as a hint in this mode and listing repo contents as a secondary way of trying
            // to find a higher generation
            final long generation;
            try {
                generation = latestIndexBlobId();
            } catch (IOException ioe) {
                listener.onFailure(new RepositoryException(metadata.name(), "Could not determine repository generation from root blobs", ioe));
                return;
            }
            genToLoad = latestKnownRepoGen.updateAndGet(known -> Math.max(known, generation));
            if (genToLoad > generation) {
                logger.info("Determined repository generation [" + generation + "] from repository contents but correct generation must be at least [" + genToLoad + "]");
            }
        } else {
            // We only rely on the generation tracked in #latestKnownRepoGen which is exclusively updated from the cluster state
            genToLoad = latestKnownRepoGen.get();
        }
        try {
            final Tuple<Long, BytesReference> cached = latestKnownRepositoryData.get();
            final RepositoryData loaded;
            // Caching is not used with #bestEffortConsistency see docs on #cacheRepositoryData for details
            if (bestEffortConsistency == false && cached != null && cached.v1() == genToLoad) {
                loaded = repositoryDataFromCachedEntry(cached);
            } else {
                loaded = getRepositoryData(genToLoad);
                // We can cache serialized in the most recent version here without regard to the actual repository metadata version
                // since we're only caching the information that we just wrote and thus won't accidentally cache any information that
                // isn't safe
                cacheRepositoryData(BytesReference.bytes(loaded.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT)), genToLoad);
            }
            listener.onResponse(loaded);
            return;
        } catch (RepositoryException e) {
            // If the generation to load changed concurrently and we didn't just try loading the same generation before we retry
            if (genToLoad != latestKnownRepoGen.get() && genToLoad != lastFailedGeneration) {
                lastFailedGeneration = genToLoad;
                logger.warn("Failed to load repository data generation [" + genToLoad + "] because a concurrent operation moved the current generation to [" + latestKnownRepoGen.get() + "]", e);
                continue;
            }
            if (bestEffortConsistency == false && ExceptionsHelper.unwrap(e, NoSuchFileException.class) != null) {
                // We did not find the expected index-N even though the cluster state continues to point at the missing value
                // of N so we mark this repository as corrupted.
                markRepoCorrupted(genToLoad, e, ActionListener.wrap(v -> listener.onFailure(corruptedStateException(e)), listener::onFailure));
            } else {
                listener.onFailure(e);
            }
            return;
        } catch (Exception e) {
            listener.onFailure(new RepositoryException(metadata.name(), "Unexpected exception when loading repository data", e));
            return;
        }
    }
}

Example 24 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreRepository method asyncCleanupUnlinkedShardLevelBlobs.

private void asyncCleanupUnlinkedShardLevelBlobs(RepositoryData oldRepositoryData, Collection<SnapshotId> snapshotIds, Collection<ShardSnapshotMetaDeleteResult> deleteResults, ActionListener<Void> listener) {
    final List<String> filesToDelete = resolveFilesToDelete(oldRepositoryData, snapshotIds, deleteResults);
    if (filesToDelete.isEmpty()) {
        listener.onResponse(null);
        return;
    }
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> {
        try {
            deleteFromContainer(blobContainer(), filesToDelete);
            l.onResponse(null);
        } catch (Exception e) {
            logger.warn(() -> new ParameterizedMessage("{} Failed to delete some blobs during snapshot delete", snapshotIds), e);
            throw e;
        }
    }));
}

Example 25 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreRepository method executeConsistentStateUpdate.

@Override
public void executeConsistentStateUpdate(Function<RepositoryData, ClusterStateUpdateTask> createUpdateTask, String source, Consumer<Exception> onFailure) {
    final RepositoryMetadata repositoryMetadataStart = metadata;
    getRepositoryData(ActionListener.wrap(repositoryData -> {
        final ClusterStateUpdateTask updateTask = createUpdateTask.apply(repositoryData);
        clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask(updateTask.priority()) {

            private boolean executedTask = false;

            @Override
            public ClusterState execute(ClusterState currentState) throws Exception {
                // to change in any form.
                if (repositoryMetadataStart.equals(getRepoMetadata(currentState))) {
                    executedTask = true;
                    return updateTask.execute(currentState);
                }
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                if (executedTask) {
                    updateTask.onFailure(source, e);
                } else {
                    onFailure.accept(e);
                }
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                if (executedTask) {
                    updateTask.clusterStateProcessed(source, oldState, newState);
                } else {
                    executeConsistentStateUpdate(createUpdateTask, source, onFailure);
                }
            }

            @Override
            public TimeValue timeout() {
                return updateTask.timeout();
            }
        });
    }, onFailure));
}