Search in sources :

Example 1 with ShardGenerations

use of org.opensearch.repositories.ShardGenerations in project OpenSearch by opensearch-project.

the class SnapshotsService method shards.

/**
 * Calculates the assignment of shards to data nodes for a new snapshot based on the given cluster state and the
 * indices that should be included in the snapshot.
 *
 * @param indices             Indices to snapshot
 * @param useShardGenerations whether to write {@link ShardGenerations} during the snapshot
 * @return list of shard to be included into current snapshot
 */
private static ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards(SnapshotsInProgress snapshotsInProgress, @Nullable SnapshotDeletionsInProgress deletionsInProgress, Metadata metadata, RoutingTable routingTable, List<IndexId> indices, boolean useShardGenerations, RepositoryData repositoryData, String repoName) {
    ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
    final ShardGenerations shardGenerations = repositoryData.shardGenerations();
    final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo(repoName, snapshotsInProgress.entries());
    final boolean readyToExecute = deletionsInProgress == null || deletionsInProgress.getEntries().stream().noneMatch(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED);
    for (IndexId index : indices) {
        final String indexName = index.getName();
        final boolean isNewIndex = repositoryData.getIndices().containsKey(indexName) == false;
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            // The index was deleted before we managed to start the snapshot - mark it as missing.
            builder.put(new ShardId(indexName, IndexMetadata.INDEX_UUID_NA_VALUE, 0), ShardSnapshotStatus.MISSING);
        } else {
            final IndexRoutingTable indexRoutingTable = routingTable.index(indexName);
            for (int i = 0; i < indexMetadata.getNumberOfShards(); i++) {
                final ShardId shardId = indexRoutingTable.shard(i).shardId();
                final String shardRepoGeneration;
                if (useShardGenerations) {
                    final String inFlightGeneration = inFlightShardStates.generationForShard(index, shardId.id(), shardGenerations);
                    if (inFlightGeneration == null && isNewIndex) {
                        assert shardGenerations.getShardGen(index, shardId.getId()) == null : "Found shard generation for new index [" + index + "]";
                        shardRepoGeneration = ShardGenerations.NEW_SHARD_GEN;
                    } else {
                        shardRepoGeneration = inFlightGeneration;
                    }
                } else {
                    shardRepoGeneration = null;
                }
                final ShardSnapshotStatus shardSnapshotStatus;
                if (indexRoutingTable == null) {
                    shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus(null, ShardState.MISSING, "missing routing table", shardRepoGeneration);
                } else {
                    ShardRouting primary = indexRoutingTable.shard(i).primaryShard();
                    if (readyToExecute == false || inFlightShardStates.isActive(indexName, i)) {
                        shardSnapshotStatus = ShardSnapshotStatus.UNASSIGNED_QUEUED;
                    } else if (primary == null || !primary.assignedToNode()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(null, ShardState.MISSING, "primary shard is not allocated", shardRepoGeneration);
                    } else if (primary.relocating() || primary.initializing()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), ShardState.WAITING, shardRepoGeneration);
                    } else if (!primary.started()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), ShardState.MISSING, "primary shard hasn't been started yet", shardRepoGeneration);
                    } else {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), shardRepoGeneration);
                    }
                }
                builder.put(shardId, shardSnapshotStatus);
            }
        }
    }
    return builder.build();
}
Also used : RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) Collections.unmodifiableList(java.util.Collections.unmodifiableList) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) ClusterStateApplier(org.opensearch.cluster.ClusterStateApplier) Regex(org.opensearch.common.regex.Regex) Strings(org.opensearch.common.Strings) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) ObjectCursor(com.carrotsearch.hppc.cursors.ObjectCursor) TransportService(org.opensearch.transport.TransportService) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) ActionFilters(org.opensearch.action.support.ActionFilters) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) ShardState(org.opensearch.cluster.SnapshotsInProgress.ShardState) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) StepListener(org.opensearch.action.StepListener) State(org.opensearch.cluster.SnapshotsInProgress.State) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) RepositoriesService(org.opensearch.repositories.RepositoriesService) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) IOException(java.io.IOException) DeleteSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest) ClusterService(org.opensearch.cluster.service.ClusterService) RestoreInProgress(org.opensearch.cluster.RestoreInProgress) RoutingTable(org.opensearch.cluster.routing.RoutingTable) SnapshotsInProgress.completed(org.opensearch.cluster.SnapshotsInProgress.completed) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ShardGenerations(org.opensearch.repositories.ShardGenerations) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexId(org.opensearch.repositories.IndexId) Locale(java.util.Locale) NotMasterException(org.opensearch.cluster.NotMasterException) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) Objects(java.util.Objects) List(java.util.List) Optional(java.util.Optional) ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) CloneSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest) HashMap(java.util.HashMap) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) UUIDs(org.opensearch.common.UUIDs) LinkedList(java.util.LinkedList) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) Iterator(java.util.Iterator) Collections.emptySet(java.util.Collections.emptySet) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) CreateSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexId(org.opensearch.repositories.IndexId) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) ShardGenerations(org.opensearch.repositories.ShardGenerations) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardId(org.opensearch.index.shard.ShardId) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 2 with ShardGenerations

use of org.opensearch.repositories.ShardGenerations in project OpenSearch by opensearch-project.

the class SnapshotsService method buildGenerations.

private static ShardGenerations buildGenerations(SnapshotsInProgress.Entry snapshot, Metadata metadata) {
    ShardGenerations.Builder builder = ShardGenerations.builder();
    final Map<String, IndexId> indexLookup = new HashMap<>();
    snapshot.indices().forEach(idx -> indexLookup.put(idx.getName(), idx));
    if (snapshot.isClone()) {
        snapshot.clones().forEach(c -> {
            final IndexId indexId = indexLookup.get(c.key.indexName());
            builder.put(indexId, c.key.shardId(), c.value.generation());
        });
    } else {
        snapshot.shards().forEach(c -> {
            if (metadata.index(c.key.getIndex()) == null) {
                assert snapshot.partial() : "Index [" + c.key.getIndex() + "] was deleted during a snapshot but snapshot was not partial.";
                return;
            }
            final IndexId indexId = indexLookup.get(c.key.getIndexName());
            if (indexId != null) {
                builder.put(indexId, c.key.id(), c.value.generation());
            }
        });
    }
    return builder.build();
}
Also used : IndexId(org.opensearch.repositories.IndexId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ShardGenerations(org.opensearch.repositories.ShardGenerations)

Example 3 with ShardGenerations

use of org.opensearch.repositories.ShardGenerations in project OpenSearch by opensearch-project.

the class SnapshotsService method startCloning.

/**
 * Determine the number of shards in each index of a clone operation and update the cluster state accordingly.
 *
 * @param repository     repository to run operation on
 * @param cloneEntry     clone operation in the cluster state
 */
private void startCloning(Repository repository, SnapshotsInProgress.Entry cloneEntry) {
    final List<IndexId> indices = cloneEntry.indices();
    final SnapshotId sourceSnapshot = cloneEntry.source();
    final Snapshot targetSnapshot = cloneEntry.snapshot();
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    // Exception handler for IO exceptions with loading index and repo metadata
    final Consumer<Exception> onFailure = e -> {
        initializingClones.remove(targetSnapshot);
        logger.info(() -> new ParameterizedMessage("Failed to start snapshot clone [{}]", cloneEntry), e);
        removeFailedSnapshotFromClusterState(targetSnapshot, e, null, null);
    };
    // 1. step, load SnapshotInfo to make sure that source snapshot was successful for the indices we want to clone
    // TODO: we could skip this step for snapshots with state SUCCESS
    final StepListener<SnapshotInfo> snapshotInfoListener = new StepListener<>();
    executor.execute(ActionRunnable.supply(snapshotInfoListener, () -> repository.getSnapshotInfo(sourceSnapshot)));
    final StepListener<Collection<Tuple<IndexId, Integer>>> allShardCountsListener = new StepListener<>();
    final GroupedActionListener<Tuple<IndexId, Integer>> shardCountListener = new GroupedActionListener<>(allShardCountsListener, indices.size());
    snapshotInfoListener.whenComplete(snapshotInfo -> {
        for (IndexId indexId : indices) {
            if (RestoreService.failed(snapshotInfo, indexId.getName())) {
                throw new SnapshotException(targetSnapshot, "Can't clone index [" + indexId + "] because its snapshot was not successful.");
            }
        }
        // 2. step, load the number of shards we have in each index to be cloned from the index metadata.
        repository.getRepositoryData(ActionListener.wrap(repositoryData -> {
            for (IndexId index : indices) {
                executor.execute(ActionRunnable.supply(shardCountListener, () -> {
                    final IndexMetadata metadata = repository.getSnapshotIndexMetaData(repositoryData, sourceSnapshot, index);
                    return Tuple.tuple(index, metadata.getNumberOfShards());
                }));
            }
        }, onFailure));
    }, onFailure);
    // 3. step, we have all the shard counts, now update the cluster state to have clone jobs in the snap entry
    allShardCountsListener.whenComplete(counts -> repository.executeConsistentStateUpdate(repoData -> new ClusterStateUpdateTask() {

        private SnapshotsInProgress.Entry updatedEntry;

        @Override
        public ClusterState execute(ClusterState currentState) {
            final SnapshotsInProgress snapshotsInProgress = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
            final List<SnapshotsInProgress.Entry> updatedEntries = new ArrayList<>(snapshotsInProgress.entries());
            boolean changed = false;
            final String localNodeId = currentState.nodes().getLocalNodeId();
            final String repoName = cloneEntry.repository();
            final ShardGenerations shardGenerations = repoData.shardGenerations();
            for (int i = 0; i < updatedEntries.size(); i++) {
                if (cloneEntry.snapshot().equals(updatedEntries.get(i).snapshot())) {
                    final ImmutableOpenMap.Builder<RepositoryShardId, ShardSnapshotStatus> clonesBuilder = ImmutableOpenMap.builder();
                    final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo(repoName, snapshotsInProgress.entries());
                    for (Tuple<IndexId, Integer> count : counts) {
                        for (int shardId = 0; shardId < count.v2(); shardId++) {
                            final RepositoryShardId repoShardId = new RepositoryShardId(count.v1(), shardId);
                            final String indexName = repoShardId.indexName();
                            if (inFlightShardStates.isActive(indexName, shardId)) {
                                clonesBuilder.put(repoShardId, ShardSnapshotStatus.UNASSIGNED_QUEUED);
                            } else {
                                clonesBuilder.put(repoShardId, new ShardSnapshotStatus(localNodeId, inFlightShardStates.generationForShard(repoShardId.index(), shardId, shardGenerations)));
                            }
                        }
                    }
                    updatedEntry = cloneEntry.withClones(clonesBuilder.build());
                    updatedEntries.set(i, updatedEntry);
                    changed = true;
                    break;
                }
            }
            return updateWithSnapshots(currentState, changed ? SnapshotsInProgress.of(updatedEntries) : null, null);
        }

        @Override
        public void onFailure(String source, Exception e) {
            initializingClones.remove(targetSnapshot);
            logger.info(() -> new ParameterizedMessage("Failed to start snapshot clone [{}]", cloneEntry), e);
            failAllListenersOnMasterFailOver(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            initializingClones.remove(targetSnapshot);
            if (updatedEntry != null) {
                final Snapshot target = updatedEntry.snapshot();
                final SnapshotId sourceSnapshot = updatedEntry.source();
                for (ObjectObjectCursor<RepositoryShardId, ShardSnapshotStatus> indexClone : updatedEntry.clones()) {
                    final ShardSnapshotStatus shardStatusBefore = indexClone.value;
                    if (shardStatusBefore.state() != ShardState.INIT) {
                        continue;
                    }
                    final RepositoryShardId repoShardId = indexClone.key;
                    runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, repository);
                }
            } else {
                // Extremely unlikely corner case of master failing over between between starting the clone and
                // starting shard clones.
                logger.warn("Did not find expected entry [{}] in the cluster state", cloneEntry);
            }
        }
    }, "start snapshot clone", onFailure), onFailure);
}
Also used : RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) Collections.unmodifiableList(java.util.Collections.unmodifiableList) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) ClusterStateApplier(org.opensearch.cluster.ClusterStateApplier) Regex(org.opensearch.common.regex.Regex) Strings(org.opensearch.common.Strings) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) ObjectCursor(com.carrotsearch.hppc.cursors.ObjectCursor) TransportService(org.opensearch.transport.TransportService) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) ActionFilters(org.opensearch.action.support.ActionFilters) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) ShardState(org.opensearch.cluster.SnapshotsInProgress.ShardState) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) StepListener(org.opensearch.action.StepListener) State(org.opensearch.cluster.SnapshotsInProgress.State) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) RepositoriesService(org.opensearch.repositories.RepositoriesService) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) IOException(java.io.IOException) DeleteSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest) ClusterService(org.opensearch.cluster.service.ClusterService) RestoreInProgress(org.opensearch.cluster.RestoreInProgress) RoutingTable(org.opensearch.cluster.routing.RoutingTable) SnapshotsInProgress.completed(org.opensearch.cluster.SnapshotsInProgress.completed) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ShardGenerations(org.opensearch.repositories.ShardGenerations) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexId(org.opensearch.repositories.IndexId) Locale(java.util.Locale) NotMasterException(org.opensearch.cluster.NotMasterException) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) Objects(java.util.Objects) List(java.util.List) Optional(java.util.Optional) ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) CloneSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest) HashMap(java.util.HashMap) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) UUIDs(org.opensearch.common.UUIDs) LinkedList(java.util.LinkedList) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) Iterator(java.util.Iterator) Collections.emptySet(java.util.Collections.emptySet) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) CreateSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) ShardGenerations(org.opensearch.repositories.ShardGenerations) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Executor(java.util.concurrent.Executor) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) IndexId(org.opensearch.repositories.IndexId) ClusterState(org.opensearch.cluster.ClusterState) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) IOException(java.io.IOException) NotMasterException(org.opensearch.cluster.NotMasterException) RepositoryException(org.opensearch.repositories.RepositoryException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) Collection(java.util.Collection) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) StepListener(org.opensearch.action.StepListener) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) Tuple(org.opensearch.common.collect.Tuple)

Example 4 with ShardGenerations

use of org.opensearch.repositories.ShardGenerations in project OpenSearch by opensearch-project.

the class SnapshotsService method finalizeSnapshotEntry.

private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata metadata, RepositoryData repositoryData) {
    assert currentlyFinalizing.contains(entry.repository());
    try {
        final String failure = entry.failure();
        final Snapshot snapshot = entry.snapshot();
        logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
        ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
        for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) {
            ShardId shardId = shardStatus.key;
            ShardSnapshotStatus status = shardStatus.value;
            final ShardState state = status.state();
            if (state.failed()) {
                shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
            } else if (state.completed() == false) {
                shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, "skipped"));
            } else {
                assert state == ShardState.SUCCESS;
            }
        }
        final ShardGenerations shardGenerations = buildGenerations(entry, metadata);
        final String repository = snapshot.getRepository();
        final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), entry.startTime(), failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata());
        final StepListener<Metadata> metadataListener = new StepListener<>();
        final Repository repo = repositoriesService.repository(snapshot.getRepository());
        if (entry.isClone()) {
            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(metadataListener, () -> {
                final Metadata.Builder metaBuilder = Metadata.builder(repo.getSnapshotGlobalMetadata(entry.source()));
                for (IndexId index : entry.indices()) {
                    metaBuilder.put(repo.getSnapshotIndexMetaData(repositoryData, entry.source(), index), false);
                }
                return metaBuilder.build();
            }));
        } else {
            metadataListener.onResponse(metadata);
        }
        metadataListener.whenComplete(meta -> repo.finalizeSnapshot(shardGenerations, repositoryData.getGenId(), metadataForSnapshot(entry, meta), snapshotInfo, entry.version(), state -> stateWithoutSnapshot(state, snapshot), ActionListener.wrap(newRepoData -> {
            completeListenersIgnoringException(endAndGetListenersToResolve(snapshot), Tuple.tuple(newRepoData, snapshotInfo));
            logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state());
            runNextQueuedOperation(newRepoData, repository, true);
        }, e -> handleFinalizationFailure(e, entry, repositoryData))), e -> handleFinalizationFailure(e, entry, repositoryData));
    } catch (Exception e) {
        assert false : new AssertionError(e);
        handleFinalizationFailure(e, entry, repositoryData);
    }
}
Also used : RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) Collections.unmodifiableList(java.util.Collections.unmodifiableList) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) ClusterStateApplier(org.opensearch.cluster.ClusterStateApplier) Regex(org.opensearch.common.regex.Regex) Strings(org.opensearch.common.Strings) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) ObjectCursor(com.carrotsearch.hppc.cursors.ObjectCursor) TransportService(org.opensearch.transport.TransportService) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) ActionFilters(org.opensearch.action.support.ActionFilters) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) ShardState(org.opensearch.cluster.SnapshotsInProgress.ShardState) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) StepListener(org.opensearch.action.StepListener) State(org.opensearch.cluster.SnapshotsInProgress.State) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) RepositoriesService(org.opensearch.repositories.RepositoriesService) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) IOException(java.io.IOException) DeleteSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest) ClusterService(org.opensearch.cluster.service.ClusterService) RestoreInProgress(org.opensearch.cluster.RestoreInProgress) RoutingTable(org.opensearch.cluster.routing.RoutingTable) SnapshotsInProgress.completed(org.opensearch.cluster.SnapshotsInProgress.completed) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ShardGenerations(org.opensearch.repositories.ShardGenerations) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexId(org.opensearch.repositories.IndexId) Locale(java.util.Locale) NotMasterException(org.opensearch.cluster.NotMasterException) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) Objects(java.util.Objects) List(java.util.List) Optional(java.util.Optional) ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) CloneSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest) HashMap(java.util.HashMap) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) UUIDs(org.opensearch.common.UUIDs) LinkedList(java.util.LinkedList) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) Iterator(java.util.Iterator) Collections.emptySet(java.util.Collections.emptySet) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) CreateSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexId(org.opensearch.repositories.IndexId) ShardGenerations(org.opensearch.repositories.ShardGenerations) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) Metadata(org.opensearch.cluster.metadata.Metadata) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) IOException(java.io.IOException) NotMasterException(org.opensearch.cluster.NotMasterException) RepositoryException(org.opensearch.repositories.RepositoryException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardId(org.opensearch.index.shard.ShardId) ShardState(org.opensearch.cluster.SnapshotsInProgress.ShardState) Repository(org.opensearch.repositories.Repository) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) StepListener(org.opensearch.action.StepListener)

Example 5 with ShardGenerations

use of org.opensearch.repositories.ShardGenerations in project OpenSearch by opensearch-project.

the class BlobStoreRepository method finalizeSnapshot.

@Override
public void finalizeSnapshot(final ShardGenerations shardGenerations, final long repositoryStateId, final Metadata clusterMetadata, SnapshotInfo snapshotInfo, Version repositoryMetaVersion, Function<ClusterState, ClusterState> stateTransformer, final ActionListener<RepositoryData> listener) {
    assert repositoryStateId > RepositoryData.UNKNOWN_REPO_GEN : "Must finalize based on a valid repository generation but received [" + repositoryStateId + "]";
    final Collection<IndexId> indices = shardGenerations.indices();
    final SnapshotId snapshotId = snapshotInfo.snapshotId();
    // Once we are done writing the updated index-N blob we remove the now unreferenced index-${uuid} blobs in each shard
    // directory if all nodes are at least at version SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION
    // If there are older version nodes in the cluster, we don't need to run this cleanup as it will have already happened
    // when writing the index-${N} to each shard directory.
    final boolean writeShardGens = SnapshotsService.useShardGenerations(repositoryMetaVersion);
    final Consumer<Exception> onUpdateFailure = e -> listener.onFailure(new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e));
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final boolean writeIndexGens = SnapshotsService.useIndexGenerations(repositoryMetaVersion);
    final StepListener<RepositoryData> repoDataListener = new StepListener<>();
    getRepositoryData(repoDataListener);
    repoDataListener.whenComplete(existingRepositoryData -> {
        final Map<IndexId, String> indexMetas;
        final Map<String, String> indexMetaIdentifiers;
        if (writeIndexGens) {
            indexMetaIdentifiers = ConcurrentCollections.newConcurrentMap();
            indexMetas = ConcurrentCollections.newConcurrentMap();
        } else {
            indexMetas = null;
            indexMetaIdentifiers = null;
        }
        final ActionListener<Void> allMetaListener = new GroupedActionListener<>(ActionListener.wrap(v -> {
            final RepositoryData updatedRepositoryData = existingRepositoryData.addSnapshot(snapshotId, snapshotInfo.state(), Version.CURRENT, shardGenerations, indexMetas, indexMetaIdentifiers);
            writeIndexGen(updatedRepositoryData, repositoryStateId, repositoryMetaVersion, stateTransformer, ActionListener.wrap(newRepoData -> {
                if (writeShardGens) {
                    cleanupOldShardGens(existingRepositoryData, updatedRepositoryData);
                }
                listener.onResponse(newRepoData);
            }, onUpdateFailure));
        }, onUpdateFailure), 2 + indices.size());
        // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method will
        // mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version of the
        // index or global metadata will be compatible with the segments written in this snapshot as well.
        // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
        // that decrements the generation it points at
        // Write Global MetaData
        executor.execute(ActionRunnable.run(allMetaListener, () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)));
        // write the index metadata for each index in the snapshot
        for (IndexId index : indices) {
            executor.execute(ActionRunnable.run(allMetaListener, () -> {
                final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
                if (writeIndexGens) {
                    final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
                    String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
                    if (metaUUID == null) {
                        // We don't yet have this version of the metadata so we write it
                        metaUUID = UUIDs.base64UUID();
                        INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
                        indexMetaIdentifiers.put(identifiers, metaUUID);
                    }
                    indexMetas.put(index, identifiers);
                } else {
                    INDEX_METADATA_FORMAT.write(clusterMetadata.index(index.getName()), indexContainer(index), snapshotId.getUUID(), compress);
                }
            }));
        }
        executor.execute(ActionRunnable.run(allMetaListener, () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)));
    }, onUpdateFailure);
}
Also used : Metadata(org.opensearch.cluster.metadata.Metadata) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) Strings(org.opensearch.common.Strings) AbortedSnapshotException(org.opensearch.snapshots.AbortedSnapshotException) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) RecoveryState(org.opensearch.indices.recovery.RecoveryState) Map(java.util.Map) Lucene(org.opensearch.common.lucene.Lucene) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) BlobStoreIndexShardSnapshot(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot) BlockingQueue(java.util.concurrent.BlockingQueue) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) Logger(org.apache.logging.log4j.Logger) RepositoryOperation(org.opensearch.repositories.RepositoryOperation) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) BytesArray(org.opensearch.common.bytes.BytesArray) BlobStoreIndexShardSnapshots(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots) FsBlobContainer(org.opensearch.common.blobstore.fs.FsBlobContainer) StepListener(org.opensearch.action.StepListener) XContentType(org.opensearch.common.xcontent.XContentType) IndexCommit(org.apache.lucene.index.IndexCommit) ThreadPool(org.opensearch.threadpool.ThreadPool) BlobContainer(org.opensearch.common.blobstore.BlobContainer) Releasable(org.opensearch.common.lease.Releasable) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) Numbers(org.opensearch.common.Numbers) SlicedInputStream(org.opensearch.index.snapshots.blobstore.SlicedInputStream) SnapshotException(org.opensearch.snapshots.SnapshotException) Streams(org.opensearch.common.io.Streams) CompressorFactory(org.opensearch.common.compress.CompressorFactory) RepositoryVerificationException(org.opensearch.repositories.RepositoryVerificationException) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) InputStreamIndexInput(org.opensearch.common.lucene.store.InputStreamIndexInput) LongStream(java.util.stream.LongStream) IndexInput(org.apache.lucene.store.IndexInput) SetOnce(org.apache.lucene.util.SetOnce) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) IOException(java.io.IOException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) NotXContentException(org.opensearch.common.compress.NotXContentException) AtomicLong(java.util.concurrent.atomic.AtomicLong) RepositoryCleanupResult(org.opensearch.repositories.RepositoryCleanupResult) BlobPath(org.opensearch.common.blobstore.BlobPath) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) CounterMetric(org.opensearch.common.metrics.CounterMetric) ShardGenerations(org.opensearch.repositories.ShardGenerations) NoSuchFileException(java.nio.file.NoSuchFileException) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) SnapshotCreationException(org.opensearch.snapshots.SnapshotCreationException) ByteSizeUnit(org.opensearch.common.unit.ByteSizeUnit) SnapshotFiles(org.opensearch.index.snapshots.blobstore.SnapshotFiles) SnapshotsService(org.opensearch.snapshots.SnapshotsService) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) MapperService(org.opensearch.index.mapper.MapperService) IndexId(org.opensearch.repositories.IndexId) XContentFactory(org.opensearch.common.xcontent.XContentFactory) RepositoryStats(org.opensearch.repositories.RepositoryStats) BlobMetadata(org.opensearch.common.blobstore.BlobMetadata) RecoverySettings(org.opensearch.indices.recovery.RecoverySettings) RepositoryException(org.opensearch.repositories.RepositoryException) FileInfo.canonicalName(org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName) BytesRef(org.apache.lucene.util.BytesRef) SnapshotId(org.opensearch.snapshots.SnapshotId) Collection(java.util.Collection) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Store(org.opensearch.index.store.Store) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) BlobStore(org.opensearch.common.blobstore.BlobStore) List(java.util.List) Optional(java.util.Optional) BytesReference(org.opensearch.common.bytes.BytesReference) RateLimitingInputStream(org.opensearch.index.snapshots.blobstore.RateLimitingInputStream) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FilterInputStream(java.io.FilterInputStream) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) IndexMetaDataGenerations(org.opensearch.repositories.IndexMetaDataGenerations) UUIDs(org.opensearch.common.UUIDs) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) IndexOutput(org.apache.lucene.store.IndexOutput) IndexShardRestoreFailedException(org.opensearch.index.snapshots.IndexShardRestoreFailedException) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) DeleteResult(org.opensearch.common.blobstore.DeleteResult) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) RateLimiter(org.apache.lucene.store.RateLimiter) InputStream(java.io.InputStream) IndexId(org.opensearch.repositories.IndexId) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) AbortedSnapshotException(org.opensearch.snapshots.AbortedSnapshotException) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) RepositoryVerificationException(org.opensearch.repositories.RepositoryVerificationException) IOException(java.io.IOException) IndexShardSnapshotFailedException(org.opensearch.index.snapshots.IndexShardSnapshotFailedException) NotXContentException(org.opensearch.common.compress.NotXContentException) NoSuchFileException(java.nio.file.NoSuchFileException) SnapshotCreationException(org.opensearch.snapshots.SnapshotCreationException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRestoreFailedException(org.opensearch.index.snapshots.IndexShardRestoreFailedException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) AbortedSnapshotException(org.opensearch.snapshots.AbortedSnapshotException) SnapshotException(org.opensearch.snapshots.SnapshotException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) Executor(java.util.concurrent.Executor) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) StepListener(org.opensearch.action.StepListener) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata)

Aggregations

IndexId (org.opensearch.repositories.IndexId)7 ShardGenerations (org.opensearch.repositories.ShardGenerations)7 IOException (java.io.IOException)5 Collection (java.util.Collection)5 Collections (java.util.Collections)5 List (java.util.List)5 Map (java.util.Map)5 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)5 Function (java.util.function.Function)5 Collectors (java.util.stream.Collectors)5 Version (org.opensearch.Version)5 ArrayList (java.util.ArrayList)4 Arrays (java.util.Arrays)4 HashMap (java.util.HashMap)4 Optional (java.util.Optional)4 Set (java.util.Set)4 Executor (java.util.concurrent.Executor)4 ObjectCursor (com.carrotsearch.hppc.cursors.ObjectCursor)3 ObjectObjectCursor (com.carrotsearch.hppc.cursors.ObjectObjectCursor)3 Collections.emptySet (java.util.Collections.emptySet)3