Search in sources :

Example 1 with Repository

use of org.opensearch.repositories.Repository in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method snapshots.

/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param snapshotsInProgress snapshots in progress in the cluster state
 * @param repositoryName      repository name
 * @param snapshotIds         snapshots for which to fetch snapshot information
 * @param ignoreUnavailable   if true, snapshots that could not be read will only be logged with a warning,
 *                            if false, they will throw an error
 * @return list of snapshots
 */
private List<SnapshotInfo> snapshots(@Nullable SnapshotsInProgress snapshotsInProgress, String repositoryName, List<SnapshotId> snapshotIds, boolean ignoreUnavailable) {
    final Set<SnapshotInfo> snapshotSet = new HashSet<>();
    final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
    // first, look at the snapshots in progress
    final List<SnapshotsInProgress.Entry> entries = SnapshotsService.currentSnapshots(snapshotsInProgress, repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList()));
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(new SnapshotInfo(entry));
        snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId());
    }
    // then, look in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    for (SnapshotId snapshotId : snapshotIdsToIterate) {
        try {
            snapshotSet.add(repository.getSnapshotInfo(snapshotId));
        } catch (Exception ex) {
            if (ignoreUnavailable) {
                logger.warn(() -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);
            } else {
                if (ex instanceof SnapshotException) {
                    throw ex;
                }
                throw new SnapshotException(repositoryName, snapshotId, "Snapshot could not be read", ex);
            }
        }
    }
    final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
    CollectionUtil.timSort(snapshotList);
    return unmodifiableList(snapshotList);
}
Also used : ArrayList(java.util.ArrayList) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) SnapshotException(org.opensearch.snapshots.SnapshotException) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) Repository(org.opensearch.repositories.Repository) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) HashSet(java.util.HashSet)

Example 2 with Repository

use of org.opensearch.repositories.Repository in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method masterOperation.

@Override
protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener<GetSnapshotsResponse> listener) {
    try {
        final String repository = request.repository();
        final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE);
        final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
        final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
        for (SnapshotInfo snapshotInfo : sortedCurrentSnapshots(snapshotsInProgress, repository)) {
            SnapshotId snapshotId = snapshotInfo.snapshotId();
            allSnapshotIds.put(snapshotId.getName(), snapshotId);
            currentSnapshots.add(snapshotInfo);
        }
        final RepositoryData repositoryData;
        if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
            repositoryData = PlainActionFuture.get(fut -> repositoriesService.getRepositoryData(repository, fut));
            for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
                allSnapshotIds.put(snapshotId.getName(), snapshotId);
            }
        } else {
            repositoryData = null;
        }
        final Set<SnapshotId> toResolve = new HashSet<>();
        if (isAllSnapshots(request.snapshots())) {
            toResolve.addAll(allSnapshotIds.values());
        } else {
            for (String snapshotOrPattern : request.snapshots()) {
                if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
                    toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
                } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                    if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                        toResolve.add(allSnapshotIds.get(snapshotOrPattern));
                    } else if (request.ignoreUnavailable() == false) {
                        throw new SnapshotMissingException(repository, snapshotOrPattern);
                    }
                } else {
                    for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
                        if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
                            toResolve.add(entry.getValue());
                        }
                    }
                }
            }
            if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
                throw new SnapshotMissingException(repository, request.snapshots()[0]);
            }
        }
        final List<SnapshotInfo> snapshotInfos;
        if (request.verbose()) {
            snapshotInfos = snapshots(snapshotsInProgress, repository, new ArrayList<>(toResolve), request.ignoreUnavailable());
        } else {
            if (repositoryData != null) {
                // want non-current snapshots as well, which are found in the repository data
                snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
            } else {
                // only want current snapshots
                snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
                CollectionUtil.timSort(snapshotInfos);
            }
        }
        listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
Also used : RepositoriesService(org.opensearch.repositories.RepositoriesService) Collections.unmodifiableList(java.util.Collections.unmodifiableList) ThreadPool(org.opensearch.threadpool.ThreadPool) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) SnapshotsService(org.opensearch.snapshots.SnapshotsService) Regex(org.opensearch.common.regex.Regex) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) IndexId(org.opensearch.repositories.IndexId) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) SnapshotException(org.opensearch.snapshots.SnapshotException) Map(java.util.Map) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) Repository(org.opensearch.repositories.Repository) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Set(java.util.Set) IOException(java.io.IOException) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) CollectionUtil(org.apache.lucene.util.CollectionUtil) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ClusterService(org.opensearch.cluster.service.ClusterService) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashSet(java.util.HashSet)

Example 3 with Repository

use of org.opensearch.repositories.Repository in project OpenSearch by opensearch-project.

the class TransportSnapshotsStatusAction method snapshotShards.

/**
 * Returns status of shards currently finished snapshots
 * <p>
 * This method is executed on master node and it's complimentary to the
 * {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
 * returns similar information but for already finished snapshots.
 * </p>
 *
 * @param repositoryName  repository name
 * @param snapshotInfo    snapshot info
 * @return map of shard id to snapshot status
 */
private Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName, final RepositoryData repositoryData, final SnapshotInfo snapshotInfo) throws IOException {
    final Repository repository = repositoriesService.repository(repositoryName);
    final Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
    for (String index : snapshotInfo.indices()) {
        IndexId indexId = repositoryData.resolveIndexId(index);
        IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotInfo.snapshotId(), indexId);
        if (indexMetadata != null) {
            int numberOfShards = indexMetadata.getNumberOfShards();
            for (int i = 0; i < numberOfShards; i++) {
                ShardId shardId = new ShardId(indexMetadata.getIndex(), i);
                SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId);
                if (shardFailure != null) {
                    shardStatus.put(shardId, IndexShardSnapshotStatus.newFailed(shardFailure.reason()));
                } else {
                    final IndexShardSnapshotStatus shardSnapshotStatus;
                    if (snapshotInfo.state() == SnapshotState.FAILED) {
                        // If the snapshot failed, but the shard's snapshot does
                        // not have an exception, it means that partial snapshots
                        // were disabled and in this case, the shard snapshot will
                        // *not* have any metadata, so attempting to read the shard
                        // snapshot status will throw an exception. Instead, we create
                        // a status for the shard to indicate that the shard snapshot
                        // could not be taken due to partial being set to false.
                        shardSnapshotStatus = IndexShardSnapshotStatus.newFailed("skipped");
                    } else {
                        shardSnapshotStatus = repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), indexId, shardId);
                    }
                    shardStatus.put(shardId, shardSnapshotStatus);
                }
            }
        }
    }
    return unmodifiableMap(shardStatus);
}
Also used : ShardId(org.opensearch.index.shard.ShardId) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) IndexId(org.opensearch.repositories.IndexId) Repository(org.opensearch.repositories.Repository) SnapshotShardFailure(org.opensearch.snapshots.SnapshotShardFailure) HashMap(java.util.HashMap) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata)

Example 4 with Repository

use of org.opensearch.repositories.Repository in project OpenSearch by opensearch-project.

the class IndexRecoveryIT method testSnapshotRecovery.

public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create repository");
    assertAcked(client().admin().cluster().preparePutRepository(REPO_NAME).setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", false)).get());
    ensureGreen();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).setIndices(INDEX_NAME).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client().admin().cluster().prepareGetSnapshots(REPO_NAME).setSnapshots(SNAP_NAME).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();
    logger.info("--> restore");
    RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).execute().actionGet();
    int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
    assertThat(totalShards, greaterThan(0));
    ensureGreen();
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    Repository repository = internalCluster().getMasterNodeInstance(RepositoriesService.class).repository(REPO_NAME);
    final RepositoryData repositoryData = PlainActionFuture.get(repository::getRepositoryData);
    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
        assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME));
        List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
        assertThat(recoveryStates.size(), equalTo(totalShards));
        for (RecoveryState recoveryState : recoveryStates) {
            SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(((SnapshotRecoverySource) recoveryState.getRecoverySource()).restoreUUID(), new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()), Version.CURRENT, repositoryData.resolveIndexId(INDEX_NAME));
            assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA);
            validateIndexRecoveryState(recoveryState.getIndex());
        }
    }
}
Also used : RestoreSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse) RecoveryResponse(org.opensearch.action.admin.indices.recovery.RecoveryResponse) RepositoryData(org.opensearch.repositories.RepositoryData) Snapshot(org.opensearch.snapshots.Snapshot) Repository(org.opensearch.repositories.Repository) SnapshotRecoverySource(org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource) CreateSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse) RepositoriesService(org.opensearch.repositories.RepositoriesService) ArrayList(java.util.ArrayList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) Map(java.util.Map) Collections.singletonMap(java.util.Collections.singletonMap)

Example 5 with Repository

use of org.opensearch.repositories.Repository in project OpenSearch by opensearch-project.

the class TransportCleanupRepositoryAction method cleanupRepo.

/**
 * Runs cleanup operations on the given repository.
 * @param repositoryName Repository to clean up
 * @param listener Listener for cleanup result
 */
private void cleanupRepo(String repositoryName, ActionListener<RepositoryCleanupResult> listener) {
    final Repository repository = repositoriesService.repository(repositoryName);
    if (repository instanceof BlobStoreRepository == false) {
        listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup"));
        return;
    }
    final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repository.getRepositoryData(repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        final long repositoryStateId = repositoryData.getGenId();
        logger.info("Running cleanup operations on repository [{}][{}]", repositoryName, repositoryStateId);
        clusterService.submitStateUpdateTask("cleanup repository [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

            private boolean startedCleanup = false;

            @Override
            public ClusterState execute(ClusterState currentState) {
                final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
                if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a repository cleanup is already in-progress in [" + repositoryCleanupInProgress + "]");
                }
                final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                if (snapshots.entries().isEmpty() == false) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently running in [" + snapshots + "]");
                }
                return ClusterState.builder(currentState).putCustom(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(Collections.singletonList(RepositoryCleanupInProgress.startedEntry(repositoryName, repositoryStateId)))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                after(e, null);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                startedCleanup = true;
                logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId);
                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup(repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
            }

            private void after(@Nullable Exception failure, @Nullable RepositoryCleanupResult result) {
                if (failure == null) {
                    logger.debug("Finished repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId);
                } else {
                    logger.debug(() -> new ParameterizedMessage("Failed to finish repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                }
                assert failure != null || result != null;
                if (startedCleanup == false) {
                    logger.debug("No cleanup task to remove from cluster state because we failed to start one", failure);
                    listener.onFailure(failure);
                    return;
                }
                clusterService.submitStateUpdateTask("remove repository cleanup task [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        return removeInProgressCleanup(currentState);
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        if (failure != null) {
                            e.addSuppressed(failure);
                        }
                        logger.warn(() -> new ParameterizedMessage("[{}] failed to remove repository cleanup task", repositoryName), e);
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        if (failure == null) {
                            logger.info("Done with repository cleanup on [{}][{}] with result [{}]", repositoryName, repositoryStateId, result);
                            listener.onResponse(result);
                        } else {
                            logger.warn(() -> new ParameterizedMessage("Failed to run repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                            listener.onFailure(failure);
                        }
                    }
                });
            }
        });
    }, listener::onFailure);
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) RepositoryCleanupResult(org.opensearch.repositories.RepositoryCleanupResult) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) Repository(org.opensearch.repositories.Repository) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) StepListener(org.opensearch.action.StepListener) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage)

Aggregations

Repository (org.opensearch.repositories.Repository)54 IOException (java.io.IOException)37 RepositoryData (org.opensearch.repositories.RepositoryData)37 List (java.util.List)35 ClusterState (org.opensearch.cluster.ClusterState)35 Map (java.util.Map)34 Collections (java.util.Collections)33 Collectors (java.util.stream.Collectors)32 SnapshotsInProgress (org.opensearch.cluster.SnapshotsInProgress)32 RepositoriesService (org.opensearch.repositories.RepositoriesService)32 Version (org.opensearch.Version)31 StepListener (org.opensearch.action.StepListener)31 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)31 Settings (org.opensearch.common.settings.Settings)31 ClusterService (org.opensearch.cluster.service.ClusterService)30 ThreadPool (org.opensearch.threadpool.ThreadPool)30 Set (java.util.Set)29 Logger (org.apache.logging.log4j.Logger)29 ActionListener (org.opensearch.action.ActionListener)29 IndexId (org.opensearch.repositories.IndexId)29