Search in sources :

Example 1 with SnapshotMissingException

use of org.opensearch.snapshots.SnapshotMissingException in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method masterOperation.

@Override
protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener<GetSnapshotsResponse> listener) {
    try {
        final String repository = request.repository();
        final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE);
        final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
        final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
        for (SnapshotInfo snapshotInfo : sortedCurrentSnapshots(snapshotsInProgress, repository)) {
            SnapshotId snapshotId = snapshotInfo.snapshotId();
            allSnapshotIds.put(snapshotId.getName(), snapshotId);
            currentSnapshots.add(snapshotInfo);
        }
        final RepositoryData repositoryData;
        if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
            repositoryData = PlainActionFuture.get(fut -> repositoriesService.getRepositoryData(repository, fut));
            for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
                allSnapshotIds.put(snapshotId.getName(), snapshotId);
            }
        } else {
            repositoryData = null;
        }
        final Set<SnapshotId> toResolve = new HashSet<>();
        if (isAllSnapshots(request.snapshots())) {
            toResolve.addAll(allSnapshotIds.values());
        } else {
            for (String snapshotOrPattern : request.snapshots()) {
                if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
                    toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
                } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                    if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                        toResolve.add(allSnapshotIds.get(snapshotOrPattern));
                    } else if (request.ignoreUnavailable() == false) {
                        throw new SnapshotMissingException(repository, snapshotOrPattern);
                    }
                } else {
                    for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
                        if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
                            toResolve.add(entry.getValue());
                        }
                    }
                }
            }
            if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
                throw new SnapshotMissingException(repository, request.snapshots()[0]);
            }
        }
        final List<SnapshotInfo> snapshotInfos;
        if (request.verbose()) {
            snapshotInfos = snapshots(snapshotsInProgress, repository, new ArrayList<>(toResolve), request.ignoreUnavailable());
        } else {
            if (repositoryData != null) {
                // want non-current snapshots as well, which are found in the repository data
                snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
            } else {
                // only want current snapshots
                snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
                CollectionUtil.timSort(snapshotInfos);
            }
        }
        listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
Also used : RepositoriesService(org.opensearch.repositories.RepositoriesService) Collections.unmodifiableList(java.util.Collections.unmodifiableList) ThreadPool(org.opensearch.threadpool.ThreadPool) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) SnapshotsService(org.opensearch.snapshots.SnapshotsService) Regex(org.opensearch.common.regex.Regex) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) IndexId(org.opensearch.repositories.IndexId) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) SnapshotException(org.opensearch.snapshots.SnapshotException) Map(java.util.Map) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) Repository(org.opensearch.repositories.Repository) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Set(java.util.Set) IOException(java.io.IOException) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) CollectionUtil(org.apache.lucene.util.CollectionUtil) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ClusterService(org.opensearch.cluster.service.ClusterService) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashSet(java.util.HashSet)

Example 2 with SnapshotMissingException

use of org.opensearch.snapshots.SnapshotMissingException in project OpenSearch by opensearch-project.

the class SnapshotDisruptionIT method testDisruptionAfterFinalization.

public void testDisruptionAfterFinalization() throws Exception {
    final String idxName = "test";
    internalCluster().startMasterOnlyNodes(3);
    final String dataNode = internalCluster().startDataOnlyNode();
    ensureStableCluster(4);
    createRandomIndex(idxName);
    createRepository("test-repo", "fs");
    final String masterNode1 = internalCluster().getMasterName();
    NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.UNRESPONSIVE);
    internalCluster().setDisruptionScheme(networkDisruption);
    ClusterService clusterService = internalCluster().clusterService(masterNode1);
    CountDownLatch disruptionStarted = new CountDownLatch(1);
    clusterService.addListener(new ClusterStateListener() {

        @Override
        public void clusterChanged(ClusterChangedEvent event) {
            SnapshotsInProgress snapshots = event.state().custom(SnapshotsInProgress.TYPE);
            if (snapshots != null && snapshots.entries().size() > 0) {
                final SnapshotsInProgress.Entry snapshotEntry = snapshots.entries().get(0);
                if (snapshotEntry.state() == SnapshotsInProgress.State.SUCCESS) {
                    final RepositoriesMetadata repoMeta = event.state().metadata().custom(RepositoriesMetadata.TYPE);
                    final RepositoryMetadata metadata = repoMeta.repository("test-repo");
                    if (metadata.pendingGeneration() > snapshotEntry.repositoryStateId()) {
                        logger.info("--> starting disruption");
                        networkDisruption.startDisrupting();
                        clusterService.removeListener(this);
                        disruptionStarted.countDown();
                    }
                }
            }
        }
    });
    final String snapshot = "test-snap";
    logger.info("--> starting snapshot");
    ActionFuture<CreateSnapshotResponse> future = client(masterNode1).admin().cluster().prepareCreateSnapshot("test-repo", snapshot).setWaitForCompletion(true).setIndices(idxName).execute();
    logger.info("--> waiting for disruption to start");
    assertTrue(disruptionStarted.await(1, TimeUnit.MINUTES));
    awaitNoMoreRunningOperations(dataNode);
    logger.info("--> verify that snapshot was successful or no longer exist");
    assertBusy(() -> {
        try {
            assertSnapshotExists("test-repo", snapshot);
        } catch (SnapshotMissingException exception) {
            logger.info("--> done verifying, snapshot doesn't exist");
        }
    }, 1, TimeUnit.MINUTES);
    logger.info("--> stopping disrupting");
    networkDisruption.stopDisrupting();
    ensureStableCluster(4, masterNode1);
    logger.info("--> done");
    try {
        future.get();
        fail("Should have failed because the node disconnected from the cluster during snapshot finalization");
    } catch (Exception ex) {
        final SnapshotException sne = (SnapshotException) ExceptionsHelper.unwrap(ex, SnapshotException.class);
        assertNotNull(sne);
        assertThat(sne.getMessage(), either(endsWith(" Failed to update cluster state during snapshot finalization")).or(endsWith(" no longer master")));
        assertThat(sne.getSnapshotName(), is(snapshot));
    }
    awaitNoMoreRunningOperations(dataNode);
}
Also used : ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) CountDownLatch(java.util.concurrent.CountDownLatch) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) SnapshotException(org.opensearch.snapshots.SnapshotException) ClusterStateListener(org.opensearch.cluster.ClusterStateListener) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) ClusterService(org.opensearch.cluster.service.ClusterService) CreateSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) NetworkDisruption(org.opensearch.test.disruption.NetworkDisruption)

Example 3 with SnapshotMissingException

use of org.opensearch.snapshots.SnapshotMissingException in project OpenSearch by opensearch-project.

the class TransportSnapshotsStatusAction method loadRepositoryData.

private void loadRepositoryData(SnapshotsInProgress snapshotsInProgress, SnapshotsStatusRequest request, List<SnapshotStatus> builder, Set<String> currentSnapshotNames, String repositoryName, ActionListener<SnapshotsStatusResponse> listener) {
    final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repositoriesService.getRepositoryData(repositoryName, repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        final Map<String, SnapshotId> matchedSnapshotIds = repositoryData.getSnapshotIds().stream().filter(s -> requestedSnapshotNames.contains(s.getName())).collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
        for (final String snapshotName : request.snapshots()) {
            if (currentSnapshotNames.contains(snapshotName)) {
                // we've already found this snapshot in the current snapshot entries, so skip over
                continue;
            }
            SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName);
            if (snapshotId == null) {
                // neither in the current snapshot entries nor found in the repository
                if (request.ignoreUnavailable()) {
                    // ignoring unavailable snapshots, so skip over
                    logger.debug("snapshot status request ignoring snapshot [{}], not found in repository [{}]", snapshotName, repositoryName);
                    continue;
                } else {
                    throw new SnapshotMissingException(repositoryName, snapshotName);
                }
            }
            SnapshotInfo snapshotInfo = snapshot(snapshotsInProgress, repositoryName, snapshotId);
            List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
            if (snapshotInfo.state().completed()) {
                Map<ShardId, IndexShardSnapshotStatus> shardStatuses = snapshotShards(repositoryName, repositoryData, snapshotInfo);
                for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatuses.entrySet()) {
                    IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue().asCopy();
                    shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus));
                }
                final SnapshotsInProgress.State state;
                switch(snapshotInfo.state()) {
                    case FAILED:
                        state = SnapshotsInProgress.State.FAILED;
                        break;
                    case SUCCESS:
                    case PARTIAL:
                        // Translating both PARTIAL and SUCCESS to SUCCESS for now
                        // TODO: add the differentiation on the metadata level in the next major release
                        state = SnapshotsInProgress.State.SUCCESS;
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                }
                final long startTime = snapshotInfo.startTime();
                final long endTime = snapshotInfo.endTime();
                assert endTime >= startTime || (endTime == 0L && snapshotInfo.state().completed() == false) : "Inconsistent timestamps found in SnapshotInfo [" + snapshotInfo + "]";
                builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder), snapshotInfo.includeGlobalState(), startTime, // Use current time to calculate overall runtime for in-progress snapshots that have endTime == 0
                (endTime == 0 ? threadPool.absoluteTimeInMillis() : endTime) - startTime));
            }
        }
        listener.onResponse(new SnapshotsStatusResponse(Collections.unmodifiableList(builder)));
    }, listener::onFailure);
}
Also used : Arrays(java.util.Arrays) SnapshotState(org.opensearch.snapshots.SnapshotState) SnapshotsService(org.opensearch.snapshots.SnapshotsService) Strings(org.opensearch.common.Strings) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) IndexId(org.opensearch.repositories.IndexId) Map(java.util.Map) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) Repository(org.opensearch.repositories.Repository) CollectionUtils(org.opensearch.common.util.CollectionUtils) SnapshotId(org.opensearch.snapshots.SnapshotId) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Set(java.util.Set) ObjectCursor(com.carrotsearch.hppc.cursors.ObjectCursor) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) SnapshotShardFailure(org.opensearch.snapshots.SnapshotShardFailure) StepListener(org.opensearch.action.StepListener) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) RepositoriesService(org.opensearch.repositories.RepositoriesService) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) SnapshotShardsService(org.opensearch.snapshots.SnapshotShardsService) Function(java.util.function.Function) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IOException(java.io.IOException) ShardId(org.opensearch.index.shard.ShardId) Sets(org.opensearch.common.util.set.Sets) Snapshot(org.opensearch.snapshots.Snapshot) ClusterService(org.opensearch.cluster.service.ClusterService) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) ArrayList(java.util.ArrayList) ShardId(org.opensearch.index.shard.ShardId) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) Snapshot(org.opensearch.snapshots.Snapshot) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) IndexShardSnapshotStatus(org.opensearch.index.snapshots.IndexShardSnapshotStatus) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) StepListener(org.opensearch.action.StepListener) Map(java.util.Map) HashMap(java.util.HashMap) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap)

Aggregations

SnapshotsInProgress (org.opensearch.cluster.SnapshotsInProgress)3 ClusterService (org.opensearch.cluster.service.ClusterService)3 SnapshotMissingException (org.opensearch.snapshots.SnapshotMissingException)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Map (java.util.Map)2 Set (java.util.Set)2 Collectors (java.util.stream.Collectors)2 LogManager (org.apache.logging.log4j.LogManager)2 Logger (org.apache.logging.log4j.Logger)2 ActionListener (org.opensearch.action.ActionListener)2 ActionFilters (org.opensearch.action.support.ActionFilters)2 TransportMasterNodeAction (org.opensearch.action.support.master.TransportMasterNodeAction)2 ClusterState (org.opensearch.cluster.ClusterState)2 ClusterBlockException (org.opensearch.cluster.block.ClusterBlockException)2 ClusterBlockLevel (org.opensearch.cluster.block.ClusterBlockLevel)2