Search in sources :

Example 6 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method masterOperation.

@Override
protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener<GetSnapshotsResponse> listener) {
    try {
        final String repository = request.repository();
        final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE);
        final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
        final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
        for (SnapshotInfo snapshotInfo : sortedCurrentSnapshots(snapshotsInProgress, repository)) {
            SnapshotId snapshotId = snapshotInfo.snapshotId();
            allSnapshotIds.put(snapshotId.getName(), snapshotId);
            currentSnapshots.add(snapshotInfo);
        }
        final RepositoryData repositoryData;
        if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
            repositoryData = PlainActionFuture.get(fut -> repositoriesService.getRepositoryData(repository, fut));
            for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
                allSnapshotIds.put(snapshotId.getName(), snapshotId);
            }
        } else {
            repositoryData = null;
        }
        final Set<SnapshotId> toResolve = new HashSet<>();
        if (isAllSnapshots(request.snapshots())) {
            toResolve.addAll(allSnapshotIds.values());
        } else {
            for (String snapshotOrPattern : request.snapshots()) {
                if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
                    toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
                } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                    if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                        toResolve.add(allSnapshotIds.get(snapshotOrPattern));
                    } else if (request.ignoreUnavailable() == false) {
                        throw new SnapshotMissingException(repository, snapshotOrPattern);
                    }
                } else {
                    for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
                        if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
                            toResolve.add(entry.getValue());
                        }
                    }
                }
            }
            if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
                throw new SnapshotMissingException(repository, request.snapshots()[0]);
            }
        }
        final List<SnapshotInfo> snapshotInfos;
        if (request.verbose()) {
            snapshotInfos = snapshots(snapshotsInProgress, repository, new ArrayList<>(toResolve), request.ignoreUnavailable());
        } else {
            if (repositoryData != null) {
                // want non-current snapshots as well, which are found in the repository data
                snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
            } else {
                // only want current snapshots
                snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
                CollectionUtil.timSort(snapshotInfos);
            }
        }
        listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
Also used : RepositoriesService(org.opensearch.repositories.RepositoriesService) Collections.unmodifiableList(java.util.Collections.unmodifiableList) ThreadPool(org.opensearch.threadpool.ThreadPool) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) SnapshotsService(org.opensearch.snapshots.SnapshotsService) Regex(org.opensearch.common.regex.Regex) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) IndexId(org.opensearch.repositories.IndexId) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) SnapshotException(org.opensearch.snapshots.SnapshotException) Map(java.util.Map) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) Repository(org.opensearch.repositories.Repository) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Set(java.util.Set) IOException(java.io.IOException) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) CollectionUtil(org.apache.lucene.util.CollectionUtil) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ClusterService(org.opensearch.cluster.service.ClusterService) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotException(org.opensearch.snapshots.SnapshotException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) SnapshotMissingException(org.opensearch.snapshots.SnapshotMissingException) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) HashSet(java.util.HashSet)

Example 7 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testOverwriteSnapshotInfoBlob.

public void testOverwriteSnapshotInfoBlob() throws Exception {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    final RepositoryMetadata metadata = new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY);
    final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(metadata);
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(metadata, xContentRegistry(), clusterService, recoverySettings, blobStoreContext, random())) {
        clusterService.addStateApplier(event -> repository.updateState(event.state()));
        // Apply state once to initialize repo properly like RepositoriesService would
        repository.updateState(clusterService.state());
        repository.start();
        // We create a snap- blob for snapshot "foo" in the first generation
        final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID());
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs.
        final AssertionError assertionError = expectThrows(AssertionError.class, () -> PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f)));
        assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n     but: was <5>"));
        // We try to write yet another snap- blob for "foo" in the next generation.
        // It passes cleanly because the content of the blob except for the timestamps.
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
    }
}
Also used : ShardGenerations(org.opensearch.repositories.ShardGenerations) NoSuchFileException(java.nio.file.NoSuchFileException) Arrays(java.util.Arrays) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) Metadata(org.opensearch.cluster.metadata.Metadata) BlobContainer(org.opensearch.common.blobstore.BlobContainer) Version(org.opensearch.Version) Function(java.util.function.Function) ByteArrayInputStream(java.io.ByteArrayInputStream) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) UUIDs(org.opensearch.common.UUIDs) ClusterSettings(org.opensearch.common.settings.ClusterSettings) RecoverySettings(org.opensearch.indices.recovery.RecoverySettings) RepositoryData(org.opensearch.repositories.RepositoryData) BlobStoreTestUtil(org.opensearch.repositories.blobstore.BlobStoreTestUtil) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) Settings(org.opensearch.common.settings.Settings) IOException(java.io.IOException) Matchers.startsWith(org.hamcrest.Matchers.startsWith) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ClusterService(org.opensearch.cluster.service.ClusterService) Collections(java.util.Collections) InputStream(java.io.InputStream) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ClusterService(org.opensearch.cluster.service.ClusterService) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) NoSuchFileException(java.nio.file.NoSuchFileException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData)

Example 8 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class BlobStoreTestUtil method assertSnapshotUUIDs.

private static void assertSnapshotUUIDs(BlobStoreRepository repository, RepositoryData repositoryData) throws IOException {
    final BlobContainer repoRoot = repository.blobContainer();
    final Collection<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
    final List<String> expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).collect(Collectors.toList());
    for (String prefix : new String[] { BlobStoreRepository.SNAPSHOT_PREFIX, BlobStoreRepository.METADATA_PREFIX }) {
        final Collection<String> foundSnapshotUUIDs = repoRoot.listBlobs().keySet().stream().filter(p -> p.startsWith(prefix)).map(p -> p.replace(prefix, "").replace(".dat", "")).collect(Collectors.toSet());
        assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY)));
    }
    final BlobContainer indicesContainer = repository.getBlobContainer().children().get("indices");
    final Map<String, BlobContainer> indices;
    if (indicesContainer == null) {
        indices = Collections.emptyMap();
    } else {
        indices = indicesContainer.children();
    }
    final Map<IndexId, Integer> maxShardCountsExpected = new HashMap<>();
    final Map<IndexId, Integer> maxShardCountsSeen = new HashMap<>();
    // Assert that for each snapshot, the relevant metadata was written to index and shard folders
    for (SnapshotId snapshotId : snapshotIds) {
        final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
        for (String index : snapshotInfo.indices()) {
            final IndexId indexId = repositoryData.resolveIndexId(index);
            assertThat(indices, hasKey(indexId.getId()));
            final BlobContainer indexContainer = indices.get(indexId.getId());
            assertThat(indexContainer.listBlobs(), hasKey(String.format(Locale.ROOT, BlobStoreRepository.METADATA_NAME_FORMAT, repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotId, indexId))));
            final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId);
            for (Map.Entry<String, BlobContainer> entry : indexContainer.children().entrySet()) {
                // Skip Lucene MockFS extraN directory
                if (entry.getKey().startsWith("extra")) {
                    continue;
                }
                final int shardId = Integer.parseInt(entry.getKey());
                final int shardCount = indexMetadata.getNumberOfShards();
                maxShardCountsExpected.compute(indexId, (i, existing) -> existing == null || existing < shardCount ? shardCount : existing);
                final BlobContainer shardContainer = entry.getValue();
                // becomes unreferenced. We should fix that and remove this conditional once its fixed.
                if (shardContainer.listBlobs().keySet().stream().anyMatch(blob -> blob.startsWith("extra") == false)) {
                    final int impliedCount = shardId - 1;
                    maxShardCountsSeen.compute(indexId, (i, existing) -> existing == null || existing < impliedCount ? impliedCount : existing);
                }
                if (shardId < shardCount && snapshotInfo.shardFailures().stream().noneMatch(shardFailure -> shardFailure.index().equals(index) && shardFailure.shardId() == shardId)) {
                    final Map<String, BlobMetadata> shardPathContents = shardContainer.listBlobs();
                    assertThat(shardPathContents, hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())));
                    assertThat(shardPathContents.keySet().stream().filter(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)).count(), lessThanOrEqualTo(2L));
                }
            }
        }
    }
    maxShardCountsSeen.forEach(((indexId, count) -> assertThat("Found unreferenced shard paths for index [" + indexId + "]", count, lessThanOrEqualTo(maxShardCountsExpected.get(indexId)))));
}
Also used : ShardGenerations(org.opensearch.repositories.ShardGenerations) NoSuchFileException(java.nio.file.NoSuchFileException) OpenSearchTestCase.buildNewFakeTransportAddress(org.opensearch.test.OpenSearchTestCase.buildNewFakeTransportAddress) Metadata(org.opensearch.cluster.metadata.Metadata) Matchers.not(org.hamcrest.Matchers.not) Version(org.opensearch.Version) ClusterStateApplier(org.opensearch.cluster.ClusterStateApplier) Matchers.hasKey(org.hamcrest.Matchers.hasKey) Strings(org.opensearch.common.Strings) XContentParser(org.opensearch.common.xcontent.XContentParser) Assert.assertThat(org.junit.Assert.assertThat) ClusterApplierService(org.opensearch.cluster.service.ClusterApplierService) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexId(org.opensearch.repositories.IndexId) ByteArrayInputStream(java.io.ByteArrayInputStream) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) Locale(java.util.Locale) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) OpenSearchTestCase.randomIntBetween(org.opensearch.test.OpenSearchTestCase.randomIntBetween) BlobMetadata(org.opensearch.common.blobstore.BlobMetadata) TimeValue(org.opensearch.common.unit.TimeValue) SnapshotId(org.opensearch.snapshots.SnapshotId) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Collection(java.util.Collection) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) Set(java.util.Set) Collectors(java.util.stream.Collectors) BlobStore(org.opensearch.common.blobstore.BlobStore) List(java.util.List) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) XContentType(org.opensearch.common.xcontent.XContentType) Mockito.any(org.mockito.Mockito.any) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) Mockito.mock(org.mockito.Mockito.mock) RepositoriesService(org.opensearch.repositories.RepositoriesService) DataInputStream(java.io.DataInputStream) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) BlobContainer(org.opensearch.common.blobstore.BlobContainer) HashMap(java.util.HashMap) InternalTestCluster(org.opensearch.test.InternalTestCluster) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) Mockito.anyString(org.mockito.Mockito.anyString) RepositoryData(org.opensearch.repositories.RepositoryData) Matchers.empty(org.hamcrest.Matchers.empty) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) BlobPath(org.opensearch.common.blobstore.BlobPath) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) SameThreadExecutorService(org.apache.lucene.util.SameThreadExecutorService) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) InputStream(java.io.InputStream) IndexId(org.opensearch.repositories.IndexId) HashMap(java.util.HashMap) Mockito.anyString(org.mockito.Mockito.anyString) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) BlobMetadata(org.opensearch.common.blobstore.BlobMetadata) BlobContainer(org.opensearch.common.blobstore.BlobContainer) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) Map(java.util.Map) HashMap(java.util.HashMap)

Example 9 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class CreateSnapshotResponseTests method createTestInstance.

@Override
protected CreateSnapshotResponse createTestInstance() {
    SnapshotId snapshotId = new SnapshotId("test", UUID.randomUUID().toString());
    List<String> indices = new ArrayList<>();
    indices.add("test0");
    indices.add("test1");
    List<String> dataStreams = new ArrayList<>();
    dataStreams.add("test0");
    dataStreams.add("test1");
    String reason = "reason";
    long startTime = System.currentTimeMillis();
    long endTime = startTime + 10000;
    int totalShards = randomIntBetween(1, 3);
    int successfulShards = randomIntBetween(0, totalShards);
    List<SnapshotShardFailure> shardFailures = new ArrayList<>();
    for (int count = successfulShards; count < totalShards; ++count) {
        shardFailures.add(new SnapshotShardFailure("node-id", new ShardId("index-" + count, UUID.randomUUID().toString(), randomInt()), "reason"));
    }
    boolean globalState = randomBoolean();
    return new CreateSnapshotResponse(new SnapshotInfo(snapshotId, indices, dataStreams, startTime, reason, endTime, totalShards, shardFailures, globalState, SnapshotInfoTests.randomUserMetadata()));
}
Also used : ShardId(org.opensearch.index.shard.ShardId) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotShardFailure(org.opensearch.snapshots.SnapshotShardFailure) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ArrayList(java.util.ArrayList)

Example 10 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class GetSnapshotsResponseTests method createTestInstance.

@Override
protected GetSnapshotsResponse createTestInstance() {
    ArrayList<SnapshotInfo> snapshots = new ArrayList<>();
    for (int i = 0; i < randomIntBetween(5, 10); ++i) {
        SnapshotId snapshotId = new SnapshotId("snapshot " + i, UUIDs.base64UUID());
        String reason = randomBoolean() ? null : "reason";
        ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2);
        List<SnapshotShardFailure> shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason"));
        snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), System.currentTimeMillis(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), SnapshotInfoTests.randomUserMetadata()));
    }
    return new GetSnapshotsResponse(snapshots);
}
Also used : ShardId(org.opensearch.index.shard.ShardId) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) SnapshotShardFailure(org.opensearch.snapshots.SnapshotShardFailure) ArrayList(java.util.ArrayList)

Aggregations

SnapshotInfo (org.opensearch.snapshots.SnapshotInfo)22 SnapshotId (org.opensearch.snapshots.SnapshotId)12 ArrayList (java.util.ArrayList)9 IOException (java.io.IOException)8 ClusterService (org.opensearch.cluster.service.ClusterService)7 Repository (org.opensearch.repositories.Repository)7 RepositoryData (org.opensearch.repositories.RepositoryData)7 Collections (java.util.Collections)6 List (java.util.List)6 Map (java.util.Map)6 ClusterState (org.opensearch.cluster.ClusterState)6 SnapshotsInProgress (org.opensearch.cluster.SnapshotsInProgress)6 IndexId (org.opensearch.repositories.IndexId)6 HashMap (java.util.HashMap)5 Set (java.util.Set)5 Collectors (java.util.stream.Collectors)5 RepositoriesService (org.opensearch.repositories.RepositoriesService)5 SnapshotMissingException (org.opensearch.snapshots.SnapshotMissingException)5 HashSet (java.util.HashSet)4 LogManager (org.apache.logging.log4j.LogManager)4