Examples with SnapshotInfo org.opensearch.snapshots.SnapshotInfo used on opensource projects

Example 6 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method masterOperation.

@Override
protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener<GetSnapshotsResponse> listener) {
    try {
        final String repository = request.repository();
        final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE);
        final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
        final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
        for (SnapshotInfo snapshotInfo : sortedCurrentSnapshots(snapshotsInProgress, repository)) {
            SnapshotId snapshotId = snapshotInfo.snapshotId();
            allSnapshotIds.put(snapshotId.getName(), snapshotId);
            currentSnapshots.add(snapshotInfo);
        }
        final RepositoryData repositoryData;
        if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
            repositoryData = PlainActionFuture.get(fut -> repositoriesService.getRepositoryData(repository, fut));
            for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
                allSnapshotIds.put(snapshotId.getName(), snapshotId);
            }
        } else {
            repositoryData = null;
        }
        final Set<SnapshotId> toResolve = new HashSet<>();
        if (isAllSnapshots(request.snapshots())) {
            toResolve.addAll(allSnapshotIds.values());
        } else {
            for (String snapshotOrPattern : request.snapshots()) {
                if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
                    toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
                } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                    if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                        toResolve.add(allSnapshotIds.get(snapshotOrPattern));
                    } else if (request.ignoreUnavailable() == false) {
                        throw new SnapshotMissingException(repository, snapshotOrPattern);
                    }
                } else {
                    for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
                        if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
                            toResolve.add(entry.getValue());
                        }
                    }
                }
            }
            if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
                throw new SnapshotMissingException(repository, request.snapshots()[0]);
            }
        }
        final List<SnapshotInfo> snapshotInfos;
        if (request.verbose()) {
            snapshotInfos = snapshots(snapshotsInProgress, repository, new ArrayList<>(toResolve), request.ignoreUnavailable());
        } else {
            if (repositoryData != null) {
                // want non-current snapshots as well, which are found in the repository data
                snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
            } else {
                // only want current snapshots
                snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
                CollectionUtil.timSort(snapshotInfos);
            }
        }
        listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
    } catch (Exception e) {
        listener.onFailure(e);
    }
}

Example 7 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testOverwriteSnapshotInfoBlob.

public void testOverwriteSnapshotInfoBlob() throws Exception {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    final RepositoryMetadata metadata = new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY);
    final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(metadata);
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(metadata, xContentRegistry(), clusterService, recoverySettings, blobStoreContext, random())) {
        clusterService.addStateApplier(event -> repository.updateState(event.state()));
        // Apply state once to initialize repo properly like RepositoriesService would
        repository.updateState(clusterService.state());
        repository.start();
        // We create a snap- blob for snapshot "foo" in the first generation
        final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID());
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs.
        final AssertionError assertionError = expectThrows(AssertionError.class, () -> PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f)));
        assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n     but: was <5>"));
        // We try to write yet another snap- blob for "foo" in the next generation.
        // It passes cleanly because the content of the blob except for the timestamps.
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
    }
}

Example 8 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class BlobStoreTestUtil method assertSnapshotUUIDs.

private static void assertSnapshotUUIDs(BlobStoreRepository repository, RepositoryData repositoryData) throws IOException {
    final BlobContainer repoRoot = repository.blobContainer();
    final Collection<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
    final List<String> expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).collect(Collectors.toList());
    for (String prefix : new String[] { BlobStoreRepository.SNAPSHOT_PREFIX, BlobStoreRepository.METADATA_PREFIX }) {
        final Collection<String> foundSnapshotUUIDs = repoRoot.listBlobs().keySet().stream().filter(p -> p.startsWith(prefix)).map(p -> p.replace(prefix, "").replace(".dat", "")).collect(Collectors.toSet());
        assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY)));
    }
    final BlobContainer indicesContainer = repository.getBlobContainer().children().get("indices");
    final Map<String, BlobContainer> indices;
    if (indicesContainer == null) {
        indices = Collections.emptyMap();
    } else {
        indices = indicesContainer.children();
    }
    final Map<IndexId, Integer> maxShardCountsExpected = new HashMap<>();
    final Map<IndexId, Integer> maxShardCountsSeen = new HashMap<>();
    // Assert that for each snapshot, the relevant metadata was written to index and shard folders
    for (SnapshotId snapshotId : snapshotIds) {
        final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
        for (String index : snapshotInfo.indices()) {
            final IndexId indexId = repositoryData.resolveIndexId(index);
            assertThat(indices, hasKey(indexId.getId()));
            final BlobContainer indexContainer = indices.get(indexId.getId());
            assertThat(indexContainer.listBlobs(), hasKey(String.format(Locale.ROOT, BlobStoreRepository.METADATA_NAME_FORMAT, repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotId, indexId))));
            final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId);
            for (Map.Entry<String, BlobContainer> entry : indexContainer.children().entrySet()) {
                // Skip Lucene MockFS extraN directory
                if (entry.getKey().startsWith("extra")) {
                    continue;
                }
                final int shardId = Integer.parseInt(entry.getKey());
                final int shardCount = indexMetadata.getNumberOfShards();
                maxShardCountsExpected.compute(indexId, (i, existing) -> existing == null || existing < shardCount ? shardCount : existing);
                final BlobContainer shardContainer = entry.getValue();
                // becomes unreferenced. We should fix that and remove this conditional once its fixed.
                if (shardContainer.listBlobs().keySet().stream().anyMatch(blob -> blob.startsWith("extra") == false)) {
                    final int impliedCount = shardId - 1;
                    maxShardCountsSeen.compute(indexId, (i, existing) -> existing == null || existing < impliedCount ? impliedCount : existing);
                }
                if (shardId < shardCount && snapshotInfo.shardFailures().stream().noneMatch(shardFailure -> shardFailure.index().equals(index) && shardFailure.shardId() == shardId)) {
                    final Map<String, BlobMetadata> shardPathContents = shardContainer.listBlobs();
                    assertThat(shardPathContents, hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())));
                    assertThat(shardPathContents.keySet().stream().filter(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)).count(), lessThanOrEqualTo(2L));
                }
            }
        }
    }
    maxShardCountsSeen.forEach(((indexId, count) -> assertThat("Found unreferenced shard paths for index [" + indexId + "]", count, lessThanOrEqualTo(maxShardCountsExpected.get(indexId)))));
}

Example 9 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class CreateSnapshotResponseTests method createTestInstance.

@Override
protected CreateSnapshotResponse createTestInstance() {
    SnapshotId snapshotId = new SnapshotId("test", UUID.randomUUID().toString());
    List<String> indices = new ArrayList<>();
    indices.add("test0");
    indices.add("test1");
    List<String> dataStreams = new ArrayList<>();
    dataStreams.add("test0");
    dataStreams.add("test1");
    String reason = "reason";
    long startTime = System.currentTimeMillis();
    long endTime = startTime + 10000;
    int totalShards = randomIntBetween(1, 3);
    int successfulShards = randomIntBetween(0, totalShards);
    List<SnapshotShardFailure> shardFailures = new ArrayList<>();
    for (int count = successfulShards; count < totalShards; ++count) {
        shardFailures.add(new SnapshotShardFailure("node-id", new ShardId("index-" + count, UUID.randomUUID().toString(), randomInt()), "reason"));
    }
    boolean globalState = randomBoolean();
    return new CreateSnapshotResponse(new SnapshotInfo(snapshotId, indices, dataStreams, startTime, reason, endTime, totalShards, shardFailures, globalState, SnapshotInfoTests.randomUserMetadata()));
}

Example 10 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class GetSnapshotsResponseTests method createTestInstance.

@Override
protected GetSnapshotsResponse createTestInstance() {
    ArrayList<SnapshotInfo> snapshots = new ArrayList<>();
    for (int i = 0; i < randomIntBetween(5, 10); ++i) {
        SnapshotId snapshotId = new SnapshotId("snapshot " + i, UUIDs.base64UUID());
        String reason = randomBoolean() ? null : "reason";
        ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2);
        List<SnapshotShardFailure> shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason"));
        snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), System.currentTimeMillis(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), SnapshotInfoTests.randomUserMetadata()));
    }
    return new GetSnapshotsResponse(snapshots);
}