Examples with RepositoryData org.opensearch.repositories.RepositoryData used on opensource projects

Example 11 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreTestUtil method assertIndexUUIDs.

private static void assertIndexUUIDs(BlobStoreRepository repository, RepositoryData repositoryData) throws IOException {
    final List<String> expectedIndexUUIDs = repositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toList());
    final BlobContainer indicesContainer = repository.blobContainer().children().get("indices");
    final List<String> foundIndexUUIDs;
    if (indicesContainer == null) {
        foundIndexUUIDs = Collections.emptyList();
    } else {
        // Skip Lucene MockFS extraN directory
        foundIndexUUIDs = indicesContainer.children().keySet().stream().filter(s -> s.startsWith("extra") == false).collect(Collectors.toList());
    }
    assertThat(foundIndexUUIDs, containsInAnyOrder(expectedIndexUUIDs.toArray(Strings.EMPTY_ARRAY)));
    for (String indexId : foundIndexUUIDs) {
        final Set<String> indexMetaGenerationsFound = indicesContainer.children().get(indexId).listBlobsByPrefix(BlobStoreRepository.METADATA_PREFIX).keySet().stream().map(p -> p.replace(BlobStoreRepository.METADATA_PREFIX, "").replace(".dat", "")).collect(Collectors.toSet());
        final Set<String> indexMetaGenerationsExpected = new HashSet<>();
        final IndexId idx = repositoryData.getIndices().values().stream().filter(i -> i.getId().equals(indexId)).findFirst().get();
        for (SnapshotId snapshotId : repositoryData.getSnapshots(idx)) {
            indexMetaGenerationsExpected.add(repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotId, idx));
        }
        // TODO: assertEquals(indexMetaGenerationsExpected, indexMetaGenerationsFound); requires cleanup functionality for
        // index meta generations blobs
        assertTrue(indexMetaGenerationsFound.containsAll(indexMetaGenerationsExpected));
    }
}

Example 12 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class OpenSearchBlobStoreRepositoryIntegTestCase method testIndicesDeletedFromRepository.

public void testIndicesDeletedFromRepository() throws Exception {
    final String repoName = createRepository("test-repo");
    Client client = client();
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 20; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    logger.info("--> take a snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap").setWaitForCompletion(true).get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> indexing more data");
    for (int i = 20; i < 40; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    logger.info("--> take another snapshot with only 2 of the 3 indices");
    createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap2").setWaitForCompletion(true).setIndices("test-idx-1", "test-idx-2").get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> delete a snapshot");
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "test-snap").get());
    logger.info("--> verify index folder deleted from blob container");
    RepositoriesService repositoriesSvc = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
    BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
    final SetOnce<BlobContainer> indicesBlobContainer = new SetOnce<>();
    final PlainActionFuture<RepositoryData> repositoryData = PlainActionFuture.newFuture();
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
        indicesBlobContainer.set(repository.blobStore().blobContainer(repository.basePath().add("indices")));
        repository.getRepositoryData(repositoryData);
    });
    for (IndexId indexId : repositoryData.actionGet().getIndices().values()) {
        if (indexId.getName().equals("test-idx-3")) {
            // deleted index
            assertFalse(indicesBlobContainer.get().blobExists(indexId.getId()));
        }
    }
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "test-snap2").get());
}

Example 13 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreTestUtil method assertSnapshotUUIDs.

private static void assertSnapshotUUIDs(BlobStoreRepository repository, RepositoryData repositoryData) throws IOException {
    final BlobContainer repoRoot = repository.blobContainer();
    final Collection<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
    final List<String> expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).collect(Collectors.toList());
    for (String prefix : new String[] { BlobStoreRepository.SNAPSHOT_PREFIX, BlobStoreRepository.METADATA_PREFIX }) {
        final Collection<String> foundSnapshotUUIDs = repoRoot.listBlobs().keySet().stream().filter(p -> p.startsWith(prefix)).map(p -> p.replace(prefix, "").replace(".dat", "")).collect(Collectors.toSet());
        assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY)));
    }
    final BlobContainer indicesContainer = repository.getBlobContainer().children().get("indices");
    final Map<String, BlobContainer> indices;
    if (indicesContainer == null) {
        indices = Collections.emptyMap();
    } else {
        indices = indicesContainer.children();
    }
    final Map<IndexId, Integer> maxShardCountsExpected = new HashMap<>();
    final Map<IndexId, Integer> maxShardCountsSeen = new HashMap<>();
    // Assert that for each snapshot, the relevant metadata was written to index and shard folders
    for (SnapshotId snapshotId : snapshotIds) {
        final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
        for (String index : snapshotInfo.indices()) {
            final IndexId indexId = repositoryData.resolveIndexId(index);
            assertThat(indices, hasKey(indexId.getId()));
            final BlobContainer indexContainer = indices.get(indexId.getId());
            assertThat(indexContainer.listBlobs(), hasKey(String.format(Locale.ROOT, BlobStoreRepository.METADATA_NAME_FORMAT, repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotId, indexId))));
            final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId);
            for (Map.Entry<String, BlobContainer> entry : indexContainer.children().entrySet()) {
                // Skip Lucene MockFS extraN directory
                if (entry.getKey().startsWith("extra")) {
                    continue;
                }
                final int shardId = Integer.parseInt(entry.getKey());
                final int shardCount = indexMetadata.getNumberOfShards();
                maxShardCountsExpected.compute(indexId, (i, existing) -> existing == null || existing < shardCount ? shardCount : existing);
                final BlobContainer shardContainer = entry.getValue();
                // becomes unreferenced. We should fix that and remove this conditional once its fixed.
                if (shardContainer.listBlobs().keySet().stream().anyMatch(blob -> blob.startsWith("extra") == false)) {
                    final int impliedCount = shardId - 1;
                    maxShardCountsSeen.compute(indexId, (i, existing) -> existing == null || existing < impliedCount ? impliedCount : existing);
                }
                if (shardId < shardCount && snapshotInfo.shardFailures().stream().noneMatch(shardFailure -> shardFailure.index().equals(index) && shardFailure.shardId() == shardId)) {
                    final Map<String, BlobMetadata> shardPathContents = shardContainer.listBlobs();
                    assertThat(shardPathContents, hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())));
                    assertThat(shardPathContents.keySet().stream().filter(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)).count(), lessThanOrEqualTo(2L));
                }
            }
        }
    }
    maxShardCountsSeen.forEach(((indexId, count) -> assertThat("Found unreferenced shard paths for index [" + indexId + "]", count, lessThanOrEqualTo(maxShardCountsExpected.get(indexId)))));
}

Example 14 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreTestUtil method assertConsistency.

/**
 * Assert that there are no unreferenced indices or unreferenced root-level metadata blobs in any repository.
 * TODO: Expand the logic here to also check for unreferenced segment blobs and shard level metadata
 * @param repository BlobStoreRepository to check
 * @param executor Executor to run all repository calls on. This is needed since the production {@link BlobStoreRepository}
 *                 implementations assert that all IO inducing calls happen on the generic or snapshot thread-pools and hence callers
 *                 of this assertion must pass an executor on those when using such an implementation.
 */
public static void assertConsistency(BlobStoreRepository repository, Executor executor) {
    final PlainActionFuture<AssertionError> listener = PlainActionFuture.newFuture();
    executor.execute(ActionRunnable.supply(listener, () -> {
        try {
            final BlobContainer blobContainer = repository.blobContainer();
            final long latestGen;
            try (DataInputStream inputStream = new DataInputStream(blobContainer.readBlob("index.latest"))) {
                latestGen = inputStream.readLong();
            } catch (NoSuchFileException e) {
                throw new AssertionError("Could not find index.latest blob for repo [" + repository + "]");
            }
            assertIndexGenerations(blobContainer, latestGen);
            final RepositoryData repositoryData;
            try (InputStream blob = blobContainer.readBlob(BlobStoreRepository.INDEX_FILE_PREFIX + latestGen);
                XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, blob)) {
                repositoryData = RepositoryData.snapshotsFromXContent(parser, latestGen, false);
            }
            assertIndexUUIDs(repository, repositoryData);
            assertSnapshotUUIDs(repository, repositoryData);
            assertShardIndexGenerations(blobContainer, repositoryData.shardGenerations());
            return null;
        } catch (AssertionError e) {
            return e;
        }
    }));
    final AssertionError err = listener.actionGet(TimeValue.timeValueMinutes(1L));
    if (err != null) {
        throw new AssertionError(err);
    }
}

Example 15 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class AbstractSnapshotIntegTestCase method initWithSnapshotVersion.

/**
 * Workaround to simulate BwC situation: taking a snapshot without indices here so that we don't create any new version shard
 * generations (the existence of which would short-circuit checks for the repo containing old version snapshots)
 */
protected String initWithSnapshotVersion(String repoName, Path repoPath, Version version) throws IOException {
    assertThat("This hack only works on an empty repository", getRepositoryData(repoName).getSnapshotIds(), empty());
    final String oldVersionSnapshot = OLD_VERSION_SNAPSHOT_PREFIX + version.id;
    final CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(repoName, oldVersionSnapshot).setIndices("does-not-exist-for-sure-*").setWaitForCompletion(true).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), is(0));
    logger.info("--> writing downgraded RepositoryData for repository metadata version [{}]", version);
    final RepositoryData repositoryData = getRepositoryData(repoName);
    final XContentBuilder jsonBuilder = JsonXContent.contentBuilder();
    repositoryData.snapshotsToXContent(jsonBuilder, version);
    final RepositoryData downgradedRepoData = RepositoryData.snapshotsFromXContent(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, Strings.toString(jsonBuilder).replace(Version.CURRENT.toString(), version.toString())), repositoryData.getGenId(), randomBoolean());
    Files.write(repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryData.getGenId()), BytesReference.toBytes(BytesReference.bytes(downgradedRepoData.snapshotsToXContent(XContentFactory.jsonBuilder(), version))), StandardOpenOption.TRUNCATE_EXISTING);
    return oldVersionSnapshot;
}