Examples with RepositoryData org.opensearch.repositories.RepositoryData used on opensource projects

Example 6 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class SharedClusterSnapshotRestoreIT method testRestoreSnapshotWithCorruptedIndexMetadata.

/**
 * Tests that a snapshot of multiple indices including one with a corrupted index metadata
 * file can still be used to restore the non corrupted indices
 */
public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception {
    final Client client = client();
    final Path repo = randomRepoPath();
    final int nbIndices = randomIntBetween(2, 3);
    final Map<String, Integer> nbDocsPerIndex = new HashMap<>();
    for (int i = 0; i < nbIndices; i++) {
        String indexName = "test-idx-" + i;
        assertAcked(prepareCreate(indexName).setSettings(indexSettingsNoReplicas(Math.min(2, numberOfShards()))));
        int nbDocs = randomIntBetween(1, 10);
        nbDocsPerIndex.put(indexName, nbDocs);
        IndexRequestBuilder[] documents = new IndexRequestBuilder[nbDocs];
        for (int j = 0; j < nbDocs; j++) {
            documents[j] = client.prepareIndex(indexName).setSource("foo", "bar");
        }
        indexRandom(true, documents);
    }
    flushAndRefresh();
    createRepository("test-repo", "fs", repo);
    final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap");
    assertThat(snapshotInfo.indices(), hasSize(nbIndices));
    final RepositoryData repositoryData = getRepositoryData("test-repo");
    final Map<String, IndexId> indexIds = repositoryData.getIndices();
    assertThat(indexIds.size(), equalTo(nbIndices));
    // Choose a random index from the snapshot
    final IndexId corruptedIndex = randomFrom(indexIds.values());
    final Path indexMetadataPath = repo.resolve("indices").resolve(corruptedIndex.getId()).resolve("meta-" + repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotInfo.snapshotId(), corruptedIndex) + ".dat");
    // Truncate the index metadata file
    try (SeekableByteChannel outChan = Files.newByteChannel(indexMetadataPath, StandardOpenOption.WRITE)) {
        outChan.truncate(randomInt(10));
    }
    List<SnapshotInfo> snapshotInfos = clusterAdmin().prepareGetSnapshots("test-repo").get().getSnapshots();
    assertThat(snapshotInfos.size(), equalTo(1));
    assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
    assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap"));
    assertAcked(client().admin().indices().prepareDelete(nbDocsPerIndex.keySet().toArray(new String[nbDocsPerIndex.size()])));
    Predicate<String> isRestorableIndex = index -> corruptedIndex.getName().equals(index) == false;
    clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setIndices(nbDocsPerIndex.keySet().stream().filter(isRestorableIndex).toArray(String[]::new)).setRestoreGlobalState(randomBoolean()).setWaitForCompletion(true).get();
    ensureGreen();
    for (Map.Entry<String, Integer> entry : nbDocsPerIndex.entrySet()) {
        if (isRestorableIndex.test(entry.getKey())) {
            assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().longValue());
        }
    }
    assertAcked(startDeleteSnapshot("test-repo", snapshotInfo.snapshotId().getName()).get());
}

Example 7 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testConcurrentlyChangeRepositoryContentsInBwCMode.

public void testConcurrentlyChangeRepositoryContentsInBwCMode() throws Exception {
    Client client = client();
    Path repo = randomRepoPath();
    final String repoName = "test-repo";
    logger.info("-->  creating repository at {}", repo.toAbsolutePath());
    assertAcked(client.admin().cluster().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", repo).put("compress", false).put(BlobStoreRepository.ALLOW_CONCURRENT_MODIFICATION.getKey(), true).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    createIndex("test-idx-1", "test-idx-2");
    logger.info("--> indexing some data");
    indexRandom(true, client().prepareIndex().setIndex("test-idx-1").setSource("foo", "bar"), client().prepareIndex().setIndex("test-idx-2").setSource("foo", "bar"));
    final String snapshot = "test-snap";
    logger.info("--> creating snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, snapshot).setWaitForCompletion(true).setIndices("test-idx-*").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    final Repository repository = internalCluster().getMasterNodeInstance(RepositoriesService.class).repository(repoName);
    logger.info("--> move index-N blob to next generation");
    final RepositoryData repositoryData = getRepositoryData(repository);
    final long beforeMoveGen = repositoryData.getGenId();
    Files.move(repo.resolve("index-" + beforeMoveGen), repo.resolve("index-" + (beforeMoveGen + 1)));
    logger.info("--> verify index-N blob is found at the new location");
    assertThat(getRepositoryData(repository).getGenId(), is(beforeMoveGen + 1));
    final SnapshotsService snapshotsService = internalCluster().getCurrentMasterNodeInstance(SnapshotsService.class);
    logger.info("--> wait for all listeners on snapshots service to be resolved to avoid snapshot task batching causing a conflict");
    assertBusy(() -> assertTrue(snapshotsService.assertAllListenersResolved()));
    logger.info("--> delete snapshot");
    client.admin().cluster().prepareDeleteSnapshot(repoName, snapshot).get();
    logger.info("--> verify index-N blob is found at the expected location");
    assertThat(getRepositoryData(repository).getGenId(), is(beforeMoveGen + 2));
    logger.info("--> make sure snapshot doesn't exist");
    expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareGetSnapshots(repoName).addSnapshots(snapshot).get());
}

Example 8 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testFindDanglingLatestGeneration.

public void testFindDanglingLatestGeneration() throws Exception {
    Path repo = randomRepoPath();
    final String repoName = "test-repo";
    createRepository(repoName, "fs", Settings.builder().put("location", repo).put("compress", false).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES));
    createIndex("test-idx-1", "test-idx-2");
    logger.info("--> indexing some data");
    indexRandom(true, client().prepareIndex().setIndex("test-idx-1").setSource("foo", "bar"), client().prepareIndex().setIndex("test-idx-2").setSource("foo", "bar"));
    final String snapshot = "test-snap";
    logger.info("--> creating snapshot");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(repoName, snapshot).setWaitForCompletion(true).setIndices("test-idx-*").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    final Repository repository = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(repoName);
    logger.info("--> move index-N blob to next generation");
    final RepositoryData repositoryData = getRepositoryData(repoName);
    final long beforeMoveGen = repositoryData.getGenId();
    Files.move(repo.resolve("index-" + beforeMoveGen), repo.resolve("index-" + (beforeMoveGen + 1)));
    logger.info("--> set next generation as pending in the cluster state");
    updateClusterState(currentState -> ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, currentState.metadata().<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE).withUpdatedGeneration(repository.getMetadata().name(), beforeMoveGen, beforeMoveGen + 1)).build()).build());
    logger.info("--> full cluster restart");
    internalCluster().fullRestart();
    ensureGreen();
    logger.info("--> verify index-N blob is found at the new location");
    assertThat(getRepositoryData(repoName).getGenId(), is(beforeMoveGen + 1));
    startDeleteSnapshot(repoName, snapshot).get();
    logger.info("--> verify index-N blob is found at the expected location");
    assertThat(getRepositoryData(repoName).getGenId(), is(beforeMoveGen + 2));
    logger.info("--> make sure snapshot doesn't exist");
    expectThrows(SnapshotMissingException.class, () -> client().admin().cluster().prepareGetSnapshots(repoName).addSnapshots(snapshot).get());
}

Example 9 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryTests method testIndexGenerationalFiles.

public void testIndexGenerationalFiles() throws Exception {
    final BlobStoreRepository repository = setupRepo();
    assertEquals(OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), RepositoryData.EMPTY);
    final long pendingGeneration = repository.metadata.pendingGeneration();
    // write to index generational file
    RepositoryData repositoryData = generateRandomRepoData();
    writeIndexGen(repository, repositoryData, RepositoryData.EMPTY_REPO_GEN);
    assertThat(OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), equalTo(repositoryData));
    final long expectedGeneration = pendingGeneration + 1L;
    assertThat(repository.latestIndexBlobId(), equalTo(expectedGeneration));
    assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(expectedGeneration));
    // adding more and writing to a new index generational file
    repositoryData = addRandomSnapshotsToRepoData(OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), true);
    writeIndexGen(repository, repositoryData, repositoryData.getGenId());
    assertEquals(OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), repositoryData);
    assertThat(repository.latestIndexBlobId(), equalTo(expectedGeneration + 1L));
    assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(expectedGeneration + 1L));
    // removing a snapshot and writing to a new index generational file
    repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).removeSnapshots(Collections.singleton(repositoryData.getSnapshotIds().iterator().next()), ShardGenerations.EMPTY);
    writeIndexGen(repository, repositoryData, repositoryData.getGenId());
    assertEquals(OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), repositoryData);
    assertThat(repository.latestIndexBlobId(), equalTo(expectedGeneration + 2L));
    assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(expectedGeneration + 2L));
}

Example 10 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testOverwriteSnapshotInfoBlob.

public void testOverwriteSnapshotInfoBlob() throws Exception {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    final RepositoryMetadata metadata = new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY);
    final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(metadata);
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(metadata, xContentRegistry(), clusterService, recoverySettings, blobStoreContext, random())) {
        clusterService.addStateApplier(event -> repository.updateState(event.state()));
        // Apply state once to initialize repo properly like RepositoriesService would
        repository.updateState(clusterService.state());
        repository.start();
        // We create a snap- blob for snapshot "foo" in the first generation
        final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID());
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs.
        final AssertionError assertionError = expectThrows(AssertionError.class, () -> PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f)));
        assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n     but: was <5>"));
        // We try to write yet another snap- blob for "foo" in the next generation.
        // It passes cleanly because the content of the blob except for the timestamps.
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
    }
}