Examples with RecoverySettings org.opensearch.indices.recovery.RecoverySettings used on opensource projects

Example 1 with RecoverySettings

use of org.opensearch.indices.recovery.RecoverySettings in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryRestoreTests method createRepository.

/**
 * Create a {@link Repository} with a random name *
 */
private Repository createRepository() {
    Settings settings = Settings.builder().put("location", randomAlphaOfLength(10)).build();
    RepositoryMetadata repositoryMetadata = new RepositoryMetadata(randomAlphaOfLength(10), FsRepository.TYPE, settings);
    final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(repositoryMetadata);
    final FsRepository repository = new FsRepository(repositoryMetadata, createEnvironment(), xContentRegistry(), clusterService, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {

        @Override
        protected void assertSnapshotOrGenericThread() {
        // eliminate thread name check as we create repo manually
        }
    };
    clusterService.addStateApplier(event -> repository.updateState(event.state()));
    // Apply state once to initialize repo properly like RepositoriesService would
    repository.updateState(clusterService.state());
    repository.start();
    return repository;
}

Example 2 with RecoverySettings

use of org.opensearch.indices.recovery.RecoverySettings in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testOverwriteSnapshotInfoBlob.

public void testOverwriteSnapshotInfoBlob() throws Exception {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    final RepositoryMetadata metadata = new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY);
    final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(metadata);
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(metadata, xContentRegistry(), clusterService, recoverySettings, blobStoreContext, random())) {
        clusterService.addStateApplier(event -> repository.updateState(event.state()));
        // Apply state once to initialize repo properly like RepositoriesService would
        repository.updateState(clusterService.state());
        repository.start();
        // We create a snap- blob for snapshot "foo" in the first generation
        final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID());
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs.
        final AssertionError assertionError = expectThrows(AssertionError.class, () -> PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f)));
        assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n     but: was <5>"));
        // We try to write yet another snap- blob for "foo" in the next generation.
        // It passes cleanly because the content of the blob except for the timestamps.
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
    }
}

Example 3 with RecoverySettings

use of org.opensearch.indices.recovery.RecoverySettings in project OpenSearch by opensearch-project.

the class AzureRepositorySettingsTests method azureRepository.

private AzureRepository azureRepository(Settings settings) {
    Settings internalSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()).putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()).put(settings).build();
    final AzureRepository azureRepository = new AzureRepository(new RepositoryMetadata("foo", "azure", internalSettings), NamedXContentRegistry.EMPTY, mock(AzureStorageService.class), BlobStoreTestUtil.mockClusterService(), new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
    assertThat(azureRepository.getBlobStore(), is(nullValue()));
    return azureRepository;
}

Example 4 with RecoverySettings

use of org.opensearch.indices.recovery.RecoverySettings in project OpenSearch by opensearch-project.

the class FsRepositoryTests method testSnapshotAndRestore.

public void testSnapshotAndRestore() throws IOException, InterruptedException {
    ThreadPool threadPool = new TestThreadPool(getClass().getSimpleName());
    try (Directory directory = newDirectory()) {
        Path repo = createTempDir();
        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()).put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()).putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()).put("location", repo).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES).build();
        int numDocs = indexDocs(directory);
        RepositoryMetadata metadata = new RepositoryMetadata("test", "fs", settings);
        FsRepository repository = new FsRepository(metadata, new Environment(settings, null), NamedXContentRegistry.EMPTY, BlobStoreTestUtil.mockClusterService(), new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
        repository.start();
        final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_INDEX_UUID, "myindexUUID").build();
        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("myindex", indexSettings);
        ShardId shardId = new ShardId(idxSettings.getIndex(), 1);
        Store store = new Store(shardId, idxSettings, directory, new DummyShardLock(shardId));
        SnapshotId snapshotId = new SnapshotId("test", "test");
        IndexId indexId = new IndexId(idxSettings.getIndex().getName(), idxSettings.getUUID());
        IndexCommit indexCommit = Lucene.getIndexCommit(Lucene.readSegmentInfos(store.directory()), store.directory());
        final PlainActionFuture<String> future1 = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> {
            IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null);
            repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1);
            future1.actionGet();
            IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
            assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
        });
        final String shardGeneration = future1.actionGet();
        Lucene.cleanLuceneIndex(directory);
        expectThrows(org.apache.lucene.index.IndexNotFoundException.class, () -> Lucene.readSegmentInfos(directory));
        DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
        ShardRouting routing = ShardRouting.newUnassigned(shardId, true, new RecoverySource.SnapshotRecoverySource("test", new Snapshot("foo", snapshotId), Version.CURRENT, indexId), new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, ""));
        routing = ShardRoutingHelper.initialize(routing, localNode.getId(), 0);
        RecoveryState state = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureA = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, snapshotId, indexId, shardId, state, futureA));
        futureA.actionGet();
        assertTrue(state.getIndex().recoveredBytes() > 0);
        assertEquals(0, state.getIndex().reusedFileCount());
        assertEquals(indexCommit.getFileNames().size(), state.getIndex().recoveredFileCount());
        assertEquals(numDocs, Lucene.readSegmentInfos(directory).totalMaxDoc());
        deleteRandomDoc(store.directory());
        SnapshotId incSnapshotId = new SnapshotId("test1", "test1");
        IndexCommit incIndexCommit = Lucene.getIndexCommit(Lucene.readSegmentInfos(store.directory()), store.directory());
        Collection<String> commitFileNames = incIndexCommit.getFileNames();
        final PlainActionFuture<String> future2 = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> {
            IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
            repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2);
            future2.actionGet();
            IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
            assertEquals(2, copy.getIncrementalFileCount());
            assertEquals(commitFileNames.size(), copy.getTotalFileCount());
        });
        // roll back to the first snap and then incrementally restore
        RecoveryState firstState = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureB = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, snapshotId, indexId, shardId, firstState, futureB));
        futureB.actionGet();
        assertEquals("should reuse everything except of .liv and .si", commitFileNames.size() - 2, firstState.getIndex().reusedFileCount());
        RecoveryState secondState = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureC = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, incSnapshotId, indexId, shardId, secondState, futureC));
        futureC.actionGet();
        assertEquals(secondState.getIndex().reusedFileCount(), commitFileNames.size() - 2);
        assertEquals(secondState.getIndex().recoveredFileCount(), 2);
        List<RecoveryState.FileDetail> recoveredFiles = secondState.getIndex().fileDetails().stream().filter(f -> f.reused() == false).collect(Collectors.toList());
        Collections.sort(recoveredFiles, Comparator.comparing(RecoveryState.FileDetail::name));
        assertTrue(recoveredFiles.get(0).name(), recoveredFiles.get(0).name().endsWith(".liv"));
        assertTrue(recoveredFiles.get(1).name(), recoveredFiles.get(1).name().endsWith("segments_" + incIndexCommit.getGeneration()));
    } finally {
        terminate(threadPool);
    }
}