Examples with BlobStoreRepository org.opensearch.repositories.blobstore.BlobStoreRepository used on opensource projects

Example 6 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testReadAfterDeleteAfterWriteThrows.

public void testReadAfterDeleteAfterWriteThrows() throws IOException {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY), xContentRegistry(), BlobStoreTestUtil.mockClusterService(), recoverySettings, blobStoreContext, random())) {
        repository.start();
        final BlobContainer blobContainer = repository.blobStore().blobContainer(repository.basePath());
        final String blobName = randomAlphaOfLength(10);
        final int lengthWritten = randomIntBetween(1, 100);
        final byte[] blobData = randomByteArrayOfLength(lengthWritten);
        blobContainer.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten, true);
        blobContainer.deleteBlobsIgnoringIfNotExists(Collections.singletonList(blobName));
        assertThrowsOnInconsistentRead(blobContainer, blobName);
        blobStoreContext.forceConsistent();
        expectThrows(NoSuchFileException.class, () -> blobContainer.readBlob(blobName));
    }
}

Example 7 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class AbstractThirdPartyRepositoryTestCase method listChildren.

private Set<String> listChildren(BlobPath path) {
    final PlainActionFuture<Set<String>> future = PlainActionFuture.newFuture();
    final BlobStoreRepository repository = getRepository();
    repository.threadPool().generic().execute(ActionRunnable.supply(future, () -> repository.blobStore().blobContainer(path).children().keySet()));
    return future.actionGet();
}

Example 8 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class AbstractThirdPartyRepositoryTestCase method testCleanup.

public void testCleanup() throws Exception {
    createIndex("test-idx-1");
    createIndex("test-idx-2");
    createIndex("test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        client().prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
        client().prepareIndex("test-idx-2").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
        client().prepareIndex("test-idx-3").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
    }
    client().admin().indices().prepareRefresh().get();
    final String snapshotName = "test-snap-" + System.currentTimeMillis();
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", snapshotName).setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots(snapshotName).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    final BlobStoreRepository repo = (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo");
    final Executor genericExec = repo.threadPool().executor(ThreadPool.Names.GENERIC);
    logger.info("--> creating a dangling index folder");
    createDanglingIndex(repo, genericExec);
    logger.info("--> deleting a snapshot to trigger repository cleanup");
    client().admin().cluster().deleteSnapshot(new DeleteSnapshotRequest("test-repo", snapshotName)).actionGet();
    assertConsistentRepository(repo, genericExec);
    logger.info("--> Create dangling index");
    createDanglingIndex(repo, genericExec);
    logger.info("--> Execute repository cleanup");
    final CleanupRepositoryResponse response = client().admin().cluster().prepareCleanupRepository("test-repo").get();
    assertCleanupResponse(response, 3L, 1L);
}

Example 9 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class HdfsTests method testSimpleWorkflow.

public void testSimpleWorkflow() {
    assumeFalse("https://github.com/elastic/elasticsearch/issues/31498", JavaVersion.current().equals(JavaVersion.parse("11")));
    Client client = client();
    AcknowledgedResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("hdfs").setSettings(Settings.builder().put("uri", "hdfs:///").put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName()).put("path", "foo").put("chunk_size", randomIntBetween(100, 1000) + "k").put("compress", randomBoolean())).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    createIndex("test-idx-1");
    createIndex("test-idx-2");
    createIndex("test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        client().prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
        client().prepareIndex("test-idx-2").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
        client().prepareIndex("test-idx-3").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
    }
    client().admin().indices().prepareRefresh().get();
    assertThat(count(client, "test-idx-1"), equalTo(100L));
    assertThat(count(client, "test-idx-2"), equalTo(100L));
    assertThat(count(client, "test-idx-3"), equalTo(100L));
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    logger.info("--> delete some data");
    for (int i = 0; i < 50; i++) {
        client.prepareDelete("test-idx-1", Integer.toString(i)).get();
    }
    for (int i = 50; i < 100; i++) {
        client.prepareDelete("test-idx-2", Integer.toString(i)).get();
    }
    for (int i = 0; i < 100; i += 2) {
        client.prepareDelete("test-idx-3", Integer.toString(i)).get();
    }
    client().admin().indices().prepareRefresh().get();
    assertThat(count(client, "test-idx-1"), equalTo(50L));
    assertThat(count(client, "test-idx-2"), equalTo(50L));
    assertThat(count(client, "test-idx-3"), equalTo(50L));
    logger.info("--> close indices");
    client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
    logger.info("--> restore all indices from the snapshot");
    RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(count(client, "test-idx-1"), equalTo(100L));
    assertThat(count(client, "test-idx-2"), equalTo(100L));
    assertThat(count(client, "test-idx-3"), equalTo(50L));
    // Test restore after index deletion
    logger.info("--> delete indices");
    client().admin().indices().prepareDelete("test-idx-1", "test-idx-2").get();
    logger.info("--> restore one index after deletion");
    restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(count(client, "test-idx-1"), equalTo(100L));
    ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
    assertThat(clusterState.getMetadata().hasIndex("test-idx-1"), equalTo(true));
    assertThat(clusterState.getMetadata().hasIndex("test-idx-2"), equalTo(false));
    final BlobStoreRepository repo = (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo");
    BlobStoreTestUtil.assertConsistency(repo, repo.threadPool().executor(ThreadPool.Names.GENERIC));
}

Example 10 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class S3BlobStoreRepositoryTests method testEnforcedCooldownPeriod.

public void testEnforcedCooldownPeriod() throws IOException {
    final String repoName = createRepository(randomName(), Settings.builder().put(repositorySettings()).put(S3Repository.COOLDOWN_PERIOD.getKey(), TEST_COOLDOWN_PERIOD).build());
    final SnapshotId fakeOldSnapshot = client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-old").setWaitForCompletion(true).setIndices().get().getSnapshotInfo().snapshotId();
    final RepositoriesService repositoriesService = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class);
    final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName);
    final RepositoryData repositoryData = getRepositoryData(repository);
    final RepositoryData modifiedRepositoryData = repositoryData.withVersions(Collections.singletonMap(fakeOldSnapshot, SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion()));
    final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), SnapshotsService.OLD_SNAPSHOT_FORMAT));
    PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () -> {
        try (InputStream stream = serialized.streamInput()) {
            repository.blobStore().blobContainer(repository.basePath()).writeBlobAtomic(BlobStoreRepository.INDEX_FILE_PREFIX + modifiedRepositoryData.getGenId(), stream, serialized.length(), true);
        }
    })));
    final String newSnapshotName = "snapshot-new";
    final long beforeThrottledSnapshot = repository.threadPool().relativeTimeInNanos();
    client().admin().cluster().prepareCreateSnapshot(repoName, newSnapshotName).setWaitForCompletion(true).setIndices().get();
    assertThat(repository.threadPool().relativeTimeInNanos() - beforeThrottledSnapshot, greaterThan(TEST_COOLDOWN_PERIOD.getNanos()));
    final long beforeThrottledDelete = repository.threadPool().relativeTimeInNanos();
    client().admin().cluster().prepareDeleteSnapshot(repoName, newSnapshotName).get();
    assertThat(repository.threadPool().relativeTimeInNanos() - beforeThrottledDelete, greaterThan(TEST_COOLDOWN_PERIOD.getNanos()));
    final long beforeFastDelete = repository.threadPool().relativeTimeInNanos();
    client().admin().cluster().prepareDeleteSnapshot(repoName, fakeOldSnapshot.getName()).get();
    assertThat(repository.threadPool().relativeTimeInNanos() - beforeFastDelete, lessThan(TEST_COOLDOWN_PERIOD.getNanos()));
}