Search in sources :

Example 1 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class AzureStorageCleanupThirdPartyTests method ensureSasTokenPermissions.

private void ensureSasTokenPermissions() {
    final BlobStoreRepository repository = getRepository();
    final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
    repository.threadPool().generic().execute(ActionRunnable.wrap(future, l -> {
        final AzureBlobStore blobStore = (AzureBlobStore) repository.blobStore();
        final String account = "default";
        final Tuple<BlobServiceClient, Supplier<Context>> client = blobStore.getService().client(account);
        final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(blobStore.toString());
        try {
            SocketAccess.doPrivilegedException(() -> blobContainer.existsWithResponse(null, client.v2().get()));
            future.onFailure(new RuntimeException("The SAS token used in this test allowed for checking container existence. This test only supports tokens " + "that grant only the documented permission requirements for the Azure repository plugin."));
        } catch (BlobStorageException e) {
            if (e.getStatusCode() == HttpURLConnection.HTTP_FORBIDDEN) {
                future.onResponse(null);
            } else {
                future.onFailure(e);
            }
        }
    }));
    future.actionGet();
}
Also used : HttpURLConnection(java.net.HttpURLConnection) Matchers.blankOrNullString(org.hamcrest.Matchers.blankOrNullString) AfterClass(org.junit.AfterClass) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) Context(com.azure.core.util.Context) BlobContainerClient(com.azure.storage.blob.BlobContainerClient) BlobStorageException(com.azure.storage.blob.models.BlobStorageException) MockSecureSettings(org.opensearch.common.settings.MockSecureSettings) ActionRunnable(org.opensearch.action.ActionRunnable) Collection(java.util.Collection) AbstractThirdPartyRepositoryTestCase(org.opensearch.repositories.AbstractThirdPartyRepositoryTestCase) Matchers.not(org.hamcrest.Matchers.not) Settings(org.opensearch.common.settings.Settings) Supplier(java.util.function.Supplier) Plugin(org.opensearch.plugins.Plugin) Strings(org.opensearch.common.Strings) Tuple(org.opensearch.common.collect.Tuple) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) SecureSettings(org.opensearch.common.settings.SecureSettings) BlobServiceClient(com.azure.storage.blob.BlobServiceClient) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Schedulers(reactor.core.scheduler.Schedulers) Context(com.azure.core.util.Context) BlobContainerClient(com.azure.storage.blob.BlobContainerClient) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) Matchers.blankOrNullString(org.hamcrest.Matchers.blankOrNullString) BlobStorageException(com.azure.storage.blob.models.BlobStorageException) Tuple(org.opensearch.common.collect.Tuple)

Example 2 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class TransportCleanupRepositoryAction method cleanupRepo.

/**
 * Runs cleanup operations on the given repository.
 * @param repositoryName Repository to clean up
 * @param listener Listener for cleanup result
 */
private void cleanupRepo(String repositoryName, ActionListener<RepositoryCleanupResult> listener) {
    final Repository repository = repositoriesService.repository(repositoryName);
    if (repository instanceof BlobStoreRepository == false) {
        listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup"));
        return;
    }
    final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repository.getRepositoryData(repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        final long repositoryStateId = repositoryData.getGenId();
        logger.info("Running cleanup operations on repository [{}][{}]", repositoryName, repositoryStateId);
        clusterService.submitStateUpdateTask("cleanup repository [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

            private boolean startedCleanup = false;

            @Override
            public ClusterState execute(ClusterState currentState) {
                final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
                if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a repository cleanup is already in-progress in [" + repositoryCleanupInProgress + "]");
                }
                final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                if (snapshots.entries().isEmpty() == false) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently running in [" + snapshots + "]");
                }
                return ClusterState.builder(currentState).putCustom(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(Collections.singletonList(RepositoryCleanupInProgress.startedEntry(repositoryName, repositoryStateId)))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                after(e, null);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                startedCleanup = true;
                logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId);
                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup(repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
            }

            private void after(@Nullable Exception failure, @Nullable RepositoryCleanupResult result) {
                if (failure == null) {
                    logger.debug("Finished repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId);
                } else {
                    logger.debug(() -> new ParameterizedMessage("Failed to finish repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                }
                assert failure != null || result != null;
                if (startedCleanup == false) {
                    logger.debug("No cleanup task to remove from cluster state because we failed to start one", failure);
                    listener.onFailure(failure);
                    return;
                }
                clusterService.submitStateUpdateTask("remove repository cleanup task [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        return removeInProgressCleanup(currentState);
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        if (failure != null) {
                            e.addSuppressed(failure);
                        }
                        logger.warn(() -> new ParameterizedMessage("[{}] failed to remove repository cleanup task", repositoryName), e);
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        if (failure == null) {
                            logger.info("Done with repository cleanup on [{}][{}] with result [{}]", repositoryName, repositoryStateId, result);
                            listener.onResponse(result);
                        } else {
                            logger.warn(() -> new ParameterizedMessage("Failed to run repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                            listener.onFailure(failure);
                        }
                    }
                });
            }
        });
    }, listener::onFailure);
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) RepositoryCleanupResult(org.opensearch.repositories.RepositoryCleanupResult) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) Repository(org.opensearch.repositories.Repository) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) StepListener(org.opensearch.action.StepListener) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage)

Example 3 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testReadAfterWriteConsistently.

public void testReadAfterWriteConsistently() throws IOException {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY), xContentRegistry(), BlobStoreTestUtil.mockClusterService(), recoverySettings, blobStoreContext, random())) {
        repository.start();
        final BlobContainer blobContainer = repository.blobStore().blobContainer(repository.basePath());
        final String blobName = randomAlphaOfLength(10);
        final int lengthWritten = randomIntBetween(1, 100);
        final byte[] blobData = randomByteArrayOfLength(lengthWritten);
        blobContainer.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten, true);
        try (InputStream in = blobContainer.readBlob(blobName)) {
            final byte[] readBytes = new byte[lengthWritten + 1];
            final int lengthSeen = in.read(readBytes);
            assertThat(lengthSeen, equalTo(lengthWritten));
            assertArrayEquals(blobData, Arrays.copyOf(readBytes, lengthWritten));
        }
    }
}
Also used : ByteArrayInputStream(java.io.ByteArrayInputStream) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) BlobContainer(org.opensearch.common.blobstore.BlobContainer)

Example 4 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testOverwriteSnapshotInfoBlob.

public void testOverwriteSnapshotInfoBlob() throws Exception {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    final RepositoryMetadata metadata = new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY);
    final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(metadata);
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(metadata, xContentRegistry(), clusterService, recoverySettings, blobStoreContext, random())) {
        clusterService.addStateApplier(event -> repository.updateState(event.state()));
        // Apply state once to initialize repo properly like RepositoriesService would
        repository.updateState(clusterService.state());
        repository.start();
        // We create a snap- blob for snapshot "foo" in the first generation
        final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID());
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs.
        final AssertionError assertionError = expectThrows(AssertionError.class, () -> PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f)));
        assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n     but: was <5>"));
        // We try to write yet another snap- blob for "foo" in the next generation.
        // It passes cleanly because the content of the blob except for the timestamps.
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
    }
}
Also used : ShardGenerations(org.opensearch.repositories.ShardGenerations) NoSuchFileException(java.nio.file.NoSuchFileException) Arrays(java.util.Arrays) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) Metadata(org.opensearch.cluster.metadata.Metadata) BlobContainer(org.opensearch.common.blobstore.BlobContainer) Version(org.opensearch.Version) Function(java.util.function.Function) ByteArrayInputStream(java.io.ByteArrayInputStream) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) UUIDs(org.opensearch.common.UUIDs) ClusterSettings(org.opensearch.common.settings.ClusterSettings) RecoverySettings(org.opensearch.indices.recovery.RecoverySettings) RepositoryData(org.opensearch.repositories.RepositoryData) BlobStoreTestUtil(org.opensearch.repositories.blobstore.BlobStoreTestUtil) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) Settings(org.opensearch.common.settings.Settings) IOException(java.io.IOException) Matchers.startsWith(org.hamcrest.Matchers.startsWith) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ClusterService(org.opensearch.cluster.service.ClusterService) Collections(java.util.Collections) InputStream(java.io.InputStream) SnapshotId(org.opensearch.snapshots.SnapshotId) SnapshotInfo(org.opensearch.snapshots.SnapshotInfo) ClusterService(org.opensearch.cluster.service.ClusterService) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) NoSuchFileException(java.nio.file.NoSuchFileException) IOException(java.io.IOException) RepositoryData(org.opensearch.repositories.RepositoryData)

Example 5 with BlobStoreRepository

use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.

the class MockEventuallyConsistentRepositoryTests method testOverwriteRandomBlobFails.

public void testOverwriteRandomBlobFails() throws IOException {
    MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
    try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY), xContentRegistry(), BlobStoreTestUtil.mockClusterService(), recoverySettings, blobStoreContext, random())) {
        repository.start();
        final BlobContainer container = repository.blobStore().blobContainer(repository.basePath());
        final String blobName = randomAlphaOfLength(10);
        final int lengthWritten = randomIntBetween(1, 100);
        final byte[] blobData = randomByteArrayOfLength(lengthWritten);
        container.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten, false);
        final AssertionError assertionError = expectThrows(AssertionError.class, () -> container.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten - 1, false));
        assertThat(assertionError.getMessage(), startsWith("Tried to overwrite blob [" + blobName + "]"));
    }
}
Also used : ByteArrayInputStream(java.io.ByteArrayInputStream) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) RepositoryMetadata(org.opensearch.cluster.metadata.RepositoryMetadata) BlobContainer(org.opensearch.common.blobstore.BlobContainer)

Aggregations

BlobStoreRepository (org.opensearch.repositories.blobstore.BlobStoreRepository)16 ByteArrayInputStream (java.io.ByteArrayInputStream)7 RepositoryMetadata (org.opensearch.cluster.metadata.RepositoryMetadata)6 BlobContainer (org.opensearch.common.blobstore.BlobContainer)6 RepositoryData (org.opensearch.repositories.RepositoryData)4 InputStream (java.io.InputStream)3 IOException (java.io.IOException)2 Executor (java.util.concurrent.Executor)2 Matchers.containsString (org.hamcrest.Matchers.containsString)2 Matchers.equalTo (org.hamcrest.Matchers.equalTo)2 CreateSnapshotResponse (org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse)2 PlainActionFuture (org.opensearch.action.support.PlainActionFuture)2 AcknowledgedResponse (org.opensearch.action.support.master.AcknowledgedResponse)2 ClusterState (org.opensearch.cluster.ClusterState)2 Settings (org.opensearch.common.settings.Settings)2 RepositoriesService (org.opensearch.repositories.RepositoriesService)2 Context (com.azure.core.util.Context)1 BlobContainerClient (com.azure.storage.blob.BlobContainerClient)1 BlobServiceClient (com.azure.storage.blob.BlobServiceClient)1 BlobStorageException (com.azure.storage.blob.models.BlobStorageException)1