use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.
the class AzureStorageCleanupThirdPartyTests method ensureSasTokenPermissions.
private void ensureSasTokenPermissions() {
final BlobStoreRepository repository = getRepository();
final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
repository.threadPool().generic().execute(ActionRunnable.wrap(future, l -> {
final AzureBlobStore blobStore = (AzureBlobStore) repository.blobStore();
final String account = "default";
final Tuple<BlobServiceClient, Supplier<Context>> client = blobStore.getService().client(account);
final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(blobStore.toString());
try {
SocketAccess.doPrivilegedException(() -> blobContainer.existsWithResponse(null, client.v2().get()));
future.onFailure(new RuntimeException("The SAS token used in this test allowed for checking container existence. This test only supports tokens " + "that grant only the documented permission requirements for the Azure repository plugin."));
} catch (BlobStorageException e) {
if (e.getStatusCode() == HttpURLConnection.HTTP_FORBIDDEN) {
future.onResponse(null);
} else {
future.onFailure(e);
}
}
}));
future.actionGet();
}
use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.
the class TransportCleanupRepositoryAction method cleanupRepo.
/**
* Runs cleanup operations on the given repository.
* @param repositoryName Repository to clean up
* @param listener Listener for cleanup result
*/
private void cleanupRepo(String repositoryName, ActionListener<RepositoryCleanupResult> listener) {
final Repository repository = repositoriesService.repository(repositoryName);
if (repository instanceof BlobStoreRepository == false) {
listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup"));
return;
}
final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
repository.getRepositoryData(repositoryDataListener);
repositoryDataListener.whenComplete(repositoryData -> {
final long repositoryStateId = repositoryData.getGenId();
logger.info("Running cleanup operations on repository [{}][{}]", repositoryName, repositoryStateId);
clusterService.submitStateUpdateTask("cleanup repository [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {
private boolean startedCleanup = false;
@Override
public ClusterState execute(ClusterState currentState) {
final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
if (repositoryCleanupInProgress.hasCleanupInProgress()) {
throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a repository cleanup is already in-progress in [" + repositoryCleanupInProgress + "]");
}
final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
if (deletionsInProgress.hasDeletionsInProgress()) {
throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently being deleted in [" + deletionsInProgress + "]");
}
SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
if (snapshots.entries().isEmpty() == false) {
throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently running in [" + snapshots + "]");
}
return ClusterState.builder(currentState).putCustom(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(Collections.singletonList(RepositoryCleanupInProgress.startedEntry(repositoryName, repositoryStateId)))).build();
}
@Override
public void onFailure(String source, Exception e) {
after(e, null);
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
startedCleanup = true;
logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId);
threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup(repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
}
private void after(@Nullable Exception failure, @Nullable RepositoryCleanupResult result) {
if (failure == null) {
logger.debug("Finished repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId);
} else {
logger.debug(() -> new ParameterizedMessage("Failed to finish repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
}
assert failure != null || result != null;
if (startedCleanup == false) {
logger.debug("No cleanup task to remove from cluster state because we failed to start one", failure);
listener.onFailure(failure);
return;
}
clusterService.submitStateUpdateTask("remove repository cleanup task [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) {
return removeInProgressCleanup(currentState);
}
@Override
public void onFailure(String source, Exception e) {
if (failure != null) {
e.addSuppressed(failure);
}
logger.warn(() -> new ParameterizedMessage("[{}] failed to remove repository cleanup task", repositoryName), e);
listener.onFailure(e);
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
if (failure == null) {
logger.info("Done with repository cleanup on [{}][{}] with result [{}]", repositoryName, repositoryStateId, result);
listener.onResponse(result);
} else {
logger.warn(() -> new ParameterizedMessage("Failed to run repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
listener.onFailure(failure);
}
}
});
}
});
}, listener::onFailure);
}
use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.
the class MockEventuallyConsistentRepositoryTests method testReadAfterWriteConsistently.
public void testReadAfterWriteConsistently() throws IOException {
MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY), xContentRegistry(), BlobStoreTestUtil.mockClusterService(), recoverySettings, blobStoreContext, random())) {
repository.start();
final BlobContainer blobContainer = repository.blobStore().blobContainer(repository.basePath());
final String blobName = randomAlphaOfLength(10);
final int lengthWritten = randomIntBetween(1, 100);
final byte[] blobData = randomByteArrayOfLength(lengthWritten);
blobContainer.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten, true);
try (InputStream in = blobContainer.readBlob(blobName)) {
final byte[] readBytes = new byte[lengthWritten + 1];
final int lengthSeen = in.read(readBytes);
assertThat(lengthSeen, equalTo(lengthWritten));
assertArrayEquals(blobData, Arrays.copyOf(readBytes, lengthWritten));
}
}
}
use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.
the class MockEventuallyConsistentRepositoryTests method testOverwriteSnapshotInfoBlob.
public void testOverwriteSnapshotInfoBlob() throws Exception {
MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
final RepositoryMetadata metadata = new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY);
final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(metadata);
try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(metadata, xContentRegistry(), clusterService, recoverySettings, blobStoreContext, random())) {
clusterService.addStateApplier(event -> repository.updateState(event.state()));
// Apply state once to initialize repo properly like RepositoriesService would
repository.updateState(clusterService.state());
repository.start();
// We create a snap- blob for snapshot "foo" in the first generation
final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID());
PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
// We try to write another snap- blob for "foo" in the next generation. It fails because the content differs.
final AssertionError assertionError = expectThrows(AssertionError.class, () -> PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f)));
assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n but: was <5>"));
// We try to write yet another snap- blob for "foo" in the next generation.
// It passes cleanly because the content of the blob except for the timestamps.
PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
}
}
use of org.opensearch.repositories.blobstore.BlobStoreRepository in project OpenSearch by opensearch-project.
the class MockEventuallyConsistentRepositoryTests method testOverwriteRandomBlobFails.
public void testOverwriteRandomBlobFails() throws IOException {
MockEventuallyConsistentRepository.Context blobStoreContext = new MockEventuallyConsistentRepository.Context();
try (BlobStoreRepository repository = new MockEventuallyConsistentRepository(new RepositoryMetadata("testRepo", "mockEventuallyConsistent", Settings.EMPTY), xContentRegistry(), BlobStoreTestUtil.mockClusterService(), recoverySettings, blobStoreContext, random())) {
repository.start();
final BlobContainer container = repository.blobStore().blobContainer(repository.basePath());
final String blobName = randomAlphaOfLength(10);
final int lengthWritten = randomIntBetween(1, 100);
final byte[] blobData = randomByteArrayOfLength(lengthWritten);
container.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten, false);
final AssertionError assertionError = expectThrows(AssertionError.class, () -> container.writeBlob(blobName, new ByteArrayInputStream(blobData), lengthWritten - 1, false));
assertThat(assertionError.getMessage(), startsWith("Tried to overwrite blob [" + blobName + "]"));
}
}
Aggregations