Examples with BlobStore org.opensearch.common.blobstore.BlobStore used on opensource projects

Example 1 with BlobStore

use of org.opensearch.common.blobstore.BlobStore in project OpenSearch by opensearch-project.

the class BlobStoreFormatTests method testBlobStoreOperations.

public void testBlobStoreOperations() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent);
    // Write blobs in different formats
    checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile", false);
    checksumSMILE.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp", true);
    // Assert that all checksum blobs can be read
    assertEquals(checksumSMILE.read(blobContainer, "check-smile", xContentRegistry()).getText(), "checksum smile");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp", xContentRegistry()).getText(), "checksum smile compressed");
}

Example 2 with BlobStore

use of org.opensearch.common.blobstore.BlobStore in project OpenSearch by opensearch-project.

the class BlobStoreFormatTests method testBlobCorruption.

public void testBlobCorruption() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    String testString = randomAlphaOfLength(randomInt(10000));
    BlobObj blobObj = new BlobObj(testString);
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent);
    checksumFormat.write(blobObj, blobContainer, "test-path", randomBoolean());
    assertEquals(checksumFormat.read(blobContainer, "test-path", xContentRegistry()).getText(), testString);
    randomCorruption(blobContainer, "test-path");
    try {
        checksumFormat.read(blobContainer, "test-path", xContentRegistry());
        fail("Should have failed due to corruption");
    } catch (OpenSearchCorruptionException ex) {
        assertThat(ex.getMessage(), containsString("test-path"));
    } catch (EOFException ex) {
    // This can happen if corrupt the byte length
    }
}

Example 3 with BlobStore

use of org.opensearch.common.blobstore.BlobStore in project OpenSearch by opensearch-project.

the class GoogleCloudStorageBlobStoreContainerTests method testDeleteBlobsIgnoringIfNotExistsThrowsIOException.

@SuppressWarnings("unchecked")
public void testDeleteBlobsIgnoringIfNotExistsThrowsIOException() throws Exception {
    final List<String> blobs = Arrays.asList("blobA", "blobB");
    final StorageBatch batch = mock(StorageBatch.class);
    if (randomBoolean()) {
        StorageBatchResult<Boolean> result = mock(StorageBatchResult.class);
        when(batch.delete(any(BlobId.class))).thenReturn(result);
        doThrow(new StorageException(new IOException("Batch submit throws a storage exception"))).when(batch).submit();
    } else {
        StorageBatchResult<Boolean> resultA = mock(StorageBatchResult.class);
        doReturn(resultA).when(batch).delete(eq(BlobId.of("bucket", "blobA")));
        doAnswer(invocation -> {
            StorageException storageException = new StorageException(new IOException("Batched delete throws a storage exception"));
            ((BatchResult.Callback) invocation.getArguments()[0]).error(storageException);
            return null;
        }).when(resultA).notify(any(StorageBatchResult.Callback.class));
        StorageBatchResult<Boolean> resultB = mock(StorageBatchResult.class);
        doReturn(resultB).when(batch).delete(eq(BlobId.of("bucket", "blobB")));
        doAnswer(invocation -> {
            if (randomBoolean()) {
                StorageException storageException = new StorageException(new IOException("Batched delete throws a storage exception"));
                ((BatchResult.Callback) invocation.getArguments()[0]).error(storageException);
            } else {
                ((BatchResult.Callback) invocation.getArguments()[0]).success(randomBoolean());
            }
            return null;
        }).when(resultB).notify(any(StorageBatchResult.Callback.class));
        doNothing().when(batch).submit();
    }
    final Storage storage = mock(Storage.class);
    when(storage.get("bucket")).thenReturn(mock(Bucket.class));
    when(storage.batch()).thenReturn(batch);
    final GoogleCloudStorageService storageService = mock(GoogleCloudStorageService.class);
    when(storageService.client(any(String.class), any(String.class), any(GoogleCloudStorageOperationsStats.class))).thenReturn(storage);
    try (BlobStore store = new GoogleCloudStorageBlobStore("bucket", "test", "repo", storageService, randomIntBetween(1, 8) * 1024)) {
        final BlobContainer container = store.blobContainer(new BlobPath());
        IOException e = expectThrows(IOException.class, () -> container.deleteBlobsIgnoringIfNotExists(blobs));
        assertThat(e.getCause(), instanceOf(StorageException.class));
    }
}

Example 4 with BlobStore

use of org.opensearch.common.blobstore.BlobStore in project OpenSearch by opensearch-project.

the class GoogleCloudStorageBlobStoreRepositoryTests method testWriteReadLarge.

public void testWriteReadLarge() throws IOException {
    try (BlobStore store = newBlobStore()) {
        final BlobContainer container = store.blobContainer(new BlobPath());
        byte[] data = randomBytes(GoogleCloudStorageBlobStore.LARGE_BLOB_THRESHOLD_BYTE_SIZE + 1);
        writeBlob(container, "foobar", new BytesArray(data), randomBoolean());
        if (randomBoolean()) {
            // override file, to check if we get latest contents
            random().nextBytes(data);
            writeBlob(container, "foobar", new BytesArray(data), false);
        }
        try (InputStream stream = container.readBlob("foobar")) {
            BytesRefBuilder target = new BytesRefBuilder();
            while (target.length() < data.length) {
                byte[] buffer = new byte[scaledRandomIntBetween(1, data.length - target.length())];
                int offset = scaledRandomIntBetween(0, buffer.length - 1);
                int read = stream.read(buffer, offset, buffer.length - offset);
                target.append(new BytesRef(buffer, offset, read));
            }
            assertEquals(data.length, target.length());
            assertArrayEquals(data, Arrays.copyOfRange(target.bytes(), 0, target.length()));
        }
        container.delete();
    }
}

Example 5 with BlobStore

use of org.opensearch.common.blobstore.BlobStore in project OpenSearch by opensearch-project.

the class BlobStoreRepository method cleanup.

/**
 * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the
 * repository.
 * TODO: Add shard level cleanups
 * TODO: Add unreferenced index metadata cleanup
 * <ul>
 *     <li>Deleting stale indices {@link #cleanupStaleIndices}</li>
 *     <li>Deleting unreferenced root level blobs {@link #cleanupStaleRootFiles}</li>
 * </ul>
 * @param repositoryStateId     Current repository state id
 * @param repositoryMetaVersion version of the updated repository metadata to write
 * @param listener              Listener to complete when done
 */
public void cleanup(long repositoryStateId, Version repositoryMetaVersion, ActionListener<RepositoryCleanupResult> listener) {
    try {
        if (isReadOnly()) {
            throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository");
        }
        Map<String, BlobMetadata> rootBlobs = blobContainer().listBlobs();
        final RepositoryData repositoryData = safeRepositoryData(repositoryStateId, rootBlobs);
        final Map<String, BlobContainer> foundIndices = blobStore().blobContainer(indicesPath()).children();
        final Set<String> survivingIndexIds = repositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet());
        final List<String> staleRootBlobs = staleRootBlobs(repositoryData, rootBlobs.keySet());
        if (survivingIndexIds.equals(foundIndices.keySet()) && staleRootBlobs.isEmpty()) {
            // Nothing to clean up we return
            listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO));
        } else {
            // write new index-N blob to ensure concurrent operations will fail
            writeIndexGen(repositoryData, repositoryStateId, repositoryMetaVersion, Function.identity(), ActionListener.wrap(v -> cleanupStaleBlobs(Collections.emptyList(), foundIndices, rootBlobs, repositoryData, ActionListener.map(listener, RepositoryCleanupResult::new)), listener::onFailure));
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}