Examples with BlobPath org.opensearch.common.blobstore.BlobPath used on opensource projects

Example 6 with BlobPath

use of org.opensearch.common.blobstore.BlobPath in project OpenSearch by opensearch-project.

the class OpenSearchBlobStoreRepositoryIntegTestCase method testDeleteBlobs.

public void testDeleteBlobs() throws IOException {
    try (BlobStore store = newBlobStore()) {
        final List<String> blobNames = Arrays.asList("foobar", "barfoo");
        final BlobContainer container = store.blobContainer(new BlobPath());
        // does not raise when blobs don't exist
        container.deleteBlobsIgnoringIfNotExists(blobNames);
        byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        final BytesArray bytesArray = new BytesArray(data);
        for (String blobName : blobNames) {
            writeBlob(container, blobName, bytesArray, randomBoolean());
        }
        assertEquals(container.listBlobs().size(), 2);
        container.deleteBlobsIgnoringIfNotExists(blobNames);
        assertTrue(container.listBlobs().isEmpty());
        // does not raise when blobs don't exist
        container.deleteBlobsIgnoringIfNotExists(blobNames);
    }
}

Example 7 with BlobPath

use of org.opensearch.common.blobstore.BlobPath in project OpenSearch by opensearch-project.

the class OpenSearchBlobStoreRepositoryIntegTestCase method testWriteRead.

public void testWriteRead() throws IOException {
    try (BlobStore store = newBlobStore()) {
        final BlobContainer container = store.blobContainer(new BlobPath());
        byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        writeBlob(container, "foobar", new BytesArray(data), randomBoolean());
        if (randomBoolean()) {
            // override file, to check if we get latest contents
            data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
            writeBlob(container, "foobar", new BytesArray(data), false);
        }
        try (InputStream stream = container.readBlob("foobar")) {
            BytesRefBuilder target = new BytesRefBuilder();
            while (target.length() < data.length) {
                byte[] buffer = new byte[scaledRandomIntBetween(1, data.length - target.length())];
                int offset = scaledRandomIntBetween(0, buffer.length - 1);
                int read = stream.read(buffer, offset, buffer.length - offset);
                if (read >= 0) {
                    target.append(new BytesRef(buffer, offset, read));
                } else {
                    fail("Expected [" + (data.length - target.length()) + "] more bytes to be readable but reached EOF");
                }
            }
            assertEquals(data.length, target.length());
            assertArrayEquals(data, Arrays.copyOfRange(target.bytes(), 0, target.length()));
        }
        container.delete();
    }
}

Example 8 with BlobPath

use of org.opensearch.common.blobstore.BlobPath in project OpenSearch by opensearch-project.

the class AbstractThirdPartyRepositoryTestCase method deleteAndAssertEmpty.

private void deleteAndAssertEmpty(BlobPath path) throws Exception {
    final BlobStoreRepository repo = getRepository();
    final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
    repo.threadPool().generic().execute(ActionRunnable.run(future, () -> repo.blobStore().blobContainer(path).delete()));
    future.actionGet();
    final BlobPath parent = path.parent();
    if (parent == null) {
        assertChildren(path, Collections.emptyList());
    } else {
        assertDeleted(parent, path.toArray()[path.toArray().length - 1]);
    }
}

Example 9 with BlobPath

use of org.opensearch.common.blobstore.BlobPath in project OpenSearch by opensearch-project.

the class FsBlobStore method buildPath.

private Path buildPath(BlobPath path) {
    String[] paths = path.toArray();
    if (paths.length == 0) {
        return path();
    }
    Path blobPath = this.path.resolve(paths[0]);
    if (paths.length > 1) {
        for (int i = 1; i < paths.length; i++) {
            blobPath = blobPath.resolve(paths[i]);
        }
    }
    return blobPath;
}

Example 10 with BlobPath

use of org.opensearch.common.blobstore.BlobPath in project OpenSearch by opensearch-project.

the class S3BlobStoreContainerTests method testExecuteMultipartUploadAborted.

public void testExecuteMultipartUploadAborted() {
    final String bucketName = randomAlphaOfLengthBetween(1, 10);
    final String blobName = randomAlphaOfLengthBetween(1, 10);
    final BlobPath blobPath = new BlobPath();
    final long blobSize = ByteSizeUnit.MB.toBytes(765);
    final long bufferSize = ByteSizeUnit.MB.toBytes(150);
    final S3BlobStore blobStore = mock(S3BlobStore.class);
    when(blobStore.bucket()).thenReturn(bucketName);
    when(blobStore.bufferSizeInBytes()).thenReturn(bufferSize);
    when(blobStore.getStorageClass()).thenReturn(randomFrom(StorageClass.values()));
    final AmazonS3 client = mock(AmazonS3.class);
    final AmazonS3Reference clientReference = new AmazonS3Reference(client);
    doAnswer(invocation -> {
        clientReference.incRef();
        return clientReference;
    }).when(blobStore).clientReference();
    final String uploadId = randomAlphaOfLength(25);
    final int stage = randomInt(2);
    final List<AmazonClientException> exceptions = Arrays.asList(new AmazonClientException("Expected initialization request to fail"), new AmazonClientException("Expected upload part request to fail"), new AmazonClientException("Expected completion request to fail"));
    if (stage == 0) {
        // Fail the initialization request
        when(client.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenThrow(exceptions.get(stage));
    } else if (stage == 1) {
        final InitiateMultipartUploadResult initResult = new InitiateMultipartUploadResult();
        initResult.setUploadId(uploadId);
        when(client.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(initResult);
        // Fail the upload part request
        when(client.uploadPart(any(UploadPartRequest.class))).thenThrow(exceptions.get(stage));
    } else {
        final InitiateMultipartUploadResult initResult = new InitiateMultipartUploadResult();
        initResult.setUploadId(uploadId);
        when(client.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(initResult);
        when(client.uploadPart(any(UploadPartRequest.class))).thenAnswer(invocationOnMock -> {
            final UploadPartRequest request = (UploadPartRequest) invocationOnMock.getArguments()[0];
            final UploadPartResult response = new UploadPartResult();
            response.setPartNumber(request.getPartNumber());
            response.setETag(randomAlphaOfLength(20));
            return response;
        });
        // Fail the completion request
        when(client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenThrow(exceptions.get(stage));
    }
    final ArgumentCaptor<AbortMultipartUploadRequest> argumentCaptor = ArgumentCaptor.forClass(AbortMultipartUploadRequest.class);
    doNothing().when(client).abortMultipartUpload(argumentCaptor.capture());
    final IOException e = expectThrows(IOException.class, () -> {
        final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
        blobContainer.executeMultipartUpload(blobStore, blobName, new ByteArrayInputStream(new byte[0]), blobSize);
    });
    assertEquals("Unable to upload object [" + blobName + "] using multipart upload", e.getMessage());
    assertThat(e.getCause(), instanceOf(AmazonClientException.class));
    assertEquals(exceptions.get(stage).getMessage(), e.getCause().getMessage());
    if (stage == 0) {
        verify(client, times(1)).initiateMultipartUpload(any(InitiateMultipartUploadRequest.class));
        verify(client, times(0)).uploadPart(any(UploadPartRequest.class));
        verify(client, times(0)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
        verify(client, times(0)).abortMultipartUpload(any(AbortMultipartUploadRequest.class));
    } else {
        verify(client, times(1)).initiateMultipartUpload(any(InitiateMultipartUploadRequest.class));
        if (stage == 1) {
            verify(client, times(1)).uploadPart(any(UploadPartRequest.class));
            verify(client, times(0)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
        } else {
            verify(client, times(6)).uploadPart(any(UploadPartRequest.class));
            verify(client, times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
        }
        verify(client, times(1)).abortMultipartUpload(any(AbortMultipartUploadRequest.class));
        final AbortMultipartUploadRequest abortRequest = argumentCaptor.getValue();
        assertEquals(bucketName, abortRequest.getBucketName());
        assertEquals(blobName, abortRequest.getKey());
        assertEquals(uploadId, abortRequest.getUploadId());
    }
}