Examples with ByteRange com.github.ambry.router.ByteRange used on opensource projects

Example 1 with ByteRange

use of com.github.ambry.router.ByteRange in project ambry by linkedin.

the class FrontendIntegrationTest method doPostGetHeadDeleteTest.

// postGetHeadDeleteTest() and multipartPostGetHeadTest() helpers
/**
 * Utility to test blob POST, GET, HEAD and DELETE operations for a specified size
 * @param contentSize the size of the blob to be tested
 * @param toPostAccount the {@link Account} to use in post headers. Can be {@code null} if only using service ID.
 * @param toPostContainer the {@link Container} to use in post headers. Can be {@code null} if only using service ID.
 * @param serviceId the serviceId to use for the POST
 * @param isPrivate the isPrivate flag to pass as part of the POST
 * @param expectedAccountName the expected account name in some response.
 * @param expectedContainerName the expected container name in some responses.
 * @param multipartPost {@code true} if multipart POST is desired, {@code false} otherwise.
 * @throws Exception
 */
private void doPostGetHeadDeleteTest(int contentSize, Account toPostAccount, Container toPostContainer, String serviceId, boolean isPrivate, String expectedAccountName, String expectedContainerName, boolean multipartPost) throws Exception {
    ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes(contentSize));
    String contentType = "application/octet-stream";
    String ownerId = "postGetHeadDeleteOwnerID";
    String accountNameInPost = toPostAccount != null ? toPostAccount.getName() : null;
    String containerNameInPost = toPostContainer != null ? toPostContainer.getName() : null;
    HttpHeaders headers = new DefaultHttpHeaders();
    setAmbryHeadersForPut(headers, 7200, isPrivate, serviceId, contentType, ownerId, accountNameInPost, containerNameInPost);
    String blobId;
    byte[] usermetadata = null;
    if (multipartPost) {
        usermetadata = UtilsTest.getRandomString(32).getBytes();
        blobId = multipartPostBlobAndVerify(headers, content, ByteBuffer.wrap(usermetadata));
    } else {
        headers.add(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key1", "value1");
        headers.add(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key2", "value2");
        blobId = postBlobAndVerify(headers, content);
    }
    headers.add(RestUtils.Headers.BLOB_SIZE, content.capacity());
    getBlobAndVerify(blobId, null, null, headers, isPrivate, content);
    getHeadAndVerify(blobId, null, null, headers, isPrivate, expectedAccountName, expectedContainerName);
    getBlobAndVerify(blobId, null, GetOption.None, headers, isPrivate, content);
    getHeadAndVerify(blobId, null, GetOption.None, headers, isPrivate, expectedAccountName, expectedContainerName);
    ByteRange range = ByteRange.fromLastNBytes(ThreadLocalRandom.current().nextLong(content.capacity() + 1));
    getBlobAndVerify(blobId, range, null, headers, isPrivate, content);
    getHeadAndVerify(blobId, range, null, headers, isPrivate, expectedAccountName, expectedContainerName);
    if (contentSize > 0) {
        range = ByteRange.fromStartOffset(ThreadLocalRandom.current().nextLong(content.capacity()));
        getBlobAndVerify(blobId, range, null, headers, isPrivate, content);
        getHeadAndVerify(blobId, range, null, headers, isPrivate, expectedAccountName, expectedContainerName);
        long random1 = ThreadLocalRandom.current().nextLong(content.capacity());
        long random2 = ThreadLocalRandom.current().nextLong(content.capacity());
        range = ByteRange.fromOffsetRange(Math.min(random1, random2), Math.max(random1, random2));
        getBlobAndVerify(blobId, range, null, headers, isPrivate, content);
        getHeadAndVerify(blobId, range, null, headers, isPrivate, expectedAccountName, expectedContainerName);
    }
    getNotModifiedBlobAndVerify(blobId, null, isPrivate);
    getUserMetadataAndVerify(blobId, null, headers, usermetadata);
    getBlobInfoAndVerify(blobId, null, headers, isPrivate, expectedAccountName, expectedContainerName, usermetadata);
    deleteBlobAndVerify(blobId);
    // check GET, HEAD and DELETE after delete.
    verifyOperationsAfterDelete(blobId, headers, isPrivate, expectedAccountName, expectedContainerName, content, usermetadata);
}

Example 2 with ByteRange

use of com.github.ambry.router.ByteRange in project ambry by linkedin.

the class FrontendIntegrationTest method verifyGetBlobResponse.

/**
 * Verifies the GET blob response.
 * @param responseParts the response received from the server.
 * @param range the {@link ByteRange} for the request.
 * @param expectedHeaders the expected headers in the response.
 * @param isPrivate {@code true} if the blob is private, {@code false} if not.
 * @param expectedContent the expected content of the blob.
 * @throws RestServiceException
 */
private void verifyGetBlobResponse(ResponseParts responseParts, ByteRange range, HttpHeaders expectedHeaders, boolean isPrivate, ByteBuffer expectedContent) throws RestServiceException {
    HttpResponse response = getHttpResponse(responseParts);
    assertEquals("Unexpected response status", range == null ? HttpResponseStatus.OK : HttpResponseStatus.PARTIAL_CONTENT, response.status());
    checkCommonGetHeadHeaders(response.headers());
    assertEquals("Content-Type does not match", expectedHeaders.get(RestUtils.Headers.AMBRY_CONTENT_TYPE), response.headers().get(HttpHeaderNames.CONTENT_TYPE));
    assertEquals(RestUtils.Headers.BLOB_SIZE + " does not match", expectedHeaders.get(RestUtils.Headers.BLOB_SIZE), response.headers().get(RestUtils.Headers.BLOB_SIZE));
    assertEquals("Accept-Ranges not set correctly", "bytes", response.headers().get(RestUtils.Headers.ACCEPT_RANGES));
    byte[] expectedContentArray = expectedContent.array();
    if (range != null) {
        long blobSize = Long.parseLong(expectedHeaders.get(RestUtils.Headers.BLOB_SIZE));
        assertEquals("Content-Range header not set correctly", RestUtils.buildContentRangeAndLength(range, blobSize).getFirst(), response.headers().get(RestUtils.Headers.CONTENT_RANGE));
        ByteRange resolvedRange = range.toResolvedByteRange(blobSize);
        expectedContentArray = Arrays.copyOfRange(expectedContentArray, (int) resolvedRange.getStartOffset(), (int) resolvedRange.getEndOffset() + 1);
    } else {
        assertNull("Content-Range header should not be set", response.headers().get(RestUtils.Headers.CONTENT_RANGE));
    }
    if (expectedContentArray.length < FRONTEND_CONFIG.frontendChunkedGetResponseThresholdInBytes) {
        assertEquals("Content-length not as expected", expectedContentArray.length, HttpUtil.getContentLength(response));
    }
    verifyCacheHeaders(isPrivate, response);
    byte[] responseContentArray = getContent(responseParts.queue, expectedContentArray.length).array();
    assertArrayEquals("GET content does not match original content", expectedContentArray, responseContentArray);
    assertTrue("Channel should be active", HttpUtil.isKeepAlive(response));
}

Example 3 with ByteRange

use of com.github.ambry.router.ByteRange in project ambry by linkedin.

the class FrontendIntegrationTest method stitchBlobAndVerify.

/**
 * Test the stitched upload flow for a specified chunk size and number of chunks.
 * @param account the {@link Account} to upload into.
 * @param container the {@link Container} to upload into.
 * @param signedChunkIds the list of signed chunk IDs to stitch together.
 * @param fullContentArray the content to compare the stitched blob against.
 * @throws Exception
 */
private void stitchBlobAndVerify(Account account, Container container, List<String> signedChunkIds, byte[] fullContentArray, long stitchedBlobSize) throws Exception {
    // stitchBlob
    HttpHeaders stitchHeaders = new DefaultHttpHeaders();
    setAmbryHeadersForPut(stitchHeaders, TTL_SECS, !container.isCacheable(), "stitcher", "video/mp4", "stitchedUploadTest", account.getName(), container.getName());
    HttpRequest httpRequest = buildRequest(HttpMethod.POST, Operations.STITCH, stitchHeaders, ByteBuffer.wrap(StitchRequestSerDe.toJson(signedChunkIds).toString().getBytes(StandardCharsets.UTF_8)));
    ResponseParts responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
    String stitchedBlobId = verifyPostAndReturnBlobId(responseParts, stitchedBlobSize, true);
    HttpHeaders expectedGetHeaders = new DefaultHttpHeaders().add(stitchHeaders);
    // Test different request types on stitched blob ID
    // (getBlobInfo, getBlob, getBlob w/ range, head, updateBlobTtl, deleteBlob)
    expectedGetHeaders.add(RestUtils.Headers.BLOB_SIZE, fullContentArray.length);
    expectedGetHeaders.set(RestUtils.Headers.LIFE_VERSION, "0");
    getBlobInfoAndVerify(stitchedBlobId, GetOption.None, expectedGetHeaders, !container.isCacheable(), account.getName(), container.getName(), null);
    List<ByteRange> ranges = new ArrayList<>();
    ranges.add(null);
    ranges.add(ByteRanges.fromLastNBytes(ThreadLocalRandom.current().nextLong(fullContentArray.length + 1)));
    ranges.add(ByteRanges.fromStartOffset(ThreadLocalRandom.current().nextLong(fullContentArray.length)));
    long random1 = ThreadLocalRandom.current().nextLong(fullContentArray.length);
    long random2 = ThreadLocalRandom.current().nextLong(fullContentArray.length);
    ranges.add(ByteRanges.fromOffsetRange(Math.min(random1, random2), Math.max(random1, random2)));
    for (ByteRange range : ranges) {
        getBlobAndVerify(stitchedBlobId, range, GetOption.None, false, expectedGetHeaders, !container.isCacheable(), ByteBuffer.wrap(fullContentArray), account.getName(), container.getName());
        getHeadAndVerify(stitchedBlobId, range, GetOption.None, expectedGetHeaders, !container.isCacheable(), account.getName(), container.getName());
    }
    updateBlobTtlAndVerify(stitchedBlobId, expectedGetHeaders, !container.isCacheable(), account.getName(), container.getName(), null);
    // Delete stitched blob.
    deleteBlobAndVerify(stitchedBlobId);
    verifyOperationsAfterDelete(stitchedBlobId, expectedGetHeaders, !container.isCacheable(), account.getName(), container.getName(), ByteBuffer.wrap(fullContentArray), null);
}

Example 4 with ByteRange

use of com.github.ambry.router.ByteRange in project ambry by linkedin.

the class FrontendIntegrationTestBase method doPostGetHeadUpdateDeleteUndeleteTest.

// postGetHeadUpdateDeleteTest() and multipartPostGetHeadUpdateDeleteTest() helpers
/**
 * Utility to test blob POST, GET, HEAD and DELETE operations for a specified size
 * @param contentSize the size of the blob to be tested
 * @param toPostAccount the {@link Account} to use in post headers. Can be {@code null} if only using service ID.
 * @param toPostContainer the {@link Container} to use in post headers. Can be {@code null} if only using service ID.
 * @param serviceId the serviceId to use for the POST
 * @param isPrivate the isPrivate flag to pass as part of the POST
 * @param expectedAccountName the expected account name in some response.
 * @param expectedContainerName the expected container name in some responses.
 * @param multipartPost {@code true} if multipart POST is desired, {@code false} otherwise.
 * @throws Exception
 */
void doPostGetHeadUpdateDeleteUndeleteTest(int contentSize, Account toPostAccount, Container toPostContainer, String serviceId, boolean isPrivate, String expectedAccountName, String expectedContainerName, boolean multipartPost) throws Exception {
    ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes(contentSize));
    String contentType = "application/octet-stream";
    String ownerId = "postGetHeadDeleteOwnerID";
    String accountNameInPost = toPostAccount != null ? toPostAccount.getName() : null;
    String containerNameInPost = toPostContainer != null ? toPostContainer.getName() : null;
    HttpHeaders headers = new DefaultHttpHeaders();
    setAmbryHeadersForPut(headers, TTL_SECS, isPrivate, serviceId, contentType, ownerId, accountNameInPost, containerNameInPost);
    String blobId;
    byte[] usermetadata = null;
    if (multipartPost) {
        usermetadata = TestUtils.getRandomString(32).getBytes();
        blobId = multipartPostBlobAndVerify(headers, content, ByteBuffer.wrap(usermetadata));
    } else {
        headers.add(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key1", "value1");
        headers.add(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key2", "value2");
        blobId = postBlobAndVerify(headers, content, contentSize);
    }
    headers.add(RestUtils.Headers.BLOB_SIZE, content.capacity());
    headers.add(RestUtils.Headers.LIFE_VERSION, "0");
    getBlobAndVerify(blobId, null, null, false, headers, isPrivate, content, expectedAccountName, expectedContainerName);
    getHeadAndVerify(blobId, null, null, headers, isPrivate, expectedAccountName, expectedContainerName);
    getBlobAndVerify(blobId, null, GetOption.None, false, headers, isPrivate, content, expectedAccountName, expectedContainerName);
    getHeadAndVerify(blobId, null, GetOption.None, headers, isPrivate, expectedAccountName, expectedContainerName);
    ByteRange range = ByteRanges.fromLastNBytes(ThreadLocalRandom.current().nextLong(content.capacity() + 1));
    headers.add(RestUtils.Headers.BLOB_SIZE, range.getRangeSize());
    getBlobAndVerify(blobId, range, null, false, headers, isPrivate, content, expectedAccountName, expectedContainerName);
    getHeadAndVerify(blobId, range, null, headers, isPrivate, expectedAccountName, expectedContainerName);
    if (contentSize > 0) {
        range = ByteRanges.fromStartOffset(ThreadLocalRandom.current().nextLong(content.capacity()));
        getBlobAndVerify(blobId, range, null, false, headers, isPrivate, content, expectedAccountName, expectedContainerName);
        getHeadAndVerify(blobId, range, null, headers, isPrivate, expectedAccountName, expectedContainerName);
        long random1 = ThreadLocalRandom.current().nextLong(content.capacity());
        long random2 = ThreadLocalRandom.current().nextLong(content.capacity());
        range = ByteRanges.fromOffsetRange(Math.min(random1, random2), Math.max(random1, random2));
        getBlobAndVerify(blobId, range, null, false, headers, isPrivate, content, expectedAccountName, expectedContainerName);
        getHeadAndVerify(blobId, range, null, headers, isPrivate, expectedAccountName, expectedContainerName);
    }
    getNotModifiedBlobAndVerify(blobId, null, isPrivate);
    getUserMetadataAndVerify(blobId, null, headers, usermetadata);
    getBlobInfoAndVerify(blobId, null, headers, isPrivate, expectedAccountName, expectedContainerName, usermetadata);
    updateBlobTtlAndVerify(blobId, headers, isPrivate, expectedAccountName, expectedContainerName, usermetadata);
    deleteBlobAndVerify(blobId);
    // check GET, HEAD, TTL update and DELETE after delete.
    verifyOperationsAfterDelete(blobId, headers, isPrivate, expectedAccountName, expectedContainerName, content, usermetadata);
    // Undelete it
    headers.add(RestUtils.Headers.LIFE_VERSION, "1");
    undeleteBlobAndVerify(blobId, headers, isPrivate, expectedAccountName, expectedContainerName, usermetadata);
}

Example 5 with ByteRange

use of com.github.ambry.router.ByteRange in project ambry by linkedin.

the class FrontendTestUrlSigningServiceFactory method doPostGetHeadDeleteTest.

// postGetHeadDeleteTest() helpers
/**
 * Tests blob POST, GET, HEAD and DELETE operations on the given {@code container}.
 * @param toPostAccount the {@link Account} to use in post headers. Can be {@code null} if only using service ID.
 * @param toPostContainer the {@link Container} to use in post headers. Can be {@code null} if only using service ID.
 * @param serviceId the serviceId to use for the POST
 * @param isPrivate the isPrivate flag to pass as part of the POST
 * @param expectedAccount the {@link Account} details that are eventually expected to be populated.
 * @param expectedContainer the {@link Container} details that are eventually expected to be populated.
 * @throws Exception
 */
private void doPostGetHeadDeleteTest(Account toPostAccount, Container toPostContainer, String serviceId, boolean isPrivate, Account expectedAccount, Container expectedContainer) throws Exception {
    final int CONTENT_LENGTH = 1024;
    ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes(CONTENT_LENGTH));
    String contentType = "application/octet-stream";
    String ownerId = "postGetHeadDeleteOwnerID";
    JSONObject headers = new JSONObject();
    String accountNameInPost = toPostAccount != null ? toPostAccount.getName() : null;
    String containerNameInPost = toPostContainer != null ? toPostContainer.getName() : null;
    setAmbryHeadersForPut(headers, 7200, isPrivate, serviceId, contentType, ownerId, accountNameInPost, containerNameInPost);
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key1", "value1");
    userMetadata.put(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key2", "value2");
    RestUtilsTest.setUserMetadataHeaders(headers, userMetadata);
    String blobId = postBlobAndVerify(headers, content, expectedAccount, expectedContainer);
    headers.put(RestUtils.Headers.BLOB_SIZE, (long) CONTENT_LENGTH);
    getBlobAndVerify(blobId, null, null, headers, content, expectedAccount, expectedContainer);
    getBlobAndVerify(blobId, null, GetOption.None, headers, content, expectedAccount, expectedContainer);
    getHeadAndVerify(blobId, null, null, headers, expectedAccount, expectedContainer);
    getHeadAndVerify(blobId, null, GetOption.None, headers, expectedAccount, expectedContainer);
    ByteRange range = ByteRange.fromStartOffset(ThreadLocalRandom.current().nextLong(CONTENT_LENGTH));
    getBlobAndVerify(blobId, range, null, headers, content, expectedAccount, expectedContainer);
    getHeadAndVerify(blobId, range, null, headers, expectedAccount, expectedContainer);
    range = ByteRange.fromLastNBytes(ThreadLocalRandom.current().nextLong(CONTENT_LENGTH + 1));
    getBlobAndVerify(blobId, range, null, headers, content, expectedAccount, expectedContainer);
    getHeadAndVerify(blobId, range, null, headers, expectedAccount, expectedContainer);
    long random1 = ThreadLocalRandom.current().nextLong(CONTENT_LENGTH);
    long random2 = ThreadLocalRandom.current().nextLong(CONTENT_LENGTH);
    range = ByteRange.fromOffsetRange(Math.min(random1, random2), Math.max(random1, random2));
    getBlobAndVerify(blobId, range, null, headers, content, expectedAccount, expectedContainer);
    getHeadAndVerify(blobId, range, null, headers, expectedAccount, expectedContainer);
    getNotModifiedBlobAndVerify(blobId, null);
    getUserMetadataAndVerify(blobId, null, headers);
    getBlobInfoAndVerify(blobId, null, headers, expectedAccount, expectedContainer);
    deleteBlobAndVerify(blobId);
    // check GET, HEAD and DELETE after delete.
    verifyOperationsAfterDelete(blobId, headers, content, expectedAccount, expectedContainer);
}