Examples with ChunkInfo com.github.ambry.router.ChunkInfo used on opensource projects

Example 1 with ChunkInfo

use of com.github.ambry.router.ChunkInfo in project ambry by linkedin.

the class PostBlobHandlerTest method stitchedUploadTest.

/**
 * Test flows related to the {@link Operations#STITCH} operation.
 * @throws Exception
 */
@Test
public void stitchedUploadTest() throws Exception {
    idConverterFactory.translation = CONVERTED_ID;
    String uploadSession = UUID.randomUUID().toString();
    long creationTimeMs = System.currentTimeMillis();
    time.setCurrentMilliseconds(creationTimeMs);
    String[] prefixToTest = new String[] { "/" + CLUSTER_NAME, "" };
    for (String prefix : prefixToTest) {
        // success cases
        // multiple chunks
        List<ChunkInfo> chunksToStitch = uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 45, 10, 200, 19, 0, 50);
        List<String> signedChunkIds = chunksToStitch.stream().map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), chunksToStitch, null);
        // one chunk
        chunksToStitch = uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 45);
        signedChunkIds = chunksToStitch.stream().map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), chunksToStitch, null);
        // failure cases
        // invalid json input
        stitchBlobAndVerify("badjsonbadjson".getBytes(StandardCharsets.UTF_8), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // no chunk ids in request
        stitchBlobAndVerify(getStitchRequestBody(Collections.emptyList()), null, restServiceExceptionChecker(RestServiceErrorCode.MissingArgs));
        stitchBlobAndVerify(new JSONObject().toString().getBytes(StandardCharsets.UTF_8), null, restServiceExceptionChecker(RestServiceErrorCode.MissingArgs));
        // differing session IDs
        signedChunkIds = uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 45, 22).stream().map(chunkInfo -> prefix + getSignedId(chunkInfo, UUID.randomUUID().toString())).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // differing containers
        signedChunkIds = Stream.concat(uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 50, 50).stream(), uploadChunksViaRouter(creationTimeMs, REF_CONTAINER_WITH_TTL_REQUIRED, 50).stream()).map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // differing accounts
        Container altAccountContainer = ACCOUNT_SERVICE.createAndAddRandomAccount().getContainerById(Container.DEFAULT_PRIVATE_CONTAINER_ID);
        signedChunkIds = Stream.concat(uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 50, 50).stream(), uploadChunksViaRouter(creationTimeMs, altAccountContainer, 50).stream()).map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // invalid blob ID
        stitchBlobAndVerify(getStitchRequestBody(Collections.singletonList(getSignedId(new ChunkInfo("abcd", 200, -1), uploadSession))), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // unsigned ID
        stitchBlobAndVerify(getStitchRequestBody(Collections.singletonList("/notASignedId")), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
    }
}

Example 2 with ChunkInfo

use of com.github.ambry.router.ChunkInfo in project ambry by linkedin.

the class PostBlobHandlerTest method stitchBlobAndVerify.

/**
 * Make a stitch blob call using {@link PostBlobHandler} and verify the result of the operation.
 * @param requestBody the body of the stitch request to supply.
 * @param expectedStitchedChunks the expected chunks stitched together.
 * @param errorChecker if non-null, expect an exception to be thrown by the post flow and verify it using this
 *                     {@link ThrowingConsumer}.
 * @throws Exception
 */
private void stitchBlobAndVerify(byte[] requestBody, List<ChunkInfo> expectedStitchedChunks, ThrowingConsumer<ExecutionException> errorChecker) throws Exception {
    JSONObject headers = new JSONObject();
    FrontendRestRequestServiceTest.setAmbryHeadersForPut(headers, TestUtils.TTL_SECS, !REF_CONTAINER.isCacheable(), SERVICE_ID, CONTENT_TYPE, OWNER_ID, REF_ACCOUNT.getName(), REF_CONTAINER.getName(), null);
    RestRequest request = getRestRequest(headers, "/" + Operations.STITCH, requestBody);
    RestResponseChannel restResponseChannel = new MockRestResponseChannel();
    FutureResult<Void> future = new FutureResult<>();
    idConverterFactory.lastInput = null;
    postBlobHandler.handle(request, restResponseChannel, future::done);
    if (errorChecker == null) {
        future.get(TIMEOUT_SECS, TimeUnit.SECONDS);
        assertEquals("Unexpected converted ID", CONVERTED_ID, restResponseChannel.getHeader(RestUtils.Headers.LOCATION));
        InMemoryRouter.InMemoryBlob blob = router.getActiveBlobs().get(idConverterFactory.lastInput);
        assertEquals("List of chunks stitched does not match expected", expectedStitchedChunks, blob.getStitchedChunks());
        ByteArrayOutputStream expectedContent = new ByteArrayOutputStream();
        expectedStitchedChunks.stream().map(chunkInfo -> router.getActiveBlobs().get(chunkInfo.getBlobId()).getBlob().array()).forEach(buf -> expectedContent.write(buf, 0, buf.length));
        assertEquals("Unexpected blob content stored", ByteBuffer.wrap(expectedContent.toByteArray()), blob.getBlob());
        // check actual size of stitched blob
        assertEquals("Unexpected blob size", Long.toString(getStitchedBlobSize(expectedStitchedChunks)), restResponseChannel.getHeader(RestUtils.Headers.BLOB_SIZE));
    } else {
        TestUtils.assertException(ExecutionException.class, () -> future.get(TIMEOUT_SECS, TimeUnit.SECONDS), errorChecker);
    }
}

Example 3 with ChunkInfo

use of com.github.ambry.router.ChunkInfo in project ambry by linkedin.

the class NamedBlobPutHandlerTest method stitchBlobAndVerify.

/**
 * Make a stitch blob call using {@link PostBlobHandler} and verify the result of the operation.
 * @param requestBody the body of the stitch request to supply.
 * @param expectedStitchedChunks the expected chunks stitched together.
 * @param errorChecker if non-null, expect an exception to be thrown by the post flow and verify it using this
 *                     {@link ThrowingConsumer}.
 * @throws Exception
 */
private void stitchBlobAndVerify(byte[] requestBody, List<ChunkInfo> expectedStitchedChunks, ThrowingConsumer<ExecutionException> errorChecker) throws Exception {
    // call
    for (long ttl : new long[] { TestUtils.TTL_SECS, Utils.Infinite_Time }) {
        JSONObject headers = new JSONObject();
        FrontendRestRequestServiceTest.setAmbryHeadersForPut(headers, ttl, !REF_CONTAINER.isCacheable(), SERVICE_ID, CONTENT_TYPE, OWNER_ID, null, null, "STITCH");
        RestRequest request = getRestRequest(headers, request_path, requestBody);
        RestResponseChannel restResponseChannel = new MockRestResponseChannel();
        FutureResult<Void> future = new FutureResult<>();
        idConverterFactory.lastInput = null;
        idConverterFactory.lastBlobInfo = null;
        idConverterFactory.lastConvertedId = null;
        namedBlobPutHandler.handle(request, restResponseChannel, future::done);
        if (errorChecker == null) {
            future.get(TIMEOUT_SECS, TimeUnit.SECONDS);
            assertEquals("Unexpected location header", idConverterFactory.lastConvertedId, restResponseChannel.getHeader(RestUtils.Headers.LOCATION));
            InMemoryRouter.InMemoryBlob blob = router.getActiveBlobs().get(idConverterFactory.lastInput);
            assertEquals("List of chunks stitched does not match expected", expectedStitchedChunks, blob.getStitchedChunks());
            ByteArrayOutputStream expectedContent = new ByteArrayOutputStream();
            expectedStitchedChunks.stream().map(chunkInfo -> router.getActiveBlobs().get(chunkInfo.getBlobId()).getBlob().array()).forEach(buf -> expectedContent.write(buf, 0, buf.length));
            assertEquals("Unexpected blob content stored", ByteBuffer.wrap(expectedContent.toByteArray()), blob.getBlob());
            // check actual size of stitched blob
            assertEquals("Unexpected blob size", Long.toString(getStitchedBlobSize(expectedStitchedChunks)), restResponseChannel.getHeader(RestUtils.Headers.BLOB_SIZE));
            assertEquals("Unexpected TTL in named blob DB", ttl, idConverterFactory.lastBlobInfo.getBlobProperties().getTimeToLiveInSeconds());
            assertEquals("Unexpected TTL in blob", ttl, blob.getBlobProperties().getTimeToLiveInSeconds());
        } else {
            TestUtils.assertException(ExecutionException.class, () -> future.get(TIMEOUT_SECS, TimeUnit.SECONDS), errorChecker);
        }
    }
}

Example 4 with ChunkInfo

use of com.github.ambry.router.ChunkInfo in project ambry by linkedin.

the class NamedBlobPutHandlerTest method uploadChunksViaRouter.

/**
 * Upload chunks using the router directly.
 * @param creationTimeMs the creation time to set for the chunks.
 * @param container the {@link Container} to create the chunks in.
 * @param chunkSizes the sizes for each chunk to upload.
 * @return a list of {@link ChunkInfo} objects that contains metadata about each chunk uploaded.
 */
private List<ChunkInfo> uploadChunksViaRouter(long creationTimeMs, Container container, int... chunkSizes) throws Exception {
    long blobTtlSecs = TimeUnit.DAYS.toSeconds(1);
    List<ChunkInfo> chunks = new ArrayList<>();
    for (int chunkSize : chunkSizes) {
        byte[] content = TestUtils.getRandomBytes(chunkSize);
        BlobProperties blobProperties = new BlobProperties(-1, SERVICE_ID, OWNER_ID, CONTENT_TYPE, !container.isCacheable(), blobTtlSecs, creationTimeMs, container.getParentAccountId(), container.getId(), container.isEncrypted(), null, null, null);
        String blobId = router.putBlob(blobProperties, null, new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content)), new PutBlobOptionsBuilder().chunkUpload(true).build()).get(TIMEOUT_SECS, TimeUnit.SECONDS);
        chunks.add(new ChunkInfo(blobId, chunkSize, Utils.addSecondsToEpochTime(creationTimeMs, blobTtlSecs)));
    }
    return chunks;
}

Example 5 with ChunkInfo

use of com.github.ambry.router.ChunkInfo in project ambry by linkedin.

the class NamedBlobPutHandlerTest method stitchNamedBlobTest.

/**
 * Test flows related to the stitch named blob operation.
 * @throws Exception
 */
@Test
public void stitchNamedBlobTest() throws Exception {
    idConverterFactory.translation = CONVERTED_ID;
    long creationTimeMs = System.currentTimeMillis();
    time.setCurrentMilliseconds(creationTimeMs);
    // success case
    String uploadSession = UUID.randomUUID().toString();
    String[] prefixToTest = new String[] { "/" + CLUSTER_NAME, "" };
    for (String prefix : prefixToTest) {
        // multiple chunks
        List<ChunkInfo> chunksToStitch = uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 45, 10, 200, 19, 0, 50);
        List<String> signedChunkIds = chunksToStitch.stream().map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), chunksToStitch, null);
        // one chunk
        chunksToStitch = uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 45);
        signedChunkIds = chunksToStitch.stream().map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), chunksToStitch, null);
        // failure cases
        // invalid json input
        stitchBlobAndVerify("badjsonbadjson".getBytes(StandardCharsets.UTF_8), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // no chunk ids in request
        stitchBlobAndVerify(getStitchRequestBody(Collections.emptyList()), null, restServiceExceptionChecker(RestServiceErrorCode.MissingArgs));
        stitchBlobAndVerify(new JSONObject().toString().getBytes(StandardCharsets.UTF_8), null, restServiceExceptionChecker(RestServiceErrorCode.MissingArgs));
        // differing session IDs
        signedChunkIds = uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 45, 22).stream().map(chunkInfo -> prefix + getSignedId(chunkInfo, UUID.randomUUID().toString())).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // differing containers
        signedChunkIds = Stream.concat(uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 50, 50).stream(), uploadChunksViaRouter(creationTimeMs, REF_CONTAINER_WITH_TTL_REQUIRED, 50).stream()).map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // differing accounts
        Container altAccountContainer = ACCOUNT_SERVICE.createAndAddRandomAccount().getContainerById(Container.DEFAULT_PRIVATE_CONTAINER_ID);
        signedChunkIds = Stream.concat(uploadChunksViaRouter(creationTimeMs, REF_CONTAINER, 50, 50).stream(), uploadChunksViaRouter(creationTimeMs, altAccountContainer, 50).stream()).map(chunkInfo -> prefix + getSignedId(chunkInfo, uploadSession)).collect(Collectors.toList());
        stitchBlobAndVerify(getStitchRequestBody(signedChunkIds), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // invalid blob ID
        stitchBlobAndVerify(getStitchRequestBody(Collections.singletonList(getSignedId(new ChunkInfo("abcd", 200, -1), uploadSession))), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
        // unsigned ID
        stitchBlobAndVerify(getStitchRequestBody(Collections.singletonList("/notASignedId")), null, restServiceExceptionChecker(RestServiceErrorCode.BadRequest));
    }
}