Examples with BlobInfo com.github.ambry.messageformat.BlobInfo used on opensource projects

Example 6 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class AmbrySecurityServiceTest method processResponseTest.

/**
 * Tests {@link AmbrySecurityService#processResponse(RestRequest, RestResponseChannel, BlobInfo, Callback)}  for
 * common as well as uncommon cases
 * @throws Exception
 */
@Test
public void processResponseTest() throws Exception {
    RestRequest restRequest = createRestRequest(RestMethod.GET, "/", null);
    // rest request being null
    TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(null, new MockRestResponseChannel(), DEFAULT_INFO).get(), null);
    // restResponseChannel being null
    TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(restRequest, null, DEFAULT_INFO).get(), null);
    // blob info being null
    TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(restRequest, new MockRestResponseChannel(), null).get(), null);
    // for unsupported methods
    RestMethod[] methods = { RestMethod.DELETE };
    for (RestMethod restMethod : methods) {
        testExceptionCasesProcessResponse(restMethod, new MockRestResponseChannel(), DEFAULT_INFO, RestServiceErrorCode.InternalServerError);
    }
    // OPTIONS (should be no errors)
    securityService.processResponse(createRestRequest(RestMethod.OPTIONS, "/", null), new MockRestResponseChannel(), null).get();
    // PUT (should be no errors)
    securityService.processResponse(createRestRequest(RestMethod.PUT, "/", null), new MockRestResponseChannel(), null).get();
    // GET signed URL (should be no errors)
    securityService.processResponse(createRestRequest(RestMethod.GET, Operations.GET_SIGNED_URL, null), new MockRestResponseChannel(), null).get();
    // HEAD
    // normal
    testHeadBlobWithVariousRanges(DEFAULT_INFO);
    // with lifeVersion
    testHeadBlobWithVariousRanges(LIFEVERSION_INFO);
    // unknown account
    testHeadBlobWithVariousRanges(UNKNOWN_INFO);
    // encrypted unknown account
    testHeadBlobWithVariousRanges(UNKNOWN_INFO_ENC);
    // with no owner id
    BlobInfo blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, null, "image/gif", false, Utils.Infinite_Time, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
    testHeadBlobWithVariousRanges(blobInfo);
    // with no content type
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, null, false, Utils.Infinite_Time, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
    testHeadBlobWithVariousRanges(blobInfo);
    // with a TTL
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
    testHeadBlobWithVariousRanges(blobInfo);
    // GET BlobInfo
    testGetSubResource(DEFAULT_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(LIFEVERSION_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(UNKNOWN_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(UNKNOWN_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(UNKNOWN_INFO_ENC, RestUtils.SubResource.BlobInfo);
    // GET UserMetadata
    testGetSubResource(DEFAULT_INFO, RestUtils.SubResource.UserMetadata);
    byte[] usermetadata = TestUtils.getRandomBytes(10);
    testGetSubResource(new BlobInfo(DEFAULT_INFO.getBlobProperties(), usermetadata), RestUtils.SubResource.UserMetadata);
    // POST
    testPostBlob();
    // GET Blob
    testGetBlobWithVariousRanges(LIFEVERSION_INFO);
    // less than chunk threshold size
    blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes - 1, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // == chunk threshold size
    blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // more than chunk threshold size
    blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes * 2, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // Get blob with content type null
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, null, true, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // Get blob in a non-cacheable container. AmbrySecurityService should not care about the isPrivate setting.
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", false, Utils.Infinite_Time, Account.UNKNOWN_ACCOUNT_ID, Container.DEFAULT_PRIVATE_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // Get blob in a cacheable container. AmbrySecurityService should not care about the isPrivate setting.
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", true, Utils.Infinite_Time, Account.UNKNOWN_ACCOUNT_ID, Container.DEFAULT_PUBLIC_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // not modified response
    // > creation time (in secs).
    testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs() + 1000);
    // == creation time
    testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs());
    // < creation time (in secs)
    testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs() - 1000);
    // Get blob for a public blob with content type as "text/html"
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "text/html", true, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // not modified response
    // > creation time (in secs).
    testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs() + 1000);
    // == creation time
    testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs());
    // < creation time (in secs)
    testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs() - 1000);
    // bad rest response channel
    testExceptionCasesProcessResponse(RestMethod.HEAD, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
    testExceptionCasesProcessResponse(RestMethod.GET, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
    testExceptionCasesProcessResponse(RestMethod.POST, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
    // security service closed
    securityService.close();
    methods = new RestMethod[] { RestMethod.POST, RestMethod.GET, RestMethod.DELETE, RestMethod.HEAD };
    for (RestMethod restMethod : methods) {
        testExceptionCasesProcessResponse(restMethod, new MockRestResponseChannel(), blobInfo, RestServiceErrorCode.ServiceUnavailable);
    }
}

Example 7 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class SimpleAmbryCostModelPolicyTest method testCalculateRequestCost.

@Test
public void testCalculateRequestCost() throws Exception {
    SimpleAmbryCostModelPolicy requestCostPolicy = new SimpleAmbryCostModelPolicy();
    RestResponseChannel restResponseChannel = mock(RestResponseChannel.class);
    when(restResponseChannel.getHeader(anyString())).thenReturn(0);
    String blobUri = "/AAYIAQSSAAgAAQAAAAAAABpFymbGwe7sRBWYa5OPlkcNHQ.bin";
    // test for a 4 MB GET request.
    BlobInfo blobInfo = getBlobInfo(4 * MB);
    RestRequest restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri, -1);
    Map<String, Double> costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // test for a small GET request (fractional CU).
    blobInfo = getBlobInfo(6 * MB);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 2);
    // test for a GET request of blob of size 0.
    blobInfo = getBlobInfo(0);
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // test for a small POST request (fractional storage cost).
    blobInfo = getBlobInfo(8 * MB);
    restRequest = createMockRequestWithMethod(RestMethod.POST, blobUri, 8 * MB);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 2, 8 * 1024 * 1024 / (double) QuotaUtils.BYTES_IN_GB);
    // test for a large POST request.
    blobInfo = getBlobInfo(4 * GB);
    restRequest = createMockRequestWithMethod(RestMethod.POST, blobUri, 4 * GB);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1024, 4);
    // test for a POST request of blob of size 0.
    blobInfo = getBlobInfo(0);
    restRequest = createMockRequestWithMethod(RestMethod.POST, blobUri, 0);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1, 0);
    // test for a HEAD request.
    restRequest = createMockRequestWithMethod(RestMethod.HEAD, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // test for a DELETE request.
    restRequest = createMockRequestWithMethod(RestMethod.DELETE, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1, 0.0);
    // test for a PUT request.
    restRequest = createMockRequestWithMethod(RestMethod.PUT, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1, 0.0);
    // test for PUT with null blob info.
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, null);
    verifyWriteCost(costMap, 1, 0.0);
    // test BlobInfo and UserMetadata GET requests
    blobInfo = getBlobInfo(40 * GB);
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri + "/BlobInfo", -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri + "/UserMetadata", -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // Plain GET should use blob size
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 10240);
// TODO add a range request case with large range
}

Example 8 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class MockRouterCallback method assertSuccess.

/**
 * Assert that the operation is complete and successful. Note that the future completion and callback invocation
 * happens outside of the GetOperation, so those are not checked here. But at this point, the operation result should
 * be ready.
 * @param op the {@link GetBlobInfoOperation} that should have completed.
 */
private void assertSuccess(GetBlobInfoOperation op) {
    Assert.assertEquals("Null expected", null, op.getOperationException());
    BlobInfo blobInfo = op.getOperationResult().getBlobResult.getBlobInfo();
    Assert.assertNull("Unexpected blob data channel in operation result", op.getOperationResult().getBlobResult.getBlobDataChannel());
    Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.haveEquivalentFields(blobProperties, blobInfo.getBlobProperties()));
    Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", BLOB_SIZE, blobInfo.getBlobProperties().getBlobSize());
    Assert.assertArrayEquals("User metadata must be the same", userMetadata, blobInfo.getUserMetadata());
}

Example 9 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class GetBlobOperationTest method getAndAssertSuccess.

/**
 * Construct GetBlob operations with appropriate callbacks, then poll those operations until they complete,
 * and ensure that the whole blob data is read out and the contents match.
 * @param getChunksBeforeRead {@code true} if all chunks should be cached by the router before reading from the
 *                            stream.
 * @param initiateReadBeforeChunkGet Whether readInto() should be initiated immediately before data chunks are
 *                                   fetched by the router to simulate chunk arrival delay.
 */
private void getAndAssertSuccess(final boolean getChunksBeforeRead, final boolean initiateReadBeforeChunkGet) throws Exception {
    final CountDownLatch readCompleteLatch = new CountDownLatch(1);
    final AtomicReference<Exception> readCompleteException = new AtomicReference<>(null);
    final AtomicLong readCompleteResult = new AtomicLong(0);
    final AtomicReference<Exception> operationException = new AtomicReference<>(null);
    final int numChunks = ((blobSize + maxChunkSize - 1) / maxChunkSize) + (blobSize > maxChunkSize ? 1 : 0);
    mockNetworkClient.resetProcessedResponseCount();
    Callback<GetBlobResultInternal> callback = new Callback<GetBlobResultInternal>() {

        @Override
        public void onCompletion(final GetBlobResultInternal result, final Exception exception) {
            if (exception != null) {
                operationException.set(exception);
                readCompleteLatch.countDown();
            } else {
                try {
                    switch(options.getBlobOptions.getOperationType()) {
                        case All:
                            BlobInfo blobInfo = result.getBlobResult.getBlobInfo();
                            Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.haveEquivalentFields(blobProperties, blobInfo.getBlobProperties()));
                            Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", blobSize, blobInfo.getBlobProperties().getBlobSize());
                            Assert.assertArrayEquals("User metadata must be the same", userMetadata, blobInfo.getUserMetadata());
                            break;
                        case Data:
                            Assert.assertNull("Unexpected blob info in operation result", result.getBlobResult.getBlobInfo());
                            break;
                    }
                } catch (Exception e) {
                    readCompleteException.set(e);
                }
                final ByteBufferAsyncWritableChannel asyncWritableChannel = new ByteBufferAsyncWritableChannel();
                final Future<Long> preSetReadIntoFuture = initiateReadBeforeChunkGet ? result.getBlobResult.getBlobDataChannel().readInto(asyncWritableChannel, null) : null;
                Utils.newThread(new Runnable() {

                    @Override
                    public void run() {
                        if (getChunksBeforeRead) {
                            // wait for all chunks (data + metadata) to be received
                            while (mockNetworkClient.getProcessedResponseCount() < numChunks * routerConfig.routerGetRequestParallelism) {
                                Thread.yield();
                            }
                        }
                        Future<Long> readIntoFuture = initiateReadBeforeChunkGet ? preSetReadIntoFuture : result.getBlobResult.getBlobDataChannel().readInto(asyncWritableChannel, null);
                        assertBlobReadSuccess(options.getBlobOptions, readIntoFuture, asyncWritableChannel, result.getBlobResult.getBlobDataChannel(), readCompleteLatch, readCompleteResult, readCompleteException);
                    }
                }, false).start();
            }
        }
    };
    GetBlobOperation op = createOperationAndComplete(callback);
    readCompleteLatch.await();
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    if (operationException.get() != null) {
        throw operationException.get();
    }
    if (readCompleteException.get() != null) {
        throw readCompleteException.get();
    }
    // Ensure that a ChannelClosed exception is not set when the ReadableStreamChannel is closed correctly.
    Assert.assertNull("Callback operation exception should be null", op.getOperationException());
    if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo) {
        int sizeWritten = blobSize;
        if (options.getBlobOptions.getRange() != null) {
            ByteRange range = options.getBlobOptions.getRange().toResolvedByteRange(blobSize);
            sizeWritten = (int) (range.getEndOffset() - range.getStartOffset() + 1);
        }
        Assert.assertEquals("Size read must equal size written", sizeWritten, readCompleteResult.get());
    }
}

Example 10 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class AmbryBlobStorageService method handlePost.

@Override
public void handlePost(RestRequest restRequest, RestResponseChannel restResponseChannel) {
    long processingStartTime = System.currentTimeMillis();
    long preProcessingTime = 0;
    handlePrechecks(restRequest, restResponseChannel);
    boolean sslUsed = restRequest.getSSLSession() != null;
    RestRequestMetrics metrics = frontendMetrics.postRequestMetricsGroup.getRestRequestMetrics(sslUsed, false);
    restRequest.getMetricsTracker().injectMetrics(metrics);
    try {
        logger.trace("Handling POST request - {}", restRequest.getUri());
        checkAvailable();
        // TODO: make this non blocking once all handling of indiviual methods is moved to their own classes
        securityService.preProcessRequest(restRequest).get();
        long propsBuildStartTime = System.currentTimeMillis();
        accountAndContainerInjector.injectAccountAndContainerForPostRequest(restRequest);
        BlobProperties blobProperties = RestUtils.buildBlobProperties(restRequest.getArgs());
        if (blobProperties.getTimeToLiveInSeconds() + TimeUnit.MILLISECONDS.toSeconds(blobProperties.getCreationTimeInMs()) > Integer.MAX_VALUE) {
            logger.debug("TTL set to very large value in POST request with BlobProperties {}", blobProperties);
            frontendMetrics.ttlTooLargeError.inc();
        }
        // inject encryption metrics if applicable
        if (blobProperties.isEncrypted()) {
            metrics = frontendMetrics.postRequestMetricsGroup.getRestRequestMetrics(sslUsed, true);
            restRequest.getMetricsTracker().injectMetrics(metrics);
        }
        byte[] usermetadata = RestUtils.buildUsermetadata(restRequest.getArgs());
        frontendMetrics.blobPropsBuildTimeInMs.update(System.currentTimeMillis() - propsBuildStartTime);
        logger.trace("Blob properties of blob being POSTed - {}", blobProperties);
        PostCallback routerCallback = new PostCallback(restRequest, restResponseChannel, new BlobInfo(blobProperties, usermetadata));
        preProcessingTime = System.currentTimeMillis() - processingStartTime;
        SecurityProcessRequestCallback securityCallback = new SecurityProcessRequestCallback(restRequest, restResponseChannel, blobProperties, usermetadata, routerCallback);
        securityService.processRequest(restRequest, securityCallback);
    } catch (Exception e) {
        submitResponse(restRequest, restResponseChannel, null, extractExecutionExceptionCause(e));
    } finally {
        frontendMetrics.postPreProcessingTimeInMs.update(preProcessingTime);
    }
}