Examples with BlobInfo com.github.ambry.messageformat.BlobInfo used on opensource projects

Example 1 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class GetBlobInfoOperation method handleBody.

/**
 * Handle the body of the response: Deserialize and set the {@link BlobInfo} to return if no decryption is required.
 * If decryption is required, submit a job for decryption.
 * @param payload the body of the response.
 * @param messageMetadata the {@link MessageMetadata} associated with the message.
 * @throws IOException if there is an IOException while deserializing the body.
 * @throws MessageFormatException if there is a MessageFormatException while deserializing the body.
 */
private void handleBody(InputStream payload, MessageMetadata messageMetadata) throws IOException, MessageFormatException {
    ByteBuffer encryptionKey = messageMetadata == null ? null : messageMetadata.getEncryptionKey();
    serverBlobProperties = MessageFormatRecord.deserializeBlobProperties(payload);
    ByteBuffer userMetadata = MessageFormatRecord.deserializeUserMetadata(payload);
    if (encryptionKey == null) {
        // if blob is not encrypted, move the state to Complete
        operationResult = new GetBlobResultInternal(new GetBlobResult(new BlobInfo(serverBlobProperties, userMetadata.array()), null), null);
    } else {
        // submit decrypt job
        progressTracker.initializeDecryptionTracker();
        logger.trace("Submitting decrypt job for {}", blobId);
        decryptJobMetricsTracker.onJobSubmission();
        long startTimeMs = System.currentTimeMillis();
        cryptoJobHandler.submitJob(new DecryptJob(blobId, encryptionKey.duplicate(), null, userMetadata, cryptoService, kms, decryptJobMetricsTracker, (DecryptJob.DecryptJobResult result, Exception exception) -> {
            decryptJobMetricsTracker.onJobResultProcessingStart();
            logger.trace("Handling decrypt job callback results for {}", blobId);
            routerMetrics.decryptTimeMs.update(System.currentTimeMillis() - startTimeMs);
            if (exception == null) {
                logger.trace("Successfully updating decrypt job callback results for {}", blobId);
                operationResult = new GetBlobResultInternal(new GetBlobResult(new BlobInfo(serverBlobProperties, result.getDecryptedUserMetadata().array()), null), null);
                progressTracker.setDecryptionSuccess();
            } else {
                decryptJobMetricsTracker.incrementOperationError();
                logger.trace("Exception {} thrown on decryption for {}", exception, blobId);
                setOperationException(new RouterException("Exception thrown on decrypting the content for " + blobId, exception, RouterErrorCode.UnexpectedInternalError));
                progressTracker.setDecryptionFailed();
            }
            decryptJobMetricsTracker.onJobResultProcessingComplete();
            routerCallback.onPollReady();
        }));
    }
}

Example 2 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class CloseWriteChannelCallback method getBlob.

@Override
public Future<GetBlobResult> getBlob(String blobId, GetBlobOptions options, Callback<GetBlobResult> callback) {
    FutureResult<GetBlobResult> futureResult = new FutureResult<>();
    handlePrechecks(futureResult, callback);
    ReadableStreamChannel blobDataChannel = null;
    BlobInfo blobInfo = null;
    Exception exception = null;
    try {
        getBlobIdFromString(blobId, clusterMap);
        if (deletedBlobs.contains(blobId) && !options.getGetOption().equals(GetOption.Include_All) && !options.getGetOption().equals(GetOption.Include_Deleted_Blobs)) {
            exception = new RouterException("Blob deleted", RouterErrorCode.BlobDeleted);
        } else if (!blobs.containsKey(blobId)) {
            exception = new RouterException("Blob not found", RouterErrorCode.BlobDoesNotExist);
        } else {
            InMemoryBlob blob = blobs.get(blobId);
            switch(options.getOperationType()) {
                case Data:
                    blobDataChannel = new ByteBufferRSC(blob.getBlob(options.getRange()));
                    break;
                case BlobInfo:
                    blobInfo = new BlobInfo(blob.getBlobProperties(), blob.getUserMetadata());
                    break;
                case All:
                    blobDataChannel = new ByteBufferRSC(blob.getBlob(options.getRange()));
                    blobInfo = new BlobInfo(blob.getBlobProperties(), blob.getUserMetadata());
                    break;
            }
        }
    } catch (RouterException e) {
        exception = e;
    } catch (Exception e) {
        exception = new RouterException(e, RouterErrorCode.UnexpectedInternalError);
    } finally {
        GetBlobResult operationResult = exception == null ? new GetBlobResult(blobInfo, blobDataChannel) : null;
        completeOperation(futureResult, callback, operationResult, exception);
    }
    return futureResult;
}

Example 3 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class PerfRouter method getBlob.

@Override
public Future<GetBlobResult> getBlob(String blobId, GetBlobOptions options, Callback<GetBlobResult> callback) {
    logger.trace("Received getBlob call");
    FutureResult<GetBlobResult> futureResult = new FutureResult<>();
    if (!routerOpen) {
        completeOperation(futureResult, callback, null, ROUTER_CLOSED_EXCEPTION);
    } else {
        GetBlobResult result = null;
        switch(options.getOperationType()) {
            case All:
                result = new GetBlobResult(new BlobInfo(blobProperties, usermetadata), new PerfRSC(chunk, blobProperties.getBlobSize()));
                break;
            case Data:
                result = new GetBlobResult(null, new PerfRSC(chunk, blobProperties.getBlobSize()));
                break;
            case BlobInfo:
                result = new GetBlobResult(new BlobInfo(blobProperties, usermetadata), null);
                break;
        }
        completeOperation(futureResult, callback, result, null);
    }
    return futureResult;
}

Example 4 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class AmbrySecurityServiceTest method processResponseTest.

/**
 * Tests {@link AmbrySecurityService#processResponse(RestRequest, RestResponseChannel, BlobInfo, Callback)}  for
 * common as well as uncommon cases
 * @throws Exception
 */
@Test
public void processResponseTest() throws Exception {
    RestRequest restRequest = createRestRequest(RestMethod.GET, "/", null);
    // rest request being null
    TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(null, new MockRestResponseChannel(), DEFAULT_INFO).get(), null);
    // restResponseChannel being null
    TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(restRequest, null, DEFAULT_INFO).get(), null);
    // blob info being null
    TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(restRequest, new MockRestResponseChannel(), null).get(), null);
    // for unsupported methods
    RestMethod[] methods = { RestMethod.DELETE };
    for (RestMethod restMethod : methods) {
        testExceptionCasesProcessResponse(restMethod, new MockRestResponseChannel(), DEFAULT_INFO, RestServiceErrorCode.InternalServerError);
    }
    // OPTIONS (should be no errors)
    securityService.processResponse(createRestRequest(RestMethod.OPTIONS, "/", null), new MockRestResponseChannel(), null).get();
    // PUT (should be no errors)
    securityService.processResponse(createRestRequest(RestMethod.PUT, "/", null), new MockRestResponseChannel(), null).get();
    // GET signed URL (should be no errors)
    securityService.processResponse(createRestRequest(RestMethod.GET, Operations.GET_SIGNED_URL, null), new MockRestResponseChannel(), null).get();
    // HEAD
    // normal
    testHeadBlobWithVariousRanges(DEFAULT_INFO);
    // with lifeVersion
    testHeadBlobWithVariousRanges(LIFEVERSION_INFO);
    // unknown account
    testHeadBlobWithVariousRanges(UNKNOWN_INFO);
    // encrypted unknown account
    testHeadBlobWithVariousRanges(UNKNOWN_INFO_ENC);
    // with no owner id
    BlobInfo blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, null, "image/gif", false, Utils.Infinite_Time, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
    testHeadBlobWithVariousRanges(blobInfo);
    // with no content type
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, null, false, Utils.Infinite_Time, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
    testHeadBlobWithVariousRanges(blobInfo);
    // with a TTL
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
    testHeadBlobWithVariousRanges(blobInfo);
    // GET BlobInfo
    testGetSubResource(DEFAULT_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(LIFEVERSION_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(UNKNOWN_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(UNKNOWN_INFO, RestUtils.SubResource.BlobInfo);
    testGetSubResource(UNKNOWN_INFO_ENC, RestUtils.SubResource.BlobInfo);
    // GET UserMetadata
    testGetSubResource(DEFAULT_INFO, RestUtils.SubResource.UserMetadata);
    byte[] usermetadata = TestUtils.getRandomBytes(10);
    testGetSubResource(new BlobInfo(DEFAULT_INFO.getBlobProperties(), usermetadata), RestUtils.SubResource.UserMetadata);
    // POST
    testPostBlob();
    // GET Blob
    testGetBlobWithVariousRanges(LIFEVERSION_INFO);
    // less than chunk threshold size
    blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes - 1, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // == chunk threshold size
    blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // more than chunk threshold size
    blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes * 2, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // Get blob with content type null
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, null, true, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // Get blob in a non-cacheable container. AmbrySecurityService should not care about the isPrivate setting.
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", false, Utils.Infinite_Time, Account.UNKNOWN_ACCOUNT_ID, Container.DEFAULT_PRIVATE_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // Get blob in a cacheable container. AmbrySecurityService should not care about the isPrivate setting.
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", true, Utils.Infinite_Time, Account.UNKNOWN_ACCOUNT_ID, Container.DEFAULT_PUBLIC_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // not modified response
    // > creation time (in secs).
    testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs() + 1000);
    // == creation time
    testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs());
    // < creation time (in secs)
    testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs() - 1000);
    // Get blob for a public blob with content type as "text/html"
    blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "text/html", true, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
    testGetBlobWithVariousRanges(blobInfo);
    // not modified response
    // > creation time (in secs).
    testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs() + 1000);
    // == creation time
    testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs());
    // < creation time (in secs)
    testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs() - 1000);
    // bad rest response channel
    testExceptionCasesProcessResponse(RestMethod.HEAD, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
    testExceptionCasesProcessResponse(RestMethod.GET, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
    testExceptionCasesProcessResponse(RestMethod.POST, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
    // security service closed
    securityService.close();
    methods = new RestMethod[] { RestMethod.POST, RestMethod.GET, RestMethod.DELETE, RestMethod.HEAD };
    for (RestMethod restMethod : methods) {
        testExceptionCasesProcessResponse(restMethod, new MockRestResponseChannel(), blobInfo, RestServiceErrorCode.ServiceUnavailable);
    }
}

Example 5 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class SimpleAmbryCostModelPolicyTest method testCalculateRequestCost.

@Test
public void testCalculateRequestCost() throws Exception {
    SimpleAmbryCostModelPolicy requestCostPolicy = new SimpleAmbryCostModelPolicy();
    RestResponseChannel restResponseChannel = mock(RestResponseChannel.class);
    when(restResponseChannel.getHeader(anyString())).thenReturn(0);
    String blobUri = "/AAYIAQSSAAgAAQAAAAAAABpFymbGwe7sRBWYa5OPlkcNHQ.bin";
    // test for a 4 MB GET request.
    BlobInfo blobInfo = getBlobInfo(4 * MB);
    RestRequest restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri, -1);
    Map<String, Double> costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // test for a small GET request (fractional CU).
    blobInfo = getBlobInfo(6 * MB);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 2);
    // test for a GET request of blob of size 0.
    blobInfo = getBlobInfo(0);
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // test for a small POST request (fractional storage cost).
    blobInfo = getBlobInfo(8 * MB);
    restRequest = createMockRequestWithMethod(RestMethod.POST, blobUri, 8 * MB);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 2, 8 * 1024 * 1024 / (double) QuotaUtils.BYTES_IN_GB);
    // test for a large POST request.
    blobInfo = getBlobInfo(4 * GB);
    restRequest = createMockRequestWithMethod(RestMethod.POST, blobUri, 4 * GB);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1024, 4);
    // test for a POST request of blob of size 0.
    blobInfo = getBlobInfo(0);
    restRequest = createMockRequestWithMethod(RestMethod.POST, blobUri, 0);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1, 0);
    // test for a HEAD request.
    restRequest = createMockRequestWithMethod(RestMethod.HEAD, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // test for a DELETE request.
    restRequest = createMockRequestWithMethod(RestMethod.DELETE, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1, 0.0);
    // test for a PUT request.
    restRequest = createMockRequestWithMethod(RestMethod.PUT, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyWriteCost(costMap, 1, 0.0);
    // test for PUT with null blob info.
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, null);
    verifyWriteCost(costMap, 1, 0.0);
    // test BlobInfo and UserMetadata GET requests
    blobInfo = getBlobInfo(40 * GB);
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri + "/BlobInfo", -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri + "/UserMetadata", -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 1);
    // Plain GET should use blob size
    restRequest = createMockRequestWithMethod(RestMethod.GET, blobUri, -1);
    costMap = requestCostPolicy.calculateRequestCost(restRequest, restResponseChannel, blobInfo);
    verifyReadCost(costMap, 10240);
// TODO add a range request case with large range
}