Examples with BlobInfo com.github.ambry.messageformat.BlobInfo used on opensource projects

Example 16 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class GetBlobInfoOperation method handleBody.

/**
 * Handle the body of the response: Deserialize and set the {@link BlobInfo} to return if no decryption is required.
 * If decryption is required, submit a job for decryption.
 * @param payload the body of the response.
 * @param messageMetadata the {@link MessageMetadata} associated with the message.
 * @param messageInfo the {@link MessageInfo} associated with the message.
 * @throws IOException if there is an IOException while deserializing the body.
 * @throws MessageFormatException if there is a MessageFormatException while deserializing the body.
 */
private void handleBody(InputStream payload, MessageMetadata messageMetadata, MessageInfo messageInfo) throws IOException, MessageFormatException {
    ByteBuffer encryptionKey = messageMetadata == null ? null : messageMetadata.getEncryptionKey();
    serverBlobProperties = MessageFormatRecord.deserializeBlobProperties(payload);
    updateTtlIfRequired(serverBlobProperties, messageInfo);
    ByteBuffer userMetadata = MessageFormatRecord.deserializeUserMetadata(payload);
    if (encryptionKey == null) {
        // if blob is not encrypted, move the state to Complete
        BlobInfo blobInfo = new BlobInfo(serverBlobProperties, userMetadata.array(), messageInfo.getLifeVersion());
        operationResult = new GetBlobResultInternal(new GetBlobResult(blobInfo, null), null);
    } else {
        // submit decrypt job
        progressTracker.initializeCryptoJobTracker(CryptoJobType.DECRYPTION);
        logger.trace("Submitting decrypt job for {}", blobId);
        decryptJobMetricsTracker.onJobSubmission();
        long startTimeMs = System.currentTimeMillis();
        cryptoJobHandler.submitJob(new DecryptJob(blobId, encryptionKey.duplicate(), null, userMetadata, cryptoService, kms, options.getBlobOptions, decryptJobMetricsTracker, (DecryptJob.DecryptJobResult result, Exception exception) -> {
            decryptJobMetricsTracker.onJobResultProcessingStart();
            logger.trace("Handling decrypt job callback results for {}", blobId);
            routerMetrics.decryptTimeMs.update(System.currentTimeMillis() - startTimeMs);
            if (exception == null) {
                logger.trace("Successfully updating decrypt job callback results for {}", blobId);
                BlobInfo blobInfo = new BlobInfo(serverBlobProperties, result.getDecryptedUserMetadata().array(), messageInfo.getLifeVersion());
                operationResult = new GetBlobResultInternal(new GetBlobResult(blobInfo, null), null);
                progressTracker.setCryptoJobSuccess();
            } else {
                decryptJobMetricsTracker.incrementOperationError();
                logger.trace("Exception {} thrown on decryption for {}", exception, blobId);
                setOperationException(new RouterException("Exception thrown on decrypting the content for " + blobId, exception, RouterErrorCode.UnexpectedInternalError));
                progressTracker.setCryptoJobFailed();
            }
            decryptJobMetricsTracker.onJobResultProcessingComplete();
            routerCallback.onPollReady();
        }));
    }
}

Example 17 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class CloudOperationTest method getBlobAndAssertSuccess.

/**
 * Construct GetBlob operations with appropriate callbacks, then poll those operations until they complete,
 * and ensure that the whole blob data is read out and the contents match.
 * @param blobId id of the blob to get
 * @param expectedLifeVersion the expected lifeVersion from get operation.
 * @param expectedBlobSize the expected blob size
 * @param expectedBlobProperties  the expected {@link BlobProperties} for the blob.
 * @param expectedUserMetadata the expected user meta data
 * @param expectPutContent the expected blob content
 * @param options options of the get blob operation
 * @throws Exception Any unexpected exception
 */
private void getBlobAndAssertSuccess(final BlobId blobId, final short expectedLifeVersion, final int expectedBlobSize, final BlobProperties expectedBlobProperties, final byte[] expectedUserMetadata, final byte[] expectPutContent, final GetBlobOptionsInternal options) throws Exception {
    final CountDownLatch readCompleteLatch = new CountDownLatch(1);
    final AtomicLong readCompleteResult = new AtomicLong(0);
    // callback to compare the data
    Callback<GetBlobResultInternal> callback = (result, exception) -> {
        Assert.assertNull("Shouldn't have exception", exception);
        try {
            BlobInfo blobInfo;
            switch(options.getBlobOptions.getOperationType()) {
                case All:
                    Assert.assertFalse("not supposed to be raw mode", options.getBlobOptions.isRawMode());
                    blobInfo = result.getBlobResult.getBlobInfo();
                    Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(expectedBlobProperties, blobInfo.getBlobProperties()));
                    Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", expectedBlobSize, blobInfo.getBlobProperties().getBlobSize());
                    Assert.assertArrayEquals("User metadata must be the same", expectedUserMetadata, blobInfo.getUserMetadata());
                    Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
                    break;
                case Data:
                    Assert.assertNull("Unexpected blob info in operation result", result.getBlobResult.getBlobInfo());
                    break;
                case BlobInfo:
                    blobInfo = result.getBlobResult.getBlobInfo();
                    Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(expectedBlobProperties, blobInfo.getBlobProperties()));
                    Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", expectedBlobSize, blobInfo.getBlobProperties().getBlobSize());
                    Assert.assertNull("Unexpected blob data in operation result", result.getBlobResult.getBlobDataChannel());
                    Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
            }
        } catch (Throwable e) {
            Assert.fail("Shouldn't receive exception here");
        }
        if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo) {
            final ByteBufferAsyncWritableChannel asyncWritableChannel = new ByteBufferAsyncWritableChannel();
            Utils.newThread(() -> {
                Future<Long> readIntoFuture = result.getBlobResult.getBlobDataChannel().readInto(asyncWritableChannel, null);
                assertBlobReadSuccess(options.getBlobOptions, readIntoFuture, asyncWritableChannel, result.getBlobResult.getBlobDataChannel(), readCompleteLatch, readCompleteResult, expectedBlobSize, expectPutContent);
            }, false).start();
        } else {
            readCompleteLatch.countDown();
        }
    };
    // create GetBlobOperation
    final Map<Integer, GetOperation> correlationIdToGetOperation = new HashMap<>();
    final RequestRegistrationCallback<GetOperation> requestRegistrationCallback = new RequestRegistrationCallback<>(correlationIdToGetOperation);
    NonBlockingRouter.currentOperationsCount.incrementAndGet();
    GetBlobOperation op = new GetBlobOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, callback, routerCallback, blobIdFactory, null, null, null, time, false, null);
    requestRegistrationCallback.setRequestsToSend(new ArrayList<>());
    // Wait operation to complete
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = RouterUtils.extractResponseAndNotifyResponseHandler(responseHandler, routerMetrics, responseInfo, stream -> GetResponse.readFrom(stream, mockClusterMap), response -> {
                ServerErrorCode serverError = response.getError();
                if (serverError == ServerErrorCode.No_Error) {
                    serverError = response.getPartitionResponseInfoList().get(0).getErrorCode();
                }
                return serverError;
            });
            op.handleResponse(responseInfo, getResponse);
            responseInfo.release();
        }
    }
    readCompleteLatch.await();
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    // Ensure that a ChannelClosed exception is not set when the ReadableStreamChannel is closed correctly.
    Assert.assertNull("Callback operation exception should be null", op.getOperationException());
    if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo && !options.getBlobOptions.isRawMode() && !options.getChunkIdsOnly) {
        int sizeWritten = expectedBlobSize;
        if (options.getBlobOptions.getRange() != null) {
            ByteRange range = options.getBlobOptions.getRange().toResolvedByteRange(expectedBlobSize, options.getBlobOptions.resolveRangeOnEmptyBlob());
            sizeWritten = (int) range.getRangeSize();
        }
        Assert.assertEquals("Size read must equal size written", sizeWritten, readCompleteResult.get());
    }
}

Example 18 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class GetBlobOperationTest method getAndAssertSuccess.

/**
 * Construct GetBlob operations with appropriate callbacks, then poll those operations until they complete,
 * and ensure that the whole blob data is read out and the contents match.
 * @param getChunksBeforeRead {@code true} if all chunks should be cached by the router before reading from the
 *                            stream.
 * @param initiateReadBeforeChunkGet Whether readInto() should be initiated immediately before data chunks are
 *                                   fetched by the router to simulate chunk arrival delay.
 * @param expectedLifeVersion the expected lifeVersion from get operation.
 */
private void getAndAssertSuccess(final boolean getChunksBeforeRead, final boolean initiateReadBeforeChunkGet, short expectedLifeVersion) throws Exception {
    final CountDownLatch readCompleteLatch = new CountDownLatch(1);
    final AtomicReference<Throwable> readCompleteThrowable = new AtomicReference<>(null);
    final AtomicLong readCompleteResult = new AtomicLong(0);
    final AtomicReference<Exception> operationException = new AtomicReference<>(null);
    final int numChunks = ((blobSize + maxChunkSize - 1) / maxChunkSize) + (blobSize > maxChunkSize ? 1 : 0);
    mockNetworkClient.resetProcessedResponseCount();
    Callback<GetBlobResultInternal> callback = (result, exception) -> {
        if (exception != null) {
            operationException.set(exception);
            readCompleteLatch.countDown();
        } else {
            try {
                if (options.getChunkIdsOnly) {
                    Assert.assertNull("Unexpected blob result when getChunkIdsOnly", result.getBlobResult);
                    if (blobSize > maxChunkSize) {
                        // CompositeBlob
                        Assert.assertNotNull("CompositeBlob should return a list of blob ids when getting chunk ids", result.storeKeys);
                        Assert.assertEquals(result.storeKeys.size(), (blobSize + maxChunkSize - 1) / maxChunkSize);
                    } else {
                        // SimpleBlob
                        Assert.assertNull("Unexpected list of blob id when getChunkIdsOnly is true on a simple blob", result.storeKeys);
                    }
                    readCompleteLatch.countDown();
                    return;
                }
                BlobInfo blobInfo;
                switch(options.getBlobOptions.getOperationType()) {
                    case All:
                        if (!options.getBlobOptions.isRawMode()) {
                            blobInfo = result.getBlobResult.getBlobInfo();
                            Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(blobProperties, blobInfo.getBlobProperties()));
                            Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", blobSize, blobInfo.getBlobProperties().getBlobSize());
                            Assert.assertArrayEquals("User metadata must be the same", userMetadata, blobInfo.getUserMetadata());
                            Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
                        }
                        break;
                    case Data:
                        Assert.assertNull("Unexpected blob info in operation result", result.getBlobResult.getBlobInfo());
                        break;
                    case BlobInfo:
                        blobInfo = result.getBlobResult.getBlobInfo();
                        Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(blobProperties, blobInfo.getBlobProperties()));
                        Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", blobSize, blobInfo.getBlobProperties().getBlobSize());
                        Assert.assertNull("Unexpected blob data in operation result", result.getBlobResult.getBlobDataChannel());
                        Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
                }
            } catch (Throwable e) {
                readCompleteThrowable.set(e);
            }
            if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo) {
                final ByteBufferAsyncWritableChannel asyncWritableChannel = new ByteBufferAsyncWritableChannel();
                final Future<Long> preSetReadIntoFuture = initiateReadBeforeChunkGet ? result.getBlobResult.getBlobDataChannel().readInto(asyncWritableChannel, null) : null;
                Utils.newThread(() -> {
                    if (getChunksBeforeRead) {
                        // wait for all chunks (data + metadata) to be received
                        while (mockNetworkClient.getProcessedResponseCount() < numChunks * routerConfig.routerGetRequestParallelism) {
                            Thread.yield();
                        }
                    }
                    Future<Long> readIntoFuture = initiateReadBeforeChunkGet ? preSetReadIntoFuture : result.getBlobResult.getBlobDataChannel().readInto(asyncWritableChannel, null);
                    assertBlobReadSuccess(options.getBlobOptions, readIntoFuture, asyncWritableChannel, result.getBlobResult.getBlobDataChannel(), readCompleteLatch, readCompleteResult, readCompleteThrowable);
                }, false).start();
            } else {
                readCompleteLatch.countDown();
            }
        }
    };
    GetBlobOperation op = createOperationAndComplete(callback);
    readCompleteLatch.await();
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    if (operationException.get() != null) {
        throw operationException.get();
    }
    if (readCompleteThrowable.get() != null) {
        throw new IllegalStateException(readCompleteThrowable.get());
    }
    // Ensure that a ChannelClosed exception is not set when the ReadableStreamChannel is closed correctly.
    Assert.assertNull("Callback operation exception should be null", op.getOperationException());
    if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo && !options.getBlobOptions.isRawMode() && !options.getChunkIdsOnly) {
        int sizeWritten = blobSize;
        if (options.getBlobOptions.getRange() != null) {
            ByteRange range = options.getBlobOptions.getRange().toResolvedByteRange(blobSize, options.getBlobOptions.resolveRangeOnEmptyBlob());
            sizeWritten = (int) range.getRangeSize();
        }
        Assert.assertEquals("Size read must equal size written", sizeWritten, readCompleteResult.get());
    }
}

Example 19 with BlobInfo

use of com.github.ambry.messageformat.BlobInfo in project ambry by linkedin.

the class MockRouterCallback method assertSuccess.

/**
 * Assert that the operation is complete and successful. Note that the future completion and callback invocation
 * happens outside of the GetOperation, so those are not checked here. But at this point, the operation result should
 * be ready.
 * @param op the {@link GetBlobInfoOperation} that should have completed.
 * @param expectedLifeVersion the expected lifeVersion.
 */
private void assertSuccess(GetBlobInfoOperation op, short expectedLifeVersion) {
    Assert.assertNull("Null expected", op.getOperationException());
    BlobInfo blobInfo = op.getOperationResult().getBlobResult.getBlobInfo();
    Assert.assertNull("Unexpected blob data channel in operation result", op.getOperationResult().getBlobResult.getBlobDataChannel());
    Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(blobProperties, blobInfo.getBlobProperties()));
    Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", BLOB_SIZE, blobInfo.getBlobProperties().getBlobSize());
    Assert.assertArrayEquals("User metadata must be the same", userMetadata, blobInfo.getUserMetadata());
    Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
}