Search in sources :

Example 1 with BlobDataType

use of com.github.ambry.commons.BlobId.BlobDataType in project ambry by linkedin.

the class MockReadableStreamChannel method verifyBlob.

/**
 * Verifies that the blob associated with the blob id returned by a successful put operation has exactly the same
 * data as the original object that was put.
 * @param requestAndResult the {@link RequestAndResult} to use for verification.
 * @param serializedRequests the mapping from blob ids to their corresponding serialized {@link PutRequest}.
 */
private void verifyBlob(RequestAndResult requestAndResult, HashMap<String, ByteBuffer> serializedRequests) throws Exception {
    String blobId = requestAndResult.result.result();
    ByteBuffer serializedRequest = serializedRequests.get(blobId);
    PutRequest request = deserializePutRequest(serializedRequest);
    NotificationBlobType notificationBlobType;
    BlobId origBlobId = new BlobId(blobId, mockClusterMap);
    boolean stitchOperation = requestAndResult.chunksToStitch != null;
    if (stitchOperation) {
        assertEquals("Stitch operations should always produce metadata blobs", BlobType.MetadataBlob, request.getBlobType());
    }
    if (request.getBlobType() == BlobType.MetadataBlob) {
        notificationBlobType = NotificationBlobType.Composite;
        assertEquals("Expected metadata", BlobDataType.METADATA, origBlobId.getBlobDataType());
        byte[] data = Utils.readBytesFromStream(request.getBlobStream(), (int) request.getBlobSize());
        CompositeBlobInfo compositeBlobInfo = MetadataContentSerDe.deserializeMetadataContentRecord(ByteBuffer.wrap(data), new BlobIdFactory(mockClusterMap));
        List<StoreKey> dataBlobIds = compositeBlobInfo.getKeys();
        long expectedMaxChunkSize;
        long expectedTotalSize;
        int expectedNumChunks;
        if (stitchOperation) {
            expectedMaxChunkSize = requestAndResult.chunksToStitch.stream().mapToLong(ChunkInfo::getChunkSizeInBytes).max().orElse(0);
            expectedTotalSize = requestAndResult.chunksToStitch.stream().mapToLong(ChunkInfo::getChunkSizeInBytes).sum();
            expectedNumChunks = requestAndResult.chunksToStitch.size();
        } else {
            expectedMaxChunkSize = chunkSize;
            expectedTotalSize = requestAndResult.putContent.length;
            expectedNumChunks = RouterUtils.getNumChunksForBlobAndChunkSize(requestAndResult.putContent.length, chunkSize);
        }
        if (metadataContentVersion <= MessageFormatRecord.Metadata_Content_Version_V2) {
            assertEquals("Wrong max chunk size in metadata", expectedMaxChunkSize, compositeBlobInfo.getChunkSize());
        }
        assertEquals("Wrong total size in metadata", expectedTotalSize, compositeBlobInfo.getTotalSize());
        assertEquals("Number of chunks is not as expected", expectedNumChunks, dataBlobIds.size());
        // Verify all dataBlobIds are DataChunk
        for (StoreKey key : dataBlobIds) {
            BlobId origDataBlobId = (BlobId) key;
            assertEquals("Expected datachunk", BlobDataType.DATACHUNK, origDataBlobId.getBlobDataType());
        }
        // verify user-metadata
        if (requestAndResult.putBlobProperties.isEncrypted()) {
            ByteBuffer userMetadata = request.getUsermetadata();
            // reason to directly call run() instead of spinning up a thread instead of calling start() is that, any exceptions or
            // assertion failures in non main thread will not fail the test.
            new DecryptJob(origBlobId, request.getBlobEncryptionKey().duplicate(), null, userMetadata, cryptoService, kms, null, new CryptoJobMetricsTracker(metrics.decryptJobMetrics), (result, exception) -> {
                assertNull("Exception should not be thrown", exception);
                assertEquals("BlobId mismatch", origBlobId, result.getBlobId());
                assertArrayEquals("UserMetadata mismatch", requestAndResult.putUserMetadata, result.getDecryptedUserMetadata().array());
            }).run();
        } else {
            assertArrayEquals("UserMetadata mismatch", requestAndResult.putUserMetadata, request.getUsermetadata().array());
        }
        if (!stitchOperation) {
            verifyCompositeBlob(requestAndResult.putBlobProperties, requestAndResult.putContent, requestAndResult.putUserMetadata, dataBlobIds, request, serializedRequests);
        }
    } else {
        notificationBlobType = requestAndResult.options.isChunkUpload() ? NotificationBlobType.DataChunk : NotificationBlobType.Simple;
        // TODO: Currently, we don't have the logic to distinguish Simple vs DataChunk for the first chunk
        // Once the logic is fixed we should assert Simple.
        BlobDataType dataType = origBlobId.getBlobDataType();
        assertTrue("Invalid blob data type", dataType == BlobDataType.DATACHUNK || dataType == BlobDataType.SIMPLE);
        byte[] content = Utils.readBytesFromStream(request.getBlobStream(), (int) request.getBlobSize());
        if (!requestAndResult.putBlobProperties.isEncrypted()) {
            assertArrayEquals("Input blob and written blob should be the same", requestAndResult.putContent, content);
            assertArrayEquals("UserMetadata mismatch for simple blob", requestAndResult.putUserMetadata, request.getUsermetadata().array());
        } else {
            ByteBuffer userMetadata = request.getUsermetadata();
            // reason to directly call run() instead of spinning up a thread instead of calling start() is that, any exceptions or
            // assertion failures in non main thread will not fail the test.
            new DecryptJob(origBlobId, request.getBlobEncryptionKey().duplicate(), Unpooled.wrappedBuffer(content), userMetadata, cryptoService, kms, null, new CryptoJobMetricsTracker(metrics.decryptJobMetrics), (result, exception) -> {
                assertNull("Exception should not be thrown", exception);
                assertEquals("BlobId mismatch", origBlobId, result.getBlobId());
                ByteBuf decryptedBlobContent = result.getDecryptedBlobContent();
                byte[] blobContent = new byte[decryptedBlobContent.readableBytes()];
                decryptedBlobContent.readBytes(blobContent);
                assertArrayEquals("Content mismatch", requestAndResult.putContent, blobContent);
                assertArrayEquals("UserMetadata mismatch", requestAndResult.putUserMetadata, result.getDecryptedUserMetadata().array());
                decryptedBlobContent.release();
            }).run();
        }
    }
    notificationSystem.verifyNotification(blobId, notificationBlobType, request.getBlobProperties());
}
Also used : Arrays(java.util.Arrays) BlobProperties(com.github.ambry.messageformat.BlobProperties) DataNodeId(com.github.ambry.clustermap.DataNodeId) ByteBufferReadableStreamChannel(com.github.ambry.commons.ByteBufferReadableStreamChannel) Random(java.util.Random) ByteBuffer(java.nio.ByteBuffer) ThrowingConsumer(com.github.ambry.utils.ThrowingConsumer) Unpooled(io.netty.buffer.Unpooled) Future(java.util.concurrent.Future) GeneralSecurityException(java.security.GeneralSecurityException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestUtils(com.github.ambry.utils.TestUtils) Map(java.util.Map) After(org.junit.After) NettyByteBufLeakHelper(com.github.ambry.utils.NettyByteBufLeakHelper) QuotaMethod(com.github.ambry.quota.QuotaMethod) Parameterized(org.junit.runners.Parameterized) Container(com.github.ambry.account.Container) Set(java.util.Set) Utils(com.github.ambry.utils.Utils) CryptoServiceConfig(com.github.ambry.config.CryptoServiceConfig) RouterConfig(com.github.ambry.config.RouterConfig) CountDownLatch(java.util.concurrent.CountDownLatch) StoreKey(com.github.ambry.store.StoreKey) List(java.util.List) MockTime(com.github.ambry.utils.MockTime) NotificationBlobType(com.github.ambry.notification.NotificationBlobType) Account(com.github.ambry.account.Account) Callback(com.github.ambry.commons.Callback) LoggingNotificationSystem(com.github.ambry.commons.LoggingNotificationSystem) BlobType(com.github.ambry.messageformat.BlobType) InMemAccountService(com.github.ambry.account.InMemAccountService) BlobId(com.github.ambry.commons.BlobId) ResponseHandler(com.github.ambry.commons.ResponseHandler) CompositeBlobInfo(com.github.ambry.messageformat.CompositeBlobInfo) DataInputStream(java.io.DataInputStream) ServerErrorCode(com.github.ambry.server.ServerErrorCode) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ByteBuf(io.netty.buffer.ByteBuf) QuotaResource(com.github.ambry.quota.QuotaResource) SystemTime(com.github.ambry.utils.SystemTime) PutRequest(com.github.ambry.protocol.PutRequest) Before(org.junit.Before) QuotaResourceType(com.github.ambry.quota.QuotaResourceType) BlobDataType(com.github.ambry.commons.BlobId.BlobDataType) Properties(java.util.Properties) LongStream(java.util.stream.LongStream) Pair(com.github.ambry.utils.Pair) Iterator(java.util.Iterator) VerifiableProperties(com.github.ambry.config.VerifiableProperties) QuotaChargeCallback(com.github.ambry.quota.QuotaChargeCallback) MetadataContentSerDe(com.github.ambry.messageformat.MetadataContentSerDe) IOException(java.io.IOException) Test(org.junit.Test) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) TimeUnit(java.util.concurrent.TimeUnit) QuotaException(com.github.ambry.quota.QuotaException) KMSConfig(com.github.ambry.config.KMSConfig) ByteBufferInputStream(com.github.ambry.utils.ByteBufferInputStream) MessageFormatRecord(com.github.ambry.messageformat.MessageFormatRecord) Assert(org.junit.Assert) Collections(java.util.Collections) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) InputStream(java.io.InputStream) CompositeBlobInfo(com.github.ambry.messageformat.CompositeBlobInfo) PutRequest(com.github.ambry.protocol.PutRequest) ByteBuf(io.netty.buffer.ByteBuf) ByteBuffer(java.nio.ByteBuffer) NotificationBlobType(com.github.ambry.notification.NotificationBlobType) StoreKey(com.github.ambry.store.StoreKey) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) BlobDataType(com.github.ambry.commons.BlobId.BlobDataType) BlobId(com.github.ambry.commons.BlobId)

Aggregations

Account (com.github.ambry.account.Account)1 Container (com.github.ambry.account.Container)1 InMemAccountService (com.github.ambry.account.InMemAccountService)1 DataNodeId (com.github.ambry.clustermap.DataNodeId)1 MockClusterMap (com.github.ambry.clustermap.MockClusterMap)1 BlobId (com.github.ambry.commons.BlobId)1 BlobDataType (com.github.ambry.commons.BlobId.BlobDataType)1 BlobIdFactory (com.github.ambry.commons.BlobIdFactory)1 ByteBufferReadableStreamChannel (com.github.ambry.commons.ByteBufferReadableStreamChannel)1 Callback (com.github.ambry.commons.Callback)1 LoggingNotificationSystem (com.github.ambry.commons.LoggingNotificationSystem)1 ResponseHandler (com.github.ambry.commons.ResponseHandler)1 CryptoServiceConfig (com.github.ambry.config.CryptoServiceConfig)1 KMSConfig (com.github.ambry.config.KMSConfig)1 RouterConfig (com.github.ambry.config.RouterConfig)1 VerifiableProperties (com.github.ambry.config.VerifiableProperties)1 BlobProperties (com.github.ambry.messageformat.BlobProperties)1 BlobType (com.github.ambry.messageformat.BlobType)1 CompositeBlobInfo (com.github.ambry.messageformat.CompositeBlobInfo)1 MessageFormatRecord (com.github.ambry.messageformat.MessageFormatRecord)1