Examples with BlobProperties com.github.ambry.messageformat.BlobProperties used on opensource projects

Example 1 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class ReplicationTest method getPutMessage.

/**
 * Constructs an entire message with header, blob properties, user metadata and blob content.
 * @param id id for which the message has to be constructed.
 * @param accountId accountId of the blob
 * @param containerId containerId of the blob
 * @param enableEncryption {@code true} if encryption needs to be enabled. {@code false} otherwise
 * @return a {@link Pair} of {@link ByteBuffer} and {@link MessageInfo} representing the entire message and the
 *         associated {@link MessageInfo}
 * @throws MessageFormatException
 * @throws IOException
 */
private Pair<ByteBuffer, MessageInfo> getPutMessage(StoreKey id, short accountId, short containerId, boolean enableEncryption) throws MessageFormatException, IOException {
    int blobSize = TestUtils.RANDOM.nextInt(500) + 501;
    int userMetadataSize = TestUtils.RANDOM.nextInt(blobSize / 2);
    int encryptionKeySize = TestUtils.RANDOM.nextInt(blobSize / 4);
    byte[] blob = new byte[blobSize];
    byte[] usermetadata = new byte[userMetadataSize];
    byte[] encryptionKey = enableEncryption ? new byte[encryptionKeySize] : null;
    TestUtils.RANDOM.nextBytes(blob);
    TestUtils.RANDOM.nextBytes(usermetadata);
    BlobProperties blobProperties = new BlobProperties(blobSize, "test", accountId, containerId, encryptionKey != null);
    MessageFormatInputStream stream = new PutMessageFormatInputStream(id, encryptionKey == null ? null : ByteBuffer.wrap(encryptionKey), blobProperties, ByteBuffer.wrap(usermetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blobSize);
    byte[] message = Utils.readBytesFromStream(stream, (int) stream.getSize());
    return new Pair<>(ByteBuffer.wrap(message), new MessageInfo(id, message.length, Utils.Infinite_Time, accountId, containerId, blobProperties.getCreationTimeInMs()));
}

Example 2 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class MockReadableStreamChannel method verifyBlob.

/**
 * Verifies that the blob associated with the blob id returned by a successful put operation has exactly the same
 * data as the original object that was put.
 * @param blobId the blobId of the blob that is to be verified.
 * @param properties the {@link BlobProperties} of the blob that is to be verified
 * @param originalPutContent original content of the blob
 * @param originalUserMetadata original user-metadata of the blob
 * @param serializedRequests the mapping from blob ids to their corresponding serialized {@link PutRequest}.
 */
private void verifyBlob(String blobId, BlobProperties properties, byte[] originalPutContent, byte[] originalUserMetadata, HashMap<String, ByteBuffer> serializedRequests) throws Exception {
    ByteBuffer serializedRequest = serializedRequests.get(blobId);
    PutRequest.ReceivedPutRequest request = deserializePutRequest(serializedRequest);
    NotificationBlobType notificationBlobType;
    if (request.getBlobType() == BlobType.MetadataBlob) {
        notificationBlobType = NotificationBlobType.Composite;
        byte[] data = Utils.readBytesFromStream(request.getBlobStream(), (int) request.getBlobSize());
        CompositeBlobInfo compositeBlobInfo = MetadataContentSerDe.deserializeMetadataContentRecord(ByteBuffer.wrap(data), new BlobIdFactory(mockClusterMap));
        Assert.assertEquals("Wrong max chunk size in metadata", chunkSize, compositeBlobInfo.getChunkSize());
        Assert.assertEquals("Wrong total size in metadata", originalPutContent.length, compositeBlobInfo.getTotalSize());
        List<StoreKey> dataBlobIds = compositeBlobInfo.getKeys();
        Assert.assertEquals("Number of chunks is not as expected", RouterUtils.getNumChunksForBlobAndChunkSize(originalPutContent.length, chunkSize), dataBlobIds.size());
        // verify user-metadata
        if (properties.isEncrypted()) {
            ByteBuffer userMetadata = request.getUsermetadata();
            BlobId origBlobId = new BlobId(blobId, mockClusterMap);
            // reason to directly call run() instead of spinning up a thread instead of calling start() is that, any exceptions or
            // assertion failures in non main thread will not fail the test.
            new DecryptJob(origBlobId, request.getBlobEncryptionKey().duplicate(), null, userMetadata, cryptoService, kms, new CryptoJobMetricsTracker(metrics.decryptJobMetrics), (result, exception) -> {
                Assert.assertNull("Exception should not be thrown", exception);
                Assert.assertEquals("BlobId mismatch", origBlobId, result.getBlobId());
                Assert.assertArrayEquals("UserMetadata mismatch", originalUserMetadata, result.getDecryptedUserMetadata().array());
            }).run();
        } else {
            Assert.assertArrayEquals("UserMetadata mismatch", originalUserMetadata, request.getUsermetadata().array());
        }
        verifyCompositeBlob(properties, originalPutContent, originalUserMetadata, dataBlobIds, request, serializedRequests);
    } else {
        notificationBlobType = NotificationBlobType.Simple;
        byte[] content = Utils.readBytesFromStream(request.getBlobStream(), (int) request.getBlobSize());
        if (!properties.isEncrypted()) {
            Assert.assertArrayEquals("Input blob and written blob should be the same", originalPutContent, content);
            Assert.assertArrayEquals("UserMetadata mismatch for simple blob", originalUserMetadata, request.getUsermetadata().array());
            notificationSystem.verifyNotification(blobId, notificationBlobType, request.getBlobProperties());
        } else {
            ByteBuffer userMetadata = request.getUsermetadata();
            BlobId origBlobId = new BlobId(blobId, mockClusterMap);
            // reason to directly call run() instead of spinning up a thread instead of calling start() is that, any exceptions or
            // assertion failures in non main thread will not fail the test.
            new DecryptJob(origBlobId, request.getBlobEncryptionKey().duplicate(), ByteBuffer.wrap(content), userMetadata, cryptoService, kms, new CryptoJobMetricsTracker(metrics.decryptJobMetrics), new Callback<DecryptJob.DecryptJobResult>() {

                @Override
                public void onCompletion(DecryptJob.DecryptJobResult result, Exception exception) {
                    Assert.assertNull("Exception should not be thrown", exception);
                    Assert.assertEquals("BlobId mismatch", origBlobId, result.getBlobId());
                    Assert.assertArrayEquals("Content mismatch", originalPutContent, result.getDecryptedBlobContent().array());
                    Assert.assertArrayEquals("UserMetadata mismatch", originalUserMetadata, result.getDecryptedUserMetadata().array());
                }
            }).run();
        }
    }
    notificationSystem.verifyNotification(blobId, notificationBlobType, request.getBlobProperties());
}

Example 3 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class AmbryRequestsTest method sendAndVerifyOperationRequest.

/**
 * Sends and verifies that an operation specific request works correctly.
 * @param requestType the type of the request to send.
 * @param ids the partitionIds to send requests for.
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
 *                          response in the constituents rather than the actual response ({@link GetResponse} and
 *                          {@link ReplicaMetadataResponse}).
 * @throws InterruptedException
 * @throws IOException
 */
private void sendAndVerifyOperationRequest(RequestOrResponseType requestType, List<? extends PartitionId> ids, ServerErrorCode expectedErrorCode) throws InterruptedException, IOException {
    for (PartitionId id : ids) {
        int correlationId = TestUtils.RANDOM.nextInt();
        String clientId = UtilsTest.getRandomString(10);
        BlobId blobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMapUtils.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), id, false);
        RequestOrResponse request;
        switch(requestType) {
            case PutRequest:
                BlobProperties properties = new BlobProperties(0, "serviceId", blobId.getAccountId(), blobId.getAccountId(), false);
                request = new PutRequest(correlationId, clientId, blobId, properties, ByteBuffer.allocate(0), ByteBuffer.allocate(0), 0, BlobType.DataBlob, null);
                break;
            case DeleteRequest:
                request = new DeleteRequest(correlationId, clientId, blobId, SystemTime.getInstance().milliseconds());
                break;
            case GetRequest:
                PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(id, Collections.singletonList(blobId));
                request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
                break;
            case ReplicaMetadataRequest:
                ReplicaMetadataRequestInfo rRequestInfo = new ReplicaMetadataRequestInfo(id, FIND_TOKEN_FACTORY.getNewFindToken(), "localhost", "/tmp");
                request = new ReplicaMetadataRequest(correlationId, clientId, Collections.singletonList(rRequestInfo), Long.MAX_VALUE);
                break;
            default:
                throw new IllegalArgumentException(requestType + " not supported by this function");
        }
        storageManager.resetStore();
        Response response = sendRequestGetResponse(request, requestType == RequestOrResponseType.GetRequest || requestType == RequestOrResponseType.ReplicaMetadataRequest ? ServerErrorCode.No_Error : expectedErrorCode);
        if (expectedErrorCode.equals(ServerErrorCode.No_Error)) {
            assertEquals("Operation received at the store not as expected", requestType, MockStorageManager.operationReceived);
        }
        if (requestType == RequestOrResponseType.GetRequest) {
            GetResponse getResponse = (GetResponse) response;
            for (PartitionResponseInfo info : getResponse.getPartitionResponseInfoList()) {
                assertEquals("Error code does not match expected", expectedErrorCode, info.getErrorCode());
            }
        } else if (requestType == RequestOrResponseType.ReplicaMetadataRequest) {
            ReplicaMetadataResponse replicaMetadataResponse = (ReplicaMetadataResponse) response;
            for (ReplicaMetadataResponseInfo info : replicaMetadataResponse.getReplicaMetadataResponseInfoList()) {
                assertEquals("Error code does not match expected", expectedErrorCode, info.getError());
            }
        }
    }
}

Example 4 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class InMemoryRouter method updateBlobTtl.

@Override
public Future<Void> updateBlobTtl(String blobId, String serviceId, long expiresAtMs, Callback<Void> callback, QuotaChargeCallback quotaChargeCallback) {
    FutureResult<Void> futureResult = new FutureResult<>();
    if (!handlePrechecks(futureResult, callback)) {
        return futureResult;
    }
    Exception exception = null;
    try {
        // to make sure Blob ID is ok
        checkBlobId(blobId);
        if (!deletedBlobs.contains(blobId) && blobs.containsKey(blobId)) {
            InMemoryBlob blob = blobs.get(blobId);
            BlobProperties currentProps = blob.blobProperties;
            long newTtlSecs = Utils.getTtlInSecsFromExpiryMs(expiresAtMs, currentProps.getCreationTimeInMs());
            blob.blobProperties.setTimeToLiveInSeconds(newTtlSecs);
            if (notificationSystem != null) {
                notificationSystem.onBlobTtlUpdated(blobId, serviceId, expiresAtMs, null, null);
            }
        } else if (deletedBlobs.contains(blobId)) {
            exception = new RouterException("Blob has been deleted", RouterErrorCode.BlobDeleted);
        } else {
            exception = new RouterException("Blob not found", RouterErrorCode.BlobDoesNotExist);
        }
    } catch (RouterException e) {
        exception = e;
    } catch (Exception e) {
        exception = new RouterException(e, RouterErrorCode.UnexpectedInternalError);
    } finally {
        completeOperation(futureResult, callback, null, exception);
    }
    return futureResult;
}

Example 5 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class HardDeleteVerifier method deserializeBlobProperties.

boolean deserializeBlobProperties(InputStream streamlog, InputStream oldStreamlog, boolean isDeleted) throws ContinueException {
    boolean caughtException = false;
    boolean caughtExceptionInOld = false;
    BlobProperties props = null;
    BlobProperties oldProps = null;
    try {
        props = MessageFormatRecord.deserializeBlobProperties(streamlog);
    } catch (MessageFormatException e) {
        caughtException = true;
    } catch (IOException e) {
        caughtException = true;
    }
    try {
        oldProps = MessageFormatRecord.deserializeBlobProperties(oldStreamlog);
    } catch (MessageFormatException e) {
        caughtExceptionInOld = true;
    } catch (IOException e) {
        caughtExceptionInOld = true;
    }
    if (!caughtException) {
        if (props.toString().compareTo(oldProps.toString()) != 0) {
            System.out.println("Blob id mismatch!");
            return false;
        }
    } else if (!caughtExceptionInOld) {
        if (isDeleted) {
            corruptDeleted++;
        } else {
            corruptNonDeleted++;
        }
        throw new ContinueException("blob properties could not be deserialized.");
    } else {
        throw new ContinueException("blob properties could not be deserialized in either");
    }
    return true;
}