use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.
the class RestUtils method buildBlobProperties.
/**
* Builds {@link BlobProperties} given the arguments associated with a request.
* @param args the arguments associated with the request. Cannot be {@code null}.
* @return the {@link BlobProperties} extracted from the arguments.
* @throws RestServiceException if required arguments aren't present or if they aren't in the format or number
* expected.
*/
public static BlobProperties buildBlobProperties(Map<String, Object> args) throws RestServiceException {
Account account = getAccountFromArgs(args);
Container container = getContainerFromArgs(args);
String serviceId = getHeader(args, Headers.SERVICE_ID, true);
String contentType = getHeader(args, Headers.AMBRY_CONTENT_TYPE, true);
String ownerId = getHeader(args, Headers.OWNER_ID, false);
String externalAssetTag = getHeader(args, Headers.EXTERNAL_ASSET_TAG, false);
String contentEncoding = getHeader(args, Headers.AMBRY_CONTENT_ENCODING, false);
String filename = getHeader(args, Headers.AMBRY_FILENAME, false);
long ttl = getTtlFromRequestHeader(args);
// This field should not matter on newly created blobs, because all privacy/cacheability decisions should be made
// based on the container properties and ACLs. For now, BlobProperties still includes this field, though.
boolean isPrivate = !container.isCacheable();
return new BlobProperties(-1, serviceId, ownerId, contentType, isPrivate, ttl, account.getId(), container.getId(), container.isEncrypted(), externalAssetTag, contentEncoding, filename);
}
use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.
the class AmbrySecurityServiceTest method verifyHeadersForGetBlob.
/**
* Verify the headers from the response are as expected
* @param restRequest the original request received.
* @param blobInfo the {@link BlobInfo} to refer to while getting headers.
* @param accountAndContainer the {@link Account} and {@link Container} of the blob being requested.
* @param range the {@link ByteRange} used for a range request, or {@code null} for non-ranged requests.
* @param restResponseChannel {@link MockRestResponseChannel} from which headers are to be verified
* @throws RestServiceException if there was any problem getting the headers.
*/
private void verifyHeadersForGetBlob(RestRequest restRequest, BlobInfo blobInfo, Pair<Account, Container> accountAndContainer, ByteRange range, MockRestResponseChannel restResponseChannel) throws RestServiceException {
BlobProperties blobProperties = blobInfo.getBlobProperties();
Assert.assertEquals("Blob size mismatch ", blobProperties.getBlobSize(), Long.parseLong(restResponseChannel.getHeader(RestUtils.Headers.BLOB_SIZE)));
if (blobProperties.getContentType() != null) {
Assert.assertEquals("Content Type mismatch", blobProperties.getContentType(), restResponseChannel.getHeader(RestUtils.Headers.CONTENT_TYPE));
if (blobProperties.getContentType().equals("text/html")) {
Assert.assertEquals("Content disposition not set for text/html Content type", "attachment", restResponseChannel.getHeader("Content-Disposition"));
} else {
Assert.assertNull("Content disposition should not have been set", restResponseChannel.getHeader("Content-Disposition"));
}
}
Assert.assertEquals("Accept ranges header not set correctly", "bytes", restResponseChannel.getHeader(RestUtils.Headers.ACCEPT_RANGES));
long contentLength = blobProperties.getBlobSize();
if (range != null) {
Pair<String, Long> rangeAndLength = RestUtils.buildContentRangeAndLength(range, contentLength, false);
Assert.assertEquals("Content range header not set correctly for range " + range, rangeAndLength.getFirst(), restResponseChannel.getHeader(RestUtils.Headers.CONTENT_RANGE));
contentLength = rangeAndLength.getSecond();
} else {
Assert.assertNull("Content range header should not be set", restResponseChannel.getHeader(RestUtils.Headers.CONTENT_RANGE));
}
if (contentLength < FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes) {
Assert.assertEquals("Content length value mismatch", contentLength, Integer.parseInt(restResponseChannel.getHeader(RestUtils.Headers.CONTENT_LENGTH)));
} else {
Assert.assertNull("Content length value should not be set", restResponseChannel.getHeader(RestUtils.Headers.CONTENT_LENGTH));
}
verifyBlobPropertiesHeaders(blobInfo.getBlobProperties(), restResponseChannel);
verifyAccountAndContainerHeaders(restResponseChannel, accountAndContainer.getFirst(), accountAndContainer.getSecond());
Assert.assertEquals("LifeVersion mismatch", Short.toString(blobInfo.getLifeVersion()), restResponseChannel.getHeader(RestUtils.Headers.LIFE_VERSION));
Map<String, String> userMetadata = blobInfo.getUserMetadata() != null ? RestUtils.buildUserMetadata(blobInfo.getUserMetadata()) : null;
if (blobInfo.getUserMetadata().length == 0 || userMetadata == null) {
Assert.assertNull("Internal key " + RestUtils.InternalKeys.SEND_USER_METADATA_AS_RESPONSE_BODY + " should not be set", restRequest.getArgs().get(RestUtils.InternalKeys.SEND_USER_METADATA_AS_RESPONSE_BODY));
} else {
USER_METADATA.forEach((key, value) -> Assert.assertEquals("Value of " + key + " not as expected", value, restResponseChannel.getHeader(key)));
}
verifyCacheHeaders(getAccountAndContainer(blobProperties).getSecond().isCacheable(), restResponseChannel);
}
use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.
the class AmbrySecurityServiceTest method processResponseTest.
/**
* Tests {@link AmbrySecurityService#processResponse(RestRequest, RestResponseChannel, BlobInfo, Callback)} for
* common as well as uncommon cases
* @throws Exception
*/
@Test
public void processResponseTest() throws Exception {
RestRequest restRequest = createRestRequest(RestMethod.GET, "/", null);
// rest request being null
TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(null, new MockRestResponseChannel(), DEFAULT_INFO).get(), null);
// restResponseChannel being null
TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(restRequest, null, DEFAULT_INFO).get(), null);
// blob info being null
TestUtils.assertException(IllegalArgumentException.class, () -> securityService.processResponse(restRequest, new MockRestResponseChannel(), null).get(), null);
// for unsupported methods
RestMethod[] methods = { RestMethod.DELETE };
for (RestMethod restMethod : methods) {
testExceptionCasesProcessResponse(restMethod, new MockRestResponseChannel(), DEFAULT_INFO, RestServiceErrorCode.InternalServerError);
}
// OPTIONS (should be no errors)
securityService.processResponse(createRestRequest(RestMethod.OPTIONS, "/", null), new MockRestResponseChannel(), null).get();
// PUT (should be no errors)
securityService.processResponse(createRestRequest(RestMethod.PUT, "/", null), new MockRestResponseChannel(), null).get();
// GET signed URL (should be no errors)
securityService.processResponse(createRestRequest(RestMethod.GET, Operations.GET_SIGNED_URL, null), new MockRestResponseChannel(), null).get();
// HEAD
// normal
testHeadBlobWithVariousRanges(DEFAULT_INFO);
// with lifeVersion
testHeadBlobWithVariousRanges(LIFEVERSION_INFO);
// unknown account
testHeadBlobWithVariousRanges(UNKNOWN_INFO);
// encrypted unknown account
testHeadBlobWithVariousRanges(UNKNOWN_INFO_ENC);
// with no owner id
BlobInfo blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, null, "image/gif", false, Utils.Infinite_Time, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
testHeadBlobWithVariousRanges(blobInfo);
// with no content type
blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, null, false, Utils.Infinite_Time, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
testHeadBlobWithVariousRanges(blobInfo);
// with a TTL
blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, REF_ACCOUNT.getId(), REF_CONTAINER.getId(), false, null, null, null), new byte[0]);
testHeadBlobWithVariousRanges(blobInfo);
// GET BlobInfo
testGetSubResource(DEFAULT_INFO, RestUtils.SubResource.BlobInfo);
testGetSubResource(LIFEVERSION_INFO, RestUtils.SubResource.BlobInfo);
testGetSubResource(UNKNOWN_INFO, RestUtils.SubResource.BlobInfo);
testGetSubResource(UNKNOWN_INFO, RestUtils.SubResource.BlobInfo);
testGetSubResource(UNKNOWN_INFO_ENC, RestUtils.SubResource.BlobInfo);
// GET UserMetadata
testGetSubResource(DEFAULT_INFO, RestUtils.SubResource.UserMetadata);
byte[] usermetadata = TestUtils.getRandomBytes(10);
testGetSubResource(new BlobInfo(DEFAULT_INFO.getBlobProperties(), usermetadata), RestUtils.SubResource.UserMetadata);
// POST
testPostBlob();
// GET Blob
testGetBlobWithVariousRanges(LIFEVERSION_INFO);
// less than chunk threshold size
blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes - 1, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// == chunk threshold size
blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// more than chunk threshold size
blobInfo = new BlobInfo(new BlobProperties(FRONTEND_CONFIG.chunkedGetResponseThresholdInBytes * 2, SERVICE_ID, OWNER_ID, "image/gif", false, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// Get blob with content type null
blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, null, true, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// Get blob in a non-cacheable container. AmbrySecurityService should not care about the isPrivate setting.
blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", false, Utils.Infinite_Time, Account.UNKNOWN_ACCOUNT_ID, Container.DEFAULT_PRIVATE_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// Get blob in a cacheable container. AmbrySecurityService should not care about the isPrivate setting.
blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "image/gif", true, Utils.Infinite_Time, Account.UNKNOWN_ACCOUNT_ID, Container.DEFAULT_PUBLIC_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// not modified response
// > creation time (in secs).
testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs() + 1000);
// == creation time
testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs());
// < creation time (in secs)
testGetNotModifiedBlob(blobInfo, blobInfo.getBlobProperties().getCreationTimeInMs() - 1000);
// Get blob for a public blob with content type as "text/html"
blobInfo = new BlobInfo(new BlobProperties(100, SERVICE_ID, OWNER_ID, "text/html", true, 10000, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID, false, null, null, null), new byte[0]);
testGetBlobWithVariousRanges(blobInfo);
// not modified response
// > creation time (in secs).
testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs() + 1000);
// == creation time
testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs());
// < creation time (in secs)
testGetNotModifiedBlob(DEFAULT_INFO, DEFAULT_INFO.getBlobProperties().getCreationTimeInMs() - 1000);
// bad rest response channel
testExceptionCasesProcessResponse(RestMethod.HEAD, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
testExceptionCasesProcessResponse(RestMethod.GET, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
testExceptionCasesProcessResponse(RestMethod.POST, new BadRestResponseChannel(), blobInfo, RestServiceErrorCode.InternalServerError);
// security service closed
securityService.close();
methods = new RestMethod[] { RestMethod.POST, RestMethod.GET, RestMethod.DELETE, RestMethod.HEAD };
for (RestMethod restMethod : methods) {
testExceptionCasesProcessResponse(restMethod, new MockRestResponseChannel(), blobInfo, RestServiceErrorCode.ServiceUnavailable);
}
}
use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.
the class BlobIdTransformer method newMessage.
/**
* Creates a Message from the old Message
* input stream, replacing the old store key and account/container IDs
* with a new store key and account/container IDs
* @param inputStream the input stream of the Message
* @param newKey the new StoreKey
* @param oldMessageInfo the {@link MessageInfo} of the message being transformed
* @return new Message message
* @throws Exception
*/
private Message newMessage(InputStream inputStream, StoreKey newKey, MessageInfo oldMessageInfo) throws Exception {
MessageHeader_Format headerFormat = getMessageHeader(inputStream);
storeKeyFactory.getStoreKey(new DataInputStream(inputStream));
BlobId newBlobId = (BlobId) newKey;
if (headerFormat.isPutRecord()) {
if (headerFormat.hasLifeVersion() && headerFormat.getLifeVersion() != oldMessageInfo.getLifeVersion()) {
// The original Put buffer might have lifeVersion as 0, but the message info might have a higher lifeVersion.
logger.trace("LifeVersion in stream: {} failed to match lifeVersion from Index: {} for key {}", headerFormat.getLifeVersion(), oldMessageInfo.getLifeVersion(), oldMessageInfo.getStoreKey());
}
ByteBuffer blobEncryptionKey = null;
if (headerFormat.hasEncryptionKeyRecord()) {
blobEncryptionKey = deserializeBlobEncryptionKey(inputStream);
}
BlobProperties oldProperties = deserializeBlobProperties(inputStream);
ByteBuffer userMetaData = deserializeUserMetadata(inputStream);
BlobData blobData = deserializeBlob(inputStream);
ByteBuf blobDataBytes = blobData.content();
long blobPropertiesSize = oldProperties.getBlobSize();
// will be rewritten with transformed IDs
if (blobData.getBlobType().equals(BlobType.MetadataBlob)) {
ByteBuffer serializedMetadataContent = blobDataBytes.nioBuffer();
CompositeBlobInfo compositeBlobInfo = MetadataContentSerDe.deserializeMetadataContentRecord(serializedMetadataContent, storeKeyFactory);
Map<StoreKey, StoreKey> convertedKeys = storeKeyConverter.convert(compositeBlobInfo.getKeys());
List<StoreKey> newKeys = new ArrayList<>();
boolean isOldMetadataKeyDifferentFromNew = !oldMessageInfo.getStoreKey().getID().equals(newKey.getID());
short metadataAccountId = newBlobId.getAccountId();
short metadataContainerId = newBlobId.getContainerId();
for (StoreKey oldDataChunkKey : compositeBlobInfo.getKeys()) {
StoreKey newDataChunkKey = convertedKeys.get(oldDataChunkKey);
if (newDataChunkKey == null) {
throw new IllegalStateException("Found metadata chunk with a deprecated data chunk. " + " Old MetadataID: " + oldMessageInfo.getStoreKey().getID() + " New MetadataID: " + newKey.getID() + " Old Datachunk ID: " + oldDataChunkKey.getID());
}
if (isOldMetadataKeyDifferentFromNew && newDataChunkKey.getID().equals(oldDataChunkKey.getID())) {
throw new IllegalStateException("Found changed metadata chunk with an unchanged data chunk" + " Old MetadataID: " + oldMessageInfo.getStoreKey().getID() + " New MetadataID: " + newKey.getID() + " Old Datachunk ID: " + oldDataChunkKey.getID());
}
if (!isOldMetadataKeyDifferentFromNew && !newDataChunkKey.getID().equals(oldDataChunkKey.getID())) {
throw new IllegalStateException("Found unchanged metadata chunk with a changed data chunk" + " Old MetadataID: " + oldMessageInfo.getStoreKey().getID() + " New MetadataID: " + newKey.getID() + " Old Datachunk ID: " + oldDataChunkKey.getID() + " New Datachunk ID: " + newDataChunkKey.getID());
}
BlobId newDataChunkBlobId = (BlobId) newDataChunkKey;
if (newDataChunkBlobId.getAccountId() != metadataAccountId || newDataChunkBlobId.getContainerId() != metadataContainerId) {
throw new IllegalStateException("Found changed metadata chunk with a datachunk with a different account/container" + " Old MetadataID: " + oldMessageInfo.getStoreKey().getID() + " New MetadataID: " + newKey.getID() + " Old Datachunk ID: " + oldDataChunkKey.getID() + " New Datachunk ID: " + newDataChunkBlobId.getID() + " Metadata AccountId: " + metadataAccountId + " Metadata ContainerId: " + metadataContainerId + " Datachunk AccountId: " + newDataChunkBlobId.getAccountId() + " Datachunk ContainerId: " + newDataChunkBlobId.getContainerId());
}
newKeys.add(newDataChunkKey);
}
ByteBuffer metadataContent;
if (compositeBlobInfo.getMetadataContentVersion() == Metadata_Content_Version_V2) {
metadataContent = MetadataContentSerDe.serializeMetadataContentV2(compositeBlobInfo.getChunkSize(), compositeBlobInfo.getTotalSize(), newKeys);
} else if (compositeBlobInfo.getMetadataContentVersion() == Metadata_Content_Version_V3) {
List<Pair<StoreKey, Long>> keyAndSizeList = new ArrayList<>();
List<CompositeBlobInfo.ChunkMetadata> chunkMetadataList = compositeBlobInfo.getChunkMetadataList();
for (int i = 0; i < newKeys.size(); i++) {
keyAndSizeList.add(new Pair<>(newKeys.get(i), chunkMetadataList.get(i).getSize()));
}
metadataContent = MetadataContentSerDe.serializeMetadataContentV3(compositeBlobInfo.getTotalSize(), keyAndSizeList);
} else {
throw new IllegalStateException("Unexpected metadata content version from composite blob: " + compositeBlobInfo.getMetadataContentVersion());
}
blobPropertiesSize = compositeBlobInfo.getTotalSize();
metadataContent.flip();
blobDataBytes.release();
blobDataBytes = Unpooled.wrappedBuffer(metadataContent);
blobData = new BlobData(blobData.getBlobType(), metadataContent.remaining(), blobDataBytes);
}
BlobProperties newProperties = new BlobProperties(blobPropertiesSize, oldProperties.getServiceId(), oldProperties.getOwnerId(), oldProperties.getContentType(), oldProperties.isPrivate(), oldProperties.getTimeToLiveInSeconds(), oldProperties.getCreationTimeInMs(), newBlobId.getAccountId(), newBlobId.getContainerId(), oldProperties.isEncrypted(), oldProperties.getExternalAssetTag(), oldProperties.getContentEncoding(), oldProperties.getFilename());
// BlobIDTransformer only exists on ambry-server and replication between servers is relying on blocking channel
// which is still using java ByteBuffer. So, no need to consider releasing stuff.
// @todo, when netty Bytebuf is adopted for blocking channel on ambry-server, remember to release this ByteBuf.
PutMessageFormatInputStream putMessageFormatInputStream = new PutMessageFormatInputStream(newKey, blobEncryptionKey, newProperties, userMetaData, new ByteBufInputStream(blobDataBytes, true), blobData.getSize(), blobData.getBlobType(), oldMessageInfo.getLifeVersion());
// Reuse the original CRC if present in the oldMessageInfo. This is important to ensure that messages that are
// received via replication are sent to the store with proper CRCs (which the store needs to detect duplicate
// messages). As an additional guard, here the original CRC is only reused if the key's ID in string form is the
// same after conversion.
Long originalCrc = oldMessageInfo.getStoreKey().getID().equals(newKey.getID()) ? oldMessageInfo.getCrc() : null;
MessageInfo info = new MessageInfo.Builder(newKey, putMessageFormatInputStream.getSize(), newProperties.getAccountId(), newProperties.getContainerId(), oldMessageInfo.getOperationTimeMs()).isTtlUpdated(oldMessageInfo.isTtlUpdated()).expirationTimeInMs(oldMessageInfo.getExpirationTimeInMs()).crc(originalCrc).lifeVersion(oldMessageInfo.getLifeVersion()).build();
return new Message(info, putMessageFormatInputStream);
} else {
throw new IllegalArgumentException("Only 'put' records are valid");
}
}
use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.
the class ReplicationTestHelper method createPutMessage.
/**
* Constructs an entire message with header, blob properties, user metadata and blob content.
* @param id id for which the message has to be constructed.
* @param accountId accountId of the blob
* @param containerId containerId of the blob
* @param enableEncryption {@code true} if encryption needs to be enabled. {@code false} otherwise
* @param lifeVersion lifeVersion for this hich the message has to be constructed.
* @return a {@link Pair} of {@link ByteBuffer} and {@link MessageInfo} representing the entire message and the
* associated {@link MessageInfo}
* @throws MessageFormatException
* @throws IOException
*/
public static PutMsgInfoAndBuffer createPutMessage(StoreKey id, short accountId, short containerId, boolean enableEncryption, short lifeVersion) throws MessageFormatException, IOException {
Random blobIdRandom = new Random(id.getID().hashCode());
int blobSize = blobIdRandom.nextInt(500) + 501;
int userMetadataSize = blobIdRandom.nextInt(blobSize / 2);
int encryptionKeySize = blobIdRandom.nextInt(blobSize / 4);
byte[] blob = new byte[blobSize];
byte[] usermetadata = new byte[userMetadataSize];
byte[] encryptionKey = enableEncryption ? new byte[encryptionKeySize] : null;
blobIdRandom.nextBytes(blob);
blobIdRandom.nextBytes(usermetadata);
BlobProperties blobProperties = new BlobProperties(blobSize, "test", null, null, false, EXPIRY_TIME_MS - CONSTANT_TIME_MS, CONSTANT_TIME_MS, accountId, containerId, encryptionKey != null, null, null, null);
MessageFormatInputStream stream = new PutMessageFormatInputStream(id, encryptionKey == null ? null : ByteBuffer.wrap(encryptionKey), blobProperties, ByteBuffer.wrap(usermetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blobSize, BlobType.DataBlob, lifeVersion);
byte[] message = Utils.readBytesFromStream(stream, (int) stream.getSize());
return new PutMsgInfoAndBuffer(ByteBuffer.wrap(message), new MessageInfo(id, message.length, false, false, false, EXPIRY_TIME_MS, null, accountId, containerId, CONSTANT_TIME_MS, lifeVersion));
}
Aggregations