use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.
the class RequestResponseTest method testGetRequestResponse.
private void testGetRequestResponse(short getVersionToUse, short messageInfoAutoVersion) throws IOException {
GetResponse.CURRENT_VERSION = getVersionToUse;
MessageInfoAndMetadataListSerde.AUTO_VERSION = messageInfoAutoVersion;
MockClusterMap clusterMap = new MockClusterMap();
short accountId = Utils.getRandomShort(TestUtils.RANDOM);
short containerId = Utils.getRandomShort(TestUtils.RANDOM);
BlobId id1 = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, accountId, containerId, clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK);
ArrayList<BlobId> blobIdList = new ArrayList<BlobId>();
blobIdList.add(id1);
PartitionRequestInfo partitionRequestInfo1 = new PartitionRequestInfo(new MockPartitionId(), blobIdList);
ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
partitionRequestInfoList.add(partitionRequestInfo1);
GetRequest getRequest = new GetRequest(1234, "clientId", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
DataInputStream requestStream = serAndPrepForRead(getRequest, -1, true);
GetRequest deserializedGetRequest = GetRequest.readFrom(requestStream, clusterMap);
Assert.assertEquals(deserializedGetRequest.getClientId(), "clientId");
Assert.assertEquals(deserializedGetRequest.getPartitionInfoList().size(), 1);
Assert.assertEquals(deserializedGetRequest.getPartitionInfoList().get(0).getBlobIds().size(), 1);
Assert.assertEquals(deserializedGetRequest.getPartitionInfoList().get(0).getBlobIds().get(0), id1);
getRequest.release();
// Test GetResponse with InputStream
long operationTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
byte[] encryptionKey = TestUtils.getRandomBytes(256);
MessageInfo messageInfo = new MessageInfo(id1, 1000, false, false, true, 1000, null, accountId, containerId, operationTimeMs, (short) 1);
MessageMetadata messageMetadata = new MessageMetadata(ByteBuffer.wrap(encryptionKey));
ArrayList<MessageInfo> messageInfoList = new ArrayList<>();
ArrayList<MessageMetadata> messageMetadataList = new ArrayList<>();
messageInfoList.add(messageInfo);
messageMetadataList.add(messageMetadata);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), messageInfoList, messageMetadataList);
List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<PartitionResponseInfo>();
partitionResponseInfoList.add(partitionResponseInfo);
byte[] buf = TestUtils.getRandomBytes(1000);
ByteArrayInputStream byteStream = new ByteArrayInputStream(buf);
GetResponse response = new GetResponse(1234, "clientId", partitionResponseInfoList, byteStream, ServerErrorCode.No_Error);
requestStream = serAndPrepForRead(response, -1, false);
GetResponse deserializedGetResponse = GetResponse.readFrom(requestStream, clusterMap);
Assert.assertEquals(deserializedGetResponse.getCorrelationId(), 1234);
Assert.assertEquals(deserializedGetResponse.getError(), ServerErrorCode.No_Error);
Assert.assertEquals(deserializedGetResponse.getPartitionResponseInfoList().size(), 1);
Assert.assertEquals(deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().size(), 1);
MessageInfo msgInfo = deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
Assert.assertEquals(msgInfo.getSize(), 1000);
Assert.assertEquals(msgInfo.getStoreKey(), id1);
Assert.assertEquals(msgInfo.getExpirationTimeInMs(), 1000);
Assert.assertEquals(deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageMetadataList().size(), 1);
if (GetResponse.getCurrentVersion() >= GetResponse.GET_RESPONSE_VERSION_V_4) {
MessageMetadata messageMetadataInResponse = deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0);
Assert.assertEquals(messageMetadata.getEncryptionKey().rewind(), messageMetadataInResponse.getEncryptionKey());
} else {
Assert.assertNull(deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
}
if (GetResponse.getCurrentVersion() >= GetResponse.GET_RESPONSE_VERSION_V_3) {
Assert.assertEquals("AccountId mismatch ", accountId, msgInfo.getAccountId());
Assert.assertEquals("ConatinerId mismatch ", containerId, msgInfo.getContainerId());
Assert.assertEquals("OperationTime mismatch ", operationTimeMs, msgInfo.getOperationTimeMs());
} else {
Assert.assertEquals("AccountId mismatch ", UNKNOWN_ACCOUNT_ID, msgInfo.getAccountId());
Assert.assertEquals("ConatinerId mismatch ", UNKNOWN_CONTAINER_ID, msgInfo.getContainerId());
Assert.assertEquals("OperationTime mismatch ", Utils.Infinite_Time, msgInfo.getOperationTimeMs());
}
if (messageInfoAutoVersion >= MessageInfoAndMetadataListSerde.VERSION_6) {
Assert.assertTrue(msgInfo.isUndeleted());
Assert.assertEquals("LifeVersion mismatch", (short) 1, msgInfo.getLifeVersion());
} else {
Assert.assertFalse(msgInfo.isUndeleted());
Assert.assertEquals("LifeVersion mismatch", (short) 0, msgInfo.getLifeVersion());
}
response.release();
// Test GetResponse with Send
for (boolean useComposite : new boolean[] { false, true }) {
for (boolean withContent : new boolean[] { false, true }) {
operationTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
encryptionKey = TestUtils.getRandomBytes(256);
messageInfo = new MessageInfo(id1, 1000, false, false, true, 1000, null, accountId, containerId, operationTimeMs, (short) 1);
messageMetadata = new MessageMetadata(ByteBuffer.wrap(encryptionKey));
messageInfoList.clear();
messageMetadataList.clear();
messageInfoList.add(messageInfo);
messageMetadataList.add(messageMetadata);
partitionResponseInfo = new PartitionResponseInfo(clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), messageInfoList, messageMetadataList);
partitionResponseInfoList.clear();
partitionResponseInfoList.add(partitionResponseInfo);
Send send;
if (withContent) {
send = new SendWithContent(1000, useComposite);
} else {
send = new SendWithoutContent(1000, useComposite);
}
response = new GetResponse(1234, "clientId", partitionResponseInfoList, send, ServerErrorCode.No_Error);
requestStream = serAndPrepForRead(response, -1, false);
deserializedGetResponse = GetResponse.readFrom(requestStream, clusterMap);
Assert.assertEquals(deserializedGetResponse.getCorrelationId(), 1234);
Assert.assertEquals(deserializedGetResponse.getError(), ServerErrorCode.No_Error);
Assert.assertEquals(deserializedGetResponse.getPartitionResponseInfoList().size(), 1);
Assert.assertEquals(deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().size(), 1);
msgInfo = deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
Assert.assertEquals(msgInfo.getSize(), 1000);
Assert.assertEquals(msgInfo.getStoreKey(), id1);
Assert.assertEquals(msgInfo.getExpirationTimeInMs(), 1000);
Assert.assertEquals(deserializedGetResponse.getPartitionResponseInfoList().get(0).getMessageMetadataList().size(), 1);
response.release();
}
}
}
use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.
the class GetBlobInfoOperation method processGetBlobInfoResponse.
/**
* Process the {@link GetResponse} extracted from a {@link ResponseInfo}
* @param getRequestInfo the associated {@link GetRequestInfo} for which this response was received.
* @param getResponse the {@link GetResponse} extracted from the {@link ResponseInfo}
* @throws IOException if there is an error during deserialization of the GetResponse.
* @throws MessageFormatException if there is an error during deserialization of the GetResponse.
*/
private void processGetBlobInfoResponse(GetRequestInfo getRequestInfo, GetResponse getResponse) throws IOException, MessageFormatException {
ServerErrorCode getError = getResponse.getError();
if (getError == ServerErrorCode.No_Error) {
int partitionsInResponse = getResponse.getPartitionResponseInfoList().size();
// Each get request issued by the router is for a single blob.
if (partitionsInResponse != 1) {
onErrorResponse(getRequestInfo.replicaId, new RouterException("Unexpected number of partition responses, expected: 1, " + "received: " + partitionsInResponse, RouterErrorCode.UnexpectedInternalError));
// Again, no need to notify the responseHandler.
} else {
getError = getResponse.getPartitionResponseInfoList().get(0).getErrorCode();
if (getError == ServerErrorCode.No_Error) {
PartitionResponseInfo partitionResponseInfo = getResponse.getPartitionResponseInfoList().get(0);
int msgsInResponse = partitionResponseInfo.getMessageInfoList().size();
if (msgsInResponse != 1) {
onErrorResponse(getRequestInfo.replicaId, new RouterException("Unexpected number of messages in a partition response, expected: 1, " + "received: " + msgsInResponse, RouterErrorCode.UnexpectedInternalError));
} else {
MessageMetadata messageMetadata = partitionResponseInfo.getMessageMetadataList().get(0);
MessageInfo messageInfo = partitionResponseInfo.getMessageInfoList().get(0);
handleBody(getResponse.getInputStream(), messageMetadata, messageInfo);
operationTracker.onResponse(getRequestInfo.replicaId, TrackedRequestFinalState.SUCCESS);
if (RouterUtils.isRemoteReplica(routerConfig, getRequestInfo.replicaId)) {
logger.trace("Cross colo request successful for remote replica in {} ", getRequestInfo.replicaId.getDataNodeId().getDatacenterName());
routerMetrics.crossColoSuccessCount.inc();
}
}
} else {
// process and set the most relevant exception.
logger.trace("Replica {} returned error {} with response correlationId {} ", getRequestInfo.replicaId.getDataNodeId(), getError, getResponse.getCorrelationId());
RouterErrorCode routerErrorCode = processServerError(getError);
if (getError == ServerErrorCode.Disk_Unavailable) {
operationTracker.onResponse(getRequestInfo.replicaId, TrackedRequestFinalState.DISK_DOWN);
setOperationException(new RouterException("Server returned: " + getError, routerErrorCode));
routerMetrics.routerRequestErrorCount.inc();
routerMetrics.getDataNodeBasedMetrics(getRequestInfo.replicaId.getDataNodeId()).getBlobInfoRequestErrorCount.inc();
} else {
if (getError == ServerErrorCode.Blob_Deleted || getError == ServerErrorCode.Blob_Expired || getError == ServerErrorCode.Blob_Authorization_Failure) {
// this is a successful response and one that completes the operation regardless of whether the
// success target has been reached or not.
operationCompleted = true;
}
// any server error code that is not equal to ServerErrorCode.No_Error, the onErrorResponse should be invoked
// because the operation itself doesn't succeed although the response in some cases is successful (i.e. Blob_Deleted)
onErrorResponse(getRequestInfo.replicaId, new RouterException("Server returned: " + getError, routerErrorCode));
}
}
}
} else {
logger.trace("Replica {} returned an error {} for a GetBlobInfoRequest with response correlationId : {} ", getRequestInfo.replicaId.getDataNodeId(), getError, getResponse.getCorrelationId());
onErrorResponse(getRequestInfo.replicaId, new RouterException("Server returned", processServerError(getError)));
}
}
use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.
the class StoredBlob method makeGetResponse.
/**
* Make a {@link GetResponse} for the given {@link GetRequest} for which the given {@link ServerErrorCode} was
* encountered. The request could be for BlobInfo or for Blob (the only two options that the router would request
* for).
* @param getRequest the {@link GetRequest} for which the response is being constructed.
* @param getError the {@link ServerErrorCode} that was encountered.
* @return the constructed {@link GetResponse}
* @throws IOException if there was an error constructing the response.
*/
GetResponse makeGetResponse(GetRequest getRequest, ServerErrorCode getError) throws IOException {
GetResponse getResponse;
if (getError == ServerErrorCode.No_Error) {
List<PartitionRequestInfo> infos = getRequest.getPartitionInfoList();
if (infos.size() != 1 || infos.get(0).getBlobIds().size() != 1) {
getError = ServerErrorCode.Unknown_Error;
}
}
ServerErrorCode serverError;
ServerErrorCode partitionError;
boolean isDataBlob = false;
try {
String id = getRequest.getPartitionInfoList().get(0).getBlobIds().get(0).getID();
isDataBlob = blobs.get(id).type == BlobType.DataBlob;
} catch (Exception ignored) {
}
if (!getErrorOnDataBlobOnly || isDataBlob) {
// set it in the partitionResponseInfo
if (getError == ServerErrorCode.No_Error || getError == ServerErrorCode.Blob_Expired || getError == ServerErrorCode.Blob_Deleted || getError == ServerErrorCode.Blob_Not_Found || getError == ServerErrorCode.Blob_Authorization_Failure || getError == ServerErrorCode.Disk_Unavailable) {
partitionError = getError;
serverError = ServerErrorCode.No_Error;
} else {
serverError = getError;
// does not matter - this will not be checked if serverError is not No_Error.
partitionError = ServerErrorCode.No_Error;
}
} else {
serverError = ServerErrorCode.No_Error;
partitionError = ServerErrorCode.No_Error;
}
if (serverError == ServerErrorCode.No_Error) {
int byteBufferSize;
ByteBuffer byteBuffer;
StoreKey key = getRequest.getPartitionInfoList().get(0).getBlobIds().get(0);
short accountId = Account.UNKNOWN_ACCOUNT_ID;
short containerId = Container.UNKNOWN_CONTAINER_ID;
long operationTimeMs = Utils.Infinite_Time;
StoredBlob blob = blobs.get(key.getID());
ServerErrorCode processedError = errorForGet(key.getID(), blob, getRequest);
MessageMetadata msgMetadata = null;
if (processedError == ServerErrorCode.No_Error) {
ByteBuffer buf = blobs.get(key.getID()).serializedSentPutRequest.duplicate();
// read off the size
buf.getLong();
// read off the type.
buf.getShort();
PutRequest originalBlobPutReq = PutRequest.readFrom(new DataInputStream(new ByteBufferInputStream(buf)), clusterMap);
switch(getRequest.getMessageFormatFlag()) {
case BlobInfo:
BlobProperties blobProperties = originalBlobPutReq.getBlobProperties();
accountId = blobProperties.getAccountId();
containerId = blobProperties.getContainerId();
operationTimeMs = blobProperties.getCreationTimeInMs();
ByteBuffer userMetadata = originalBlobPutReq.getUsermetadata();
byteBufferSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties) + MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
byteBuffer = ByteBuffer.allocate(byteBufferSize);
if (originalBlobPutReq.getBlobEncryptionKey() != null) {
msgMetadata = new MessageMetadata(originalBlobPutReq.getBlobEncryptionKey().duplicate());
}
MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(byteBuffer, blobProperties);
MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(byteBuffer, userMetadata);
break;
case Blob:
switch(blobFormatVersion) {
case MessageFormatRecord.Blob_Version_V2:
if (originalBlobPutReq.getBlobEncryptionKey() != null) {
msgMetadata = new MessageMetadata(originalBlobPutReq.getBlobEncryptionKey().duplicate());
}
byteBufferSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
byteBuffer = ByteBuffer.allocate(byteBufferSize);
MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize(), originalBlobPutReq.getBlobType());
break;
case MessageFormatRecord.Blob_Version_V1:
byteBufferSize = (int) MessageFormatRecord.Blob_Format_V1.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
byteBuffer = ByteBuffer.allocate(byteBufferSize);
MessageFormatRecord.Blob_Format_V1.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize());
break;
default:
throw new IllegalStateException("Blob format version " + blobFormatVersion + " not supported.");
}
byteBuffer.put(Utils.readBytesFromStream(originalBlobPutReq.getBlobStream(), (int) originalBlobPutReq.getBlobSize()));
Crc32 crc = new Crc32();
crc.update(byteBuffer.array(), 0, byteBuffer.position());
byteBuffer.putLong(crc.getValue());
break;
case All:
blobProperties = originalBlobPutReq.getBlobProperties();
accountId = blobProperties.getAccountId();
containerId = blobProperties.getContainerId();
userMetadata = originalBlobPutReq.getUsermetadata();
operationTimeMs = originalBlobPutReq.getBlobProperties().getCreationTimeInMs();
int blobHeaderSize = MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize();
int blobEncryptionRecordSize = originalBlobPutReq.getBlobEncryptionKey() != null ? MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(originalBlobPutReq.getBlobEncryptionKey().duplicate()) : 0;
int blobPropertiesSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
int blobInfoSize = blobPropertiesSize + userMetadataSize;
int blobRecordSize;
switch(blobFormatVersion) {
case MessageFormatRecord.Blob_Version_V2:
blobRecordSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
break;
case MessageFormatRecord.Blob_Version_V1:
blobRecordSize = (int) MessageFormatRecord.Blob_Format_V1.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
break;
default:
throw new IllegalStateException("Blob format version " + blobFormatVersion + " not supported.");
}
byteBufferSize = blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize + blobInfoSize + blobRecordSize;
byteBuffer = ByteBuffer.allocate(byteBufferSize);
try {
MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(byteBuffer, blobEncryptionRecordSize + blobInfoSize + blobRecordSize, originalBlobPutReq.getBlobEncryptionKey() == null ? Message_Header_Invalid_Relative_Offset : blobHeaderSize + key.sizeInBytes(), blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize, Message_Header_Invalid_Relative_Offset, blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize + blobPropertiesSize, blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize + blobInfoSize);
} catch (MessageFormatException e) {
e.printStackTrace();
}
byteBuffer.put(key.toBytes());
if (originalBlobPutReq.getBlobEncryptionKey() != null) {
MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(byteBuffer, originalBlobPutReq.getBlobEncryptionKey().duplicate());
msgMetadata = new MessageMetadata(originalBlobPutReq.getBlobEncryptionKey().duplicate());
}
MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(byteBuffer, blobProperties);
MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(byteBuffer, userMetadata);
int blobRecordStart = byteBuffer.position();
switch(blobFormatVersion) {
case MessageFormatRecord.Blob_Version_V2:
MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize(), originalBlobPutReq.getBlobType());
break;
case MessageFormatRecord.Blob_Version_V1:
MessageFormatRecord.Blob_Format_V1.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize());
break;
default:
throw new IllegalStateException("Blob format version " + blobFormatVersion + " not supported.");
}
byteBuffer.put(Utils.readBytesFromStream(originalBlobPutReq.getBlobStream(), (int) originalBlobPutReq.getBlobSize()));
crc = new Crc32();
crc.update(byteBuffer.array(), blobRecordStart, blobRecordSize - MessageFormatRecord.Crc_Size);
byteBuffer.putLong(crc.getValue());
break;
default:
throw new IOException("GetRequest flag is not supported: " + getRequest.getMessageFormatFlag());
}
} else if (processedError == ServerErrorCode.Blob_Deleted) {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Deleted;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
} else if (processedError == ServerErrorCode.Blob_Expired) {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Expired;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
} else if (processedError == ServerErrorCode.Blob_Authorization_Failure) {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Authorization_Failure;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
} else {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Not_Found;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
}
byteBuffer.flip();
ByteBufferSend responseSend = new ByteBufferSend(byteBuffer);
List<MessageInfo> messageInfoList = new ArrayList<>();
List<MessageMetadata> messageMetadataList = new ArrayList<>();
List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<PartitionResponseInfo>();
if (partitionError == ServerErrorCode.No_Error) {
messageInfoList.add(new MessageInfo(key, byteBufferSize, false, blob.isTtlUpdated(), blob.isUndeleted(), blob.expiresAt, null, accountId, containerId, operationTimeMs, blob.lifeVersion));
messageMetadataList.add(msgMetadata);
}
PartitionResponseInfo partitionResponseInfo = partitionError == ServerErrorCode.No_Error ? new PartitionResponseInfo(getRequest.getPartitionInfoList().get(0).getPartition(), messageInfoList, messageMetadataList) : new PartitionResponseInfo(getRequest.getPartitionInfoList().get(0).getPartition(), partitionError);
partitionResponseInfoList.add(partitionResponseInfo);
getResponse = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, responseSend, serverError);
} else {
getResponse = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), new ArrayList<PartitionResponseInfo>(), new ByteBufferSend(ByteBuffer.allocate(0)), serverError);
}
return getResponse;
}
Aggregations