use of com.github.ambry.protocol.GetResponse in project ambry by linkedin.
the class ServerTestUtil method checkBlobContent.
private static void checkBlobContent(MockClusterMap clusterMap, BlobId blobId, BlockingChannel channel, byte[] dataToCheck, byte[] encryptionKey) throws IOException, MessageFormatException {
ArrayList<BlobId> listIds = new ArrayList<BlobId>();
listIds.add(blobId);
ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
partitionRequestInfoList.clear();
PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobId.getPartition(), listIds);
partitionRequestInfoList.add(partitionRequestInfo);
GetRequest getRequest3 = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
channel.send(getRequest3);
InputStream stream = channel.receive().getInputStream();
GetResponse resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
assertEquals(ServerErrorCode.No_Error, resp.getError());
BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
byte[] blobout = new byte[(int) blobData.getSize()];
int readsize = 0;
while (readsize < blobData.getSize()) {
readsize += blobData.getStream().read(blobout, readsize, (int) blobData.getSize() - readsize);
}
Assert.assertArrayEquals(dataToCheck, blobout);
if (encryptionKey != null) {
Assert.assertNotNull("EncryptionKey should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
Assert.assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
} else {
Assert.assertNull("EncryptionKey should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
}
}
use of com.github.ambry.protocol.GetResponse in project ambry by linkedin.
the class Verifier method run.
@Override
public void run() {
try {
ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
while (requestsVerified.get() != totalRequests.get() && !cancelTest.get()) {
Payload payload = payloadQueue.poll(1000, TimeUnit.MILLISECONDS);
if (payload != null) {
notificationSystem.awaitBlobCreations(payload.blobId);
for (MockDataNodeId dataNodeId : clusterMap.getDataNodes()) {
ConnectedChannel channel1 = null;
try {
Port port = new Port(portType == PortType.PLAINTEXT ? dataNodeId.getPort() : dataNodeId.getSSLPort(), portType);
channel1 = connectionPool.checkOutConnection("localhost", port, 10000);
ArrayList<BlobId> ids = new ArrayList<BlobId>();
ids.add(new BlobId(payload.blobId, clusterMap));
partitionRequestInfoList.clear();
PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
partitionRequestInfoList.add(partitionRequestInfo);
GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
channel1.send(getRequest);
InputStream stream = channel1.receive().getInputStream();
GetResponse resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
if (resp.getError() != ServerErrorCode.No_Error) {
System.out.println(dataNodeId.getHostname() + " " + dataNodeId.getPort() + " " + resp.getError());
throw new IllegalStateException();
} else {
try {
BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
if (propertyOutput.getBlobSize() != payload.blobProperties.getBlobSize()) {
String exceptionMsg = "blob size not matching " + " expected " + payload.blobProperties.getBlobSize() + " actual " + propertyOutput.getBlobSize();
System.out.println(exceptionMsg);
throw new IllegalStateException(exceptionMsg);
}
if (!propertyOutput.getServiceId().equals(payload.blobProperties.getServiceId())) {
String exceptionMsg = "service id not matching " + " expected " + payload.blobProperties.getServiceId() + " actual " + propertyOutput.getBlobSize();
System.out.println(exceptionMsg);
throw new IllegalStateException(exceptionMsg);
}
if (propertyOutput.getAccountId() != payload.blobProperties.getAccountId()) {
String exceptionMsg = "accountid not matching " + " expected " + payload.blobProperties.getAccountId() + " actual " + propertyOutput.getAccountId();
System.out.println(exceptionMsg);
throw new IllegalStateException(exceptionMsg);
}
if (propertyOutput.getContainerId() != payload.blobProperties.getContainerId()) {
String exceptionMsg = "containerId not matching " + " expected " + payload.blobProperties.getContainerId() + " actual " + propertyOutput.getContainerId();
System.out.println(exceptionMsg);
throw new IllegalStateException(exceptionMsg);
}
if (propertyOutput.isEncrypted() != payload.blobProperties.isEncrypted()) {
String exceptionMsg = "IsEncrypted not matching " + " expected " + payload.blobProperties.isEncrypted() + " actual " + propertyOutput.isEncrypted();
System.out.println(exceptionMsg);
throw new IllegalStateException(exceptionMsg);
}
} catch (MessageFormatException e) {
e.printStackTrace();
throw new IllegalStateException();
}
}
// get user metadata
ids.clear();
ids.add(new BlobId(payload.blobId, clusterMap));
partitionRequestInfoList.clear();
partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
partitionRequestInfoList.add(partitionRequestInfo);
getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
channel1.send(getRequest);
stream = channel1.receive().getInputStream();
resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
if (resp.getError() != ServerErrorCode.No_Error) {
System.out.println("Error after get user metadata " + resp.getError());
throw new IllegalStateException();
} else {
try {
ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
if (userMetadataOutput.compareTo(ByteBuffer.wrap(payload.metadata)) != 0) {
throw new IllegalStateException();
}
} catch (MessageFormatException e) {
e.printStackTrace();
throw new IllegalStateException();
}
}
// get blob
ids.clear();
ids.add(new BlobId(payload.blobId, clusterMap));
partitionRequestInfoList.clear();
partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
partitionRequestInfoList.add(partitionRequestInfo);
getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
channel1.send(getRequest);
stream = channel1.receive().getInputStream();
resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
// System.out.println("response from get " + resp.getError());
if (resp.getError() != ServerErrorCode.No_Error) {
System.out.println("Error after get blob " + resp.getError());
throw new IllegalStateException();
} else {
try {
BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
byte[] blobout = new byte[(int) blobData.getSize()];
int readsize = 0;
while (readsize < blobData.getSize()) {
readsize += blobData.getStream().read(blobout, readsize, (int) blobData.getSize() - readsize);
}
if (ByteBuffer.wrap(blobout).compareTo(ByteBuffer.wrap(payload.blob)) != 0) {
throw new IllegalStateException();
}
} catch (MessageFormatException e) {
e.printStackTrace();
throw new IllegalStateException();
}
}
// get blob all
getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
channel1.send(getRequest);
stream = channel1.receive().getInputStream();
resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
if (resp.getError() != ServerErrorCode.No_Error) {
System.out.println("Error after get blob " + resp.getError());
throw new IllegalStateException();
} else {
try {
BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), new BlobIdFactory(clusterMap));
byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
int readsize = 0;
while (readsize < blobAll.getBlobData().getSize()) {
readsize += blobAll.getBlobData().getStream().read(blobout, readsize, (int) blobAll.getBlobData().getSize() - readsize);
}
if (ByteBuffer.wrap(blobout).compareTo(ByteBuffer.wrap(payload.blob)) != 0) {
throw new IllegalStateException();
}
} catch (MessageFormatException e) {
e.printStackTrace();
throw new IllegalStateException();
}
}
} catch (Exception e) {
if (channel1 != null) {
connectionPool.destroyConnection(channel1);
channel1 = null;
}
} finally {
if (channel1 != null) {
connectionPool.checkInConnection(channel1);
channel1 = null;
}
}
}
requestsVerified.incrementAndGet();
}
}
} catch (Exception e) {
e.printStackTrace();
cancelTest.set(true);
} finally {
completedLatch.countDown();
}
}
use of com.github.ambry.protocol.GetResponse in project ambry by linkedin.
the class AmbryRequests method handleGetRequest.
public void handleGetRequest(Request request) throws IOException, InterruptedException {
GetRequest getRequest = GetRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
Histogram responseQueueTime = null;
Histogram responseSendTime = null;
Histogram responseTotalTime = null;
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
metrics.getBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobRequestRate.mark();
responseQueueTime = metrics.getBlobResponseQueueTimeInMs;
responseSendTime = metrics.getBlobSendTimeInMs;
responseTotalTime = metrics.getBlobTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
metrics.getBlobPropertiesRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobPropertiesRequestRate.mark();
responseQueueTime = metrics.getBlobPropertiesResponseQueueTimeInMs;
responseSendTime = metrics.getBlobPropertiesSendTimeInMs;
responseTotalTime = metrics.getBlobPropertiesTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
metrics.getBlobUserMetadataRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobUserMetadataRequestRate.mark();
responseQueueTime = metrics.getBlobUserMetadataResponseQueueTimeInMs;
responseSendTime = metrics.getBlobUserMetadataSendTimeInMs;
responseTotalTime = metrics.getBlobUserMetadataTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
metrics.getBlobInfoRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobInfoRequestRate.mark();
responseQueueTime = metrics.getBlobInfoResponseQueueTimeInMs;
responseSendTime = metrics.getBlobInfoSendTimeInMs;
responseTotalTime = metrics.getBlobInfoTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
metrics.getBlobAllRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobAllRequestRate.mark();
responseQueueTime = metrics.getBlobAllResponseQueueTimeInMs;
responseSendTime = metrics.getBlobAllSendTimeInMs;
responseTotalTime = metrics.getBlobAllTotalTimeInMs;
}
long startTime = SystemTime.getInstance().milliseconds();
GetResponse response = null;
try {
List<Send> messagesToSendList = new ArrayList<Send>(getRequest.getPartitionInfoList().size());
List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<PartitionResponseInfo>(getRequest.getPartitionInfoList().size());
for (PartitionRequestInfo partitionRequestInfo : getRequest.getPartitionInfoList()) {
ServerErrorCode error = validateRequest(partitionRequestInfo.getPartition(), RequestOrResponseType.GetRequest);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating get request failed for partition {} with error {}", partitionRequestInfo.getPartition(), error);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), error);
partitionResponseInfoList.add(partitionResponseInfo);
} else {
try {
Store storeToGet = storageManager.getStore(partitionRequestInfo.getPartition());
EnumSet<StoreGetOptions> storeGetOptions = EnumSet.noneOf(StoreGetOptions.class);
// Currently only one option is supported.
if (getRequest.getGetOption() == GetOption.Include_Expired_Blobs) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Expired);
}
if (getRequest.getGetOption() == GetOption.Include_Deleted_Blobs) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted);
}
if (getRequest.getGetOption() == GetOption.Include_All) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted, StoreGetOptions.Store_Include_Expired);
}
StoreInfo info = storeToGet.get(partitionRequestInfo.getBlobIds(), storeGetOptions);
MessageFormatSend blobsToSend = new MessageFormatSend(info.getMessageReadSet(), getRequest.getMessageFormatFlag(), messageFormatMetrics, storeKeyFactory);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), info.getMessageReadSetInfo(), blobsToSend.getMessageMetadataList());
messagesToSendList.add(blobsToSend);
partitionResponseInfoList.add(partitionResponseInfo);
} catch (StoreException e) {
boolean logInErrorLevel = false;
if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
metrics.idNotFoundError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
metrics.ttlExpiredError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
metrics.idDeletedError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
metrics.getAuthorizationFailure.inc();
} else {
metrics.unExpectedStoreGetError.inc();
logInErrorLevel = true;
}
if (logInErrorLevel) {
logger.error("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
} else {
logger.trace("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
}
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
partitionResponseInfoList.add(partitionResponseInfo);
} catch (MessageFormatException e) {
logger.error("Message format exception on a get with error code " + e.getErrorCode() + " for partitionRequestInfo " + partitionRequestInfo, e);
if (e.getErrorCode() == MessageFormatErrorCodes.Data_Corrupt) {
metrics.dataCorruptError.inc();
} else if (e.getErrorCode() == MessageFormatErrorCodes.Unknown_Format_Version) {
metrics.unknownFormatError.inc();
}
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getMessageFormatErrorMapping(e.getErrorCode()));
partitionResponseInfoList.add(partitionResponseInfo);
}
}
}
CompositeSend compositeSend = new CompositeSend(messagesToSendList);
response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, compositeSend, ServerErrorCode.No_Error);
} catch (Exception e) {
logger.error("Unknown exception for request " + getRequest, e);
response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), ServerErrorCode.Unknown_Error);
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", getRequest, response, processingTime);
if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
metrics.getBlobProcessingTimeInMs.update(processingTime);
metrics.updateGetBlobProcessingTimeBySize(response.sizeInBytes(), processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
metrics.getBlobPropertiesProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
metrics.getBlobUserMetadataProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
metrics.getBlobInfoProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
metrics.getBlobAllProcessingTimeInMs.update(processingTime);
metrics.updateGetBlobProcessingTimeBySize(response.sizeInBytes(), processingTime);
}
}
sendGetResponse(requestResponseChannel, response, request, responseQueueTime, responseSendTime, responseTotalTime, totalTimeSpent, response.sizeInBytes(), getRequest.getMessageFormatFlag(), metrics);
}
use of com.github.ambry.protocol.GetResponse in project ambry by linkedin.
the class AmbryRequestsTest method sendAndVerifyOperationRequest.
/**
* Sends and verifies that an operation specific request works correctly.
* @param requestType the type of the request to send.
* @param ids the partitionIds to send requests for.
* @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
* response in the constituents rather than the actual response ({@link GetResponse} and
* {@link ReplicaMetadataResponse}).
* @throws InterruptedException
* @throws IOException
*/
private void sendAndVerifyOperationRequest(RequestOrResponseType requestType, List<? extends PartitionId> ids, ServerErrorCode expectedErrorCode) throws InterruptedException, IOException {
for (PartitionId id : ids) {
int correlationId = TestUtils.RANDOM.nextInt();
String clientId = UtilsTest.getRandomString(10);
BlobId blobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMapUtils.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), id, false);
RequestOrResponse request;
switch(requestType) {
case PutRequest:
BlobProperties properties = new BlobProperties(0, "serviceId", blobId.getAccountId(), blobId.getAccountId(), false);
request = new PutRequest(correlationId, clientId, blobId, properties, ByteBuffer.allocate(0), ByteBuffer.allocate(0), 0, BlobType.DataBlob, null);
break;
case DeleteRequest:
request = new DeleteRequest(correlationId, clientId, blobId, SystemTime.getInstance().milliseconds());
break;
case GetRequest:
PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(id, Collections.singletonList(blobId));
request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
break;
case ReplicaMetadataRequest:
ReplicaMetadataRequestInfo rRequestInfo = new ReplicaMetadataRequestInfo(id, FIND_TOKEN_FACTORY.getNewFindToken(), "localhost", "/tmp");
request = new ReplicaMetadataRequest(correlationId, clientId, Collections.singletonList(rRequestInfo), Long.MAX_VALUE);
break;
default:
throw new IllegalArgumentException(requestType + " not supported by this function");
}
storageManager.resetStore();
Response response = sendRequestGetResponse(request, requestType == RequestOrResponseType.GetRequest || requestType == RequestOrResponseType.ReplicaMetadataRequest ? ServerErrorCode.No_Error : expectedErrorCode);
if (expectedErrorCode.equals(ServerErrorCode.No_Error)) {
assertEquals("Operation received at the store not as expected", requestType, MockStorageManager.operationReceived);
}
if (requestType == RequestOrResponseType.GetRequest) {
GetResponse getResponse = (GetResponse) response;
for (PartitionResponseInfo info : getResponse.getPartitionResponseInfoList()) {
assertEquals("Error code does not match expected", expectedErrorCode, info.getErrorCode());
}
} else if (requestType == RequestOrResponseType.ReplicaMetadataRequest) {
ReplicaMetadataResponse replicaMetadataResponse = (ReplicaMetadataResponse) response;
for (ReplicaMetadataResponseInfo info : replicaMetadataResponse.getReplicaMetadataResponseInfoList()) {
assertEquals("Error code does not match expected", expectedErrorCode, info.getError());
}
}
}
}
use of com.github.ambry.protocol.GetResponse in project ambry by linkedin.
the class ServerAdminTool method getGetResponse.
/**
* Sends a {@link GetRequest} based on the provided parameters and returns the response stream if the request was
* successful. {@code null} otherwise.
* @param dataNodeId the {@link DataNodeId} to contact.
* @param blobId the {@link BlobId} to operate on.
* @param flags the {@link MessageFormatFlags} associated with the {@link GetRequest}.
* @param getOption the {@link GetOption} to send with the {@link GetRequest}.
* @param clusterMap the {@link ClusterMap} to use.
* @return the {@link ServerErrorCode} and response stream if the request was successful. {@code null} for the
* response stream otherwise.
* @throws Exception
*/
private Pair<ServerErrorCode, InputStream> getGetResponse(DataNodeId dataNodeId, BlobId blobId, MessageFormatFlags flags, GetOption getOption, ClusterMap clusterMap) throws Exception {
PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobId.getPartition(), Collections.singletonList(blobId));
List<PartitionRequestInfo> partitionRequestInfos = new ArrayList<>();
partitionRequestInfos.add(partitionRequestInfo);
GetRequest getRequest = new GetRequest(correlationId.incrementAndGet(), CLIENT_ID, flags, partitionRequestInfos, getOption);
InputStream serverResponseStream = new ByteBufferInputStream(sendRequestGetResponse(dataNodeId, getRequest));
GetResponse getResponse = GetResponse.readFrom(new DataInputStream(serverResponseStream), clusterMap);
ServerErrorCode partitionErrorCode = getResponse.getPartitionResponseInfoList().get(0).getErrorCode();
ServerErrorCode errorCode = partitionErrorCode == ServerErrorCode.No_Error ? getResponse.getError() : partitionErrorCode;
InputStream stream = errorCode == ServerErrorCode.No_Error ? getResponse.getInputStream() : null;
return new Pair<>(errorCode, stream);
}
Aggregations