use of com.github.ambry.messageformat.PutMessageFormatInputStream in project ambry by linkedin.
the class CloudAndStoreReplicationTest method cloudRecoveryTestForLargeBlob.
/**
* Test replication from vcr to server nodes, and from server to server nodes for large blobs.
* Creates one vcr node and two server nodes.
* Uploads data to vcr node and verifies that they have been replicated.
* Uploads data to one of the server nodes and verifies that they have been replicated.
* @throws Exception If an exception happens.
*/
@Test
public void cloudRecoveryTestForLargeBlob() throws Exception {
// Create blobs and upload to cloud destination.
Map<BlobId, Integer> blobIdToSizeMap = new HashMap<>();
// Currently ambry supports max size of 4MB for blobs.
int blobSize = FOUR_MB_SZ;
for (BlobId blobId : cloudBlobIds) {
PutMessageFormatInputStream putMessageFormatInputStream = ServerTestUtil.getPutMessageInputStreamForBlob(blobId, blobSize, blobIdToSizeMap, accountId, containerId);
long time = System.currentTimeMillis();
CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, time, Utils.Infinite_Time, putMessageFormatInputStream.getSize(), CloudBlobMetadata.EncryptionOrigin.NONE);
latchBasedInMemoryCloudDestination.uploadBlob(blobId, putMessageFormatInputStream.getSize(), cloudBlobMetadata, putMessageFormatInputStream);
}
// Create blobs and upload to one of the server nodes.
sendBlobToDataNode(partitionLeaderRecoveryNode, blobSize, blobIdToSizeMap);
// Waiting for download attempt
assertTrue("Did not recover all blobs in 1 minute", latchBasedInMemoryCloudDestination.awaitDownload(1, TimeUnit.MINUTES));
// Waiting for replication to complete
Thread.sleep(10000);
// Test cloud to store and store to store replication by sending get request to server nodes.
testGetOnServerNodes(blobIdToSizeMap);
}
use of com.github.ambry.messageformat.PutMessageFormatInputStream in project ambry by linkedin.
the class AmbryRequests method handlePutRequest.
public void handlePutRequest(Request request) throws IOException, InterruptedException {
PutRequest.ReceivedPutRequest receivedRequest = PutRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
metrics.putBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.putBlobRequestRate.mark();
long startTime = SystemTime.getInstance().milliseconds();
PutResponse response = null;
try {
ServerErrorCode error = validateRequest(receivedRequest.getBlobId().getPartition(), RequestOrResponseType.PutRequest);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating put request failed with error {} for request {}", error, receivedRequest);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), error);
} else {
MessageFormatInputStream stream = new PutMessageFormatInputStream(receivedRequest.getBlobId(), receivedRequest.getBlobEncryptionKey(), receivedRequest.getBlobProperties(), receivedRequest.getUsermetadata(), receivedRequest.getBlobStream(), receivedRequest.getBlobSize(), receivedRequest.getBlobType());
MessageInfo info = new MessageInfo(receivedRequest.getBlobId(), stream.getSize(), false, Utils.addSecondsToEpochTime(receivedRequest.getBlobProperties().getCreationTimeInMs(), receivedRequest.getBlobProperties().getTimeToLiveInSeconds()), receivedRequest.getCrc(), receivedRequest.getBlobProperties().getAccountId(), receivedRequest.getBlobProperties().getContainerId(), receivedRequest.getBlobProperties().getCreationTimeInMs());
ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
infoList.add(info);
MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
Store storeToPut = storageManager.getStore(receivedRequest.getBlobId().getPartition());
storeToPut.put(writeset);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.No_Error);
metrics.blobSizeInBytes.update(receivedRequest.getBlobSize());
metrics.blobUserMetadataSizeInBytes.update(receivedRequest.getUsermetadata().limit());
if (notification != null) {
notification.onBlobReplicaCreated(currentNode.getHostname(), currentNode.getPort(), receivedRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
}
}
} catch (StoreException e) {
logger.error("Store exception on a put with error code " + e.getErrorCode() + " for request " + receivedRequest, e);
if (e.getErrorCode() == StoreErrorCodes.Already_Exist) {
metrics.idAlreadyExistError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.IOError) {
metrics.storeIOError.inc();
} else {
metrics.unExpectedStorePutError.inc();
}
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
} catch (Exception e) {
logger.error("Unknown exception on a put for request " + receivedRequest, e);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.Unknown_Error);
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", receivedRequest, response, processingTime);
metrics.putBlobProcessingTimeInMs.update(processingTime);
metrics.updatePutBlobProcessingTimeBySize(receivedRequest.getBlobSize(), processingTime);
}
sendPutResponse(requestResponseChannel, response, request, metrics.putBlobResponseQueueTimeInMs, metrics.putBlobSendTimeInMs, metrics.putBlobTotalTimeInMs, totalTimeSpent, receivedRequest.getBlobSize(), metrics);
}
use of com.github.ambry.messageformat.PutMessageFormatInputStream in project ambry by linkedin.
the class AmbryRequests method handlePutRequest.
@Override
public void handlePutRequest(NetworkRequest request) throws IOException, InterruptedException {
PutRequest receivedRequest;
if (request instanceof LocalChannelRequest) {
// This is a case where handlePutRequest is called when frontends are writing to Azure. In this case, this method
// is called by request handler threads running within the frontend router itself. So, the request can be directly
// referenced as java objects without any need for deserialization.
PutRequest sentRequest = (PutRequest) ((LocalChannelRequest) request).getRequestInfo().getRequest();
// However, we will create a new PutRequest object to represent the received Put request since the blob content
// 'buffer' in PutRequest is accessed as 'stream' while writing to Store. Also, crc value for this request
// would be null since it is only calculated (on the fly) when sending the request to network. It might be okay to
// use null crc here since the scenario for which we are using crc (i.e. possibility of collisions due to fast
// replication) as described in this PR https://github.com/linkedin/ambry/pull/549 might not be applicable when
// frontends are talking to Azure.
receivedRequest = new PutRequest(sentRequest.getCorrelationId(), sentRequest.getClientId(), sentRequest.getBlobId(), sentRequest.getBlobProperties(), sentRequest.getUsermetadata(), sentRequest.getBlobSize(), sentRequest.getBlobType(), sentRequest.getBlobEncryptionKey(), new ByteBufInputStream(sentRequest.getBlob()), null);
} else {
InputStream is = request.getInputStream();
DataInputStream dis = is instanceof DataInputStream ? (DataInputStream) is : new DataInputStream(is);
receivedRequest = PutRequest.readFrom(dis, clusterMap);
}
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
metrics.putBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.putBlobRequestRate.mark();
long startTime = SystemTime.getInstance().milliseconds();
PutResponse response = null;
try {
ServerErrorCode error = validateRequest(receivedRequest.getBlobId().getPartition(), RequestOrResponseType.PutRequest, false);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating put request failed with error {} for request {}", error, receivedRequest);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), error);
} else {
MessageFormatInputStream stream = new PutMessageFormatInputStream(receivedRequest.getBlobId(), receivedRequest.getBlobEncryptionKey(), receivedRequest.getBlobProperties(), receivedRequest.getUsermetadata(), receivedRequest.getBlobStream(), receivedRequest.getBlobSize(), receivedRequest.getBlobType());
BlobProperties properties = receivedRequest.getBlobProperties();
long expirationTime = Utils.addSecondsToEpochTime(receivedRequest.getBlobProperties().getCreationTimeInMs(), properties.getTimeToLiveInSeconds());
MessageInfo info = new MessageInfo.Builder(receivedRequest.getBlobId(), stream.getSize(), properties.getAccountId(), properties.getContainerId(), properties.getCreationTimeInMs()).expirationTimeInMs(expirationTime).crc(receivedRequest.getCrc()).lifeVersion(MessageInfo.LIFE_VERSION_FROM_FRONTEND).build();
ArrayList<MessageInfo> infoList = new ArrayList<>();
infoList.add(info);
MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
Store storeToPut = storeManager.getStore(receivedRequest.getBlobId().getPartition());
storeToPut.put(writeset);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.No_Error);
metrics.blobSizeInBytes.update(receivedRequest.getBlobSize());
metrics.blobUserMetadataSizeInBytes.update(receivedRequest.getUsermetadata().limit());
if (notification != null) {
notification.onBlobReplicaCreated(currentNode.getHostname(), currentNode.getPort(), receivedRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
}
}
} catch (StoreException e) {
logger.error("Store exception on a put with error code {} for request {}", e.getErrorCode(), receivedRequest, e);
if (e.getErrorCode() == StoreErrorCodes.Already_Exist) {
metrics.idAlreadyExistError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.IOError) {
metrics.storeIOError.inc();
} else {
metrics.unExpectedStorePutError.inc();
}
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
} catch (Exception e) {
logger.error("Unknown exception on a put for request {}", receivedRequest, e);
response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.Unknown_Error);
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", receivedRequest, response, processingTime);
metrics.putBlobProcessingTimeInMs.update(processingTime);
metrics.updatePutBlobProcessingTimeBySize(receivedRequest.getBlobSize(), processingTime);
}
sendPutResponse(requestResponseChannel, response, request, metrics.putBlobResponseQueueTimeInMs, metrics.putBlobSendTimeInMs, metrics.putBlobTotalTimeInMs, totalTimeSpent, receivedRequest.getBlobSize(), metrics);
}
use of com.github.ambry.messageformat.PutMessageFormatInputStream in project ambry by linkedin.
the class CloudAndStoreReplicationTest method basicCloudRecoveryTest.
/**
* Test replication from vcr to server nodes, and from server to server nodes. Creates one vcr node and two server nodes.
* Uploads data to vcr node and verifies that they have been replicated.
* Uploads data to one of the server nodes and verifies that they have been replicated.
* @throws Exception If an exception happens.
*/
@Test
public void basicCloudRecoveryTest() throws Exception {
// Create blobs and upload to cloud destination.
Map<BlobId, Integer> blobIdToSizeMap = new HashMap<>();
for (BlobId blobId : cloudBlobIds) {
int blobSize = Utils.getRandomShort(TestUtils.RANDOM);
PutMessageFormatInputStream putMessageFormatInputStream = ServerTestUtil.getPutMessageInputStreamForBlob(blobId, blobSize, blobIdToSizeMap, accountId, containerId);
long time = System.currentTimeMillis();
CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, time, Utils.Infinite_Time, putMessageFormatInputStream.getSize(), CloudBlobMetadata.EncryptionOrigin.NONE);
latchBasedInMemoryCloudDestination.uploadBlob(blobId, putMessageFormatInputStream.getSize(), cloudBlobMetadata, putMessageFormatInputStream);
}
// Create blobs and upload to one of the server nodes.
sendBlobToDataNode(partitionLeaderRecoveryNode, Utils.getRandomShort(TestUtils.RANDOM), blobIdToSizeMap);
// Waiting for download attempt
assertTrue("Did not recover all blobs in 1 minute", latchBasedInMemoryCloudDestination.awaitDownload(1, TimeUnit.MINUTES));
// Waiting for replication to complete
Thread.sleep(10000);
// Test cloud to store and store to store replication by sending get request to server nodes.
testGetOnServerNodes(blobIdToSizeMap);
}
use of com.github.ambry.messageformat.PutMessageFormatInputStream in project ambry by linkedin.
the class ServerTestUtil method getPutMessageInputStreamForBlob.
/**
* Create {@link PutMessageFormatInputStream} for a blob with given {@link BlobId} and update {@code blobIdToSizeMap}.
* @param blobId {@link BlobId} object.
* @param blobSize size of blob.
* @param blobIdToSizeMap {@link Map} of {@link BlobId} to size of blob uploaded.
* @return {@link PutMessageFormatInputStream} object.
* @throws Exception
*/
static PutMessageFormatInputStream getPutMessageInputStreamForBlob(BlobId blobId, int blobSize, Map<BlobId, Integer> blobIdToSizeMap, short accountId, short containerId) throws Exception {
int userMetaDataSize = 100;
byte[] userMetadata = new byte[userMetaDataSize];
TestUtils.RANDOM.nextBytes(userMetadata);
byte[] data = new byte[blobSize];
BlobProperties blobProperties = new BlobProperties(blobSize, "serviceid1", null, null, false, Utils.Infinite_Time, accountId, containerId, false, null, null, null);
TestUtils.RANDOM.nextBytes(data);
blobIdToSizeMap.put(blobId, blobSize);
return new PutMessageFormatInputStream(blobId, null, blobProperties, ByteBuffer.wrap(userMetadata), new ByteBufferInputStream(ByteBuffer.wrap(data)), blobSize);
}
Aggregations