use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.
the class GetBlobOperationTest method testRequestTimeoutAndBlobNotFoundLocalTimeout.
/**
* Test the case where 2 local replicas timed out. The remaining one local replica and rest remote replicas respond
* with Blob_Not_Found.
* @throws Exception
*/
@Test
public void testRequestTimeoutAndBlobNotFoundLocalTimeout() throws Exception {
assumeTrue(operationTrackerType.equals(AdaptiveOperationTracker.class.getSimpleName()));
doPut();
GetBlobOperation op = createOperation(routerConfig, null);
AdaptiveOperationTracker tracker = (AdaptiveOperationTracker) op.getFirstChunkOperationTrackerInUse();
correlationIdToGetOperation.clear();
for (MockServer server : mockServerLayout.getMockServers()) {
server.setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
}
op.poll(requestRegistrationCallback);
time.sleep(routerConfig.routerRequestTimeoutMs + 1);
// The request should have response from one local replica and all remote replicas.
while (!op.isOperationComplete()) {
op.poll(requestRegistrationCallback);
List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
for (ResponseInfo responseInfo : responses) {
GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
op.handleResponse(responseInfo, getResponse);
responseInfo.release();
}
}
RouterException routerException = (RouterException) op.getOperationException();
// error code should be OperationTimedOut because it precedes BlobDoesNotExist
Assert.assertEquals(RouterErrorCode.OperationTimedOut, routerException.getErrorCode());
Histogram localColoTracker = tracker.getLatencyHistogram(RouterTestHelpers.getAnyReplica(blobId, true, localDcName));
Histogram crossColoTracker = tracker.getLatencyHistogram(RouterTestHelpers.getAnyReplica(blobId, false, localDcName));
// the count of data points in local colo Histogram should be 1, because first 2 request timed out
Assert.assertEquals("The number of data points in local colo latency histogram is not expected", 1, localColoTracker.getCount());
// the count of data points in cross colo Histogram should be 6 because all remote replicas respond with proper error code
Assert.assertEquals("The number of data points in cross colo latency histogram is not expected", 6, crossColoTracker.getCount());
}
use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.
the class GetBlobOperationTest method createOperationAndComplete.
/**
* Create a getBlob operation with the specified callback and poll until completion.
* @param callback the callback to run after completion of the operation, or {@code null} if no callback.
* @return the operation
* @throws Exception
*/
private GetBlobOperation createOperationAndComplete(Callback<GetBlobResultInternal> callback) throws Exception {
GetBlobOperation op = createOperation(routerConfig, callback);
while (!op.isOperationComplete()) {
op.poll(requestRegistrationCallback);
List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
for (ResponseInfo responseInfo : responses) {
DataInputStream dis = new NettyByteBufDataInputStream(responseInfo.content());
GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(dis, mockClusterMap) : null;
op.handleResponse(responseInfo, getResponse);
responseInfo.release();
}
}
return op;
}
use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.
the class MockRouterCallback method completeOp.
/**
* Completes {@code op}
* @param op the {@link GetBlobInfoOperation} to complete
* @throws IOException
*/
private void completeOp(GetBlobInfoOperation op) throws IOException {
while (!op.isOperationComplete()) {
op.poll(requestRegistrationCallback);
List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
for (ResponseInfo responseInfo : responses) {
GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
op.handleResponse(responseInfo, getResponse);
if (op.isOperationComplete()) {
break;
}
}
responses.forEach(ResponseInfo::release);
}
}
use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.
the class MockRouterCallback method testPollAndResponseHandling.
/**
* Test basic successful operation completion, by polling and handing over responses to the BlobInfo operation.
* @throws Exception
*/
@Test
public void testPollAndResponseHandling() throws Exception {
for (short expectedLifeVersion : new short[] { 0, 1 }) {
NonBlockingRouter.currentOperationsCount.incrementAndGet();
// Now set the lifeVersion
for (MockServer mockServer : mockServerLayout.getMockServers()) {
if (mockServer.getBlobs().containsKey(blobId.getID())) {
mockServer.getBlobs().get(blobId.getID()).lifeVersion = expectedLifeVersion;
}
}
GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, null, routerCallback, kms, cryptoService, cryptoJobHandler, time, false, quotaChargeCallback);
ArrayList<RequestInfo> requestListToFill = new ArrayList<>();
requestRegistrationCallback.setRequestsToSend(requestListToFill);
op.poll(requestRegistrationCallback);
Assert.assertEquals("There should only be as many requests at this point as requestParallelism", requestParallelism, correlationIdToGetOperation.size());
CountDownLatch onPollLatch = new CountDownLatch(1);
if (testEncryption) {
routerCallback.setOnPollLatch(onPollLatch);
}
List<ResponseInfo> responses = sendAndWaitForResponses(requestListToFill);
for (ResponseInfo responseInfo : responses) {
GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
op.handleResponse(responseInfo, getResponse);
if (op.isOperationComplete()) {
break;
}
}
responses.forEach(ResponseInfo::release);
if (testEncryption) {
Assert.assertTrue("Latch should have been zeroed ", onPollLatch.await(500, TimeUnit.MILLISECONDS));
op.poll(requestRegistrationCallback);
}
Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
assertSuccess(op, expectedLifeVersion);
// poll again to make sure that counters aren't triggered again (check in @After)
op.poll(requestRegistrationCallback);
correlationIdToGetOperation.clear();
}
}
use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.
the class PutOperationTest method testSendIncomplete.
/**
* Ensure that if any of the requests associated with the buffer of a PutChunk is not completely read out even
* after the associated chunk is complete, the buffer is not reused even though the PutChunk is reused.
*/
@Test
public void testSendIncomplete() throws Exception {
int numChunks = routerConfig.routerMaxInMemPutChunks + 1;
BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
byte[] userMetadata = new byte[10];
byte[] content = new byte[chunkSize * numChunks];
random.nextBytes(content);
ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
FutureResult<String> future = new FutureResult<>();
MockNetworkClient mockNetworkClient = new MockNetworkClient();
PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, channel, PutBlobOptions.DEFAULT, future, null, new RouterCallback(mockNetworkClient, new ArrayList<>()), null, null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
op.startOperation();
List<RequestInfo> requestInfos = new ArrayList<>();
requestRegistrationCallback.setRequestsToSend(requestInfos);
// Since this channel is in memory, one call to fill chunks would end up filling the maximum number of PutChunks.
op.fillChunks();
Assert.assertTrue("ReadyForPollCallback should have been invoked as chunks were fully filled", mockNetworkClient.getAndClearWokenUpStatus());
// A poll should therefore return requestParallelism number of requests from each chunk
op.poll(requestRegistrationCallback);
Assert.assertEquals(routerConfig.routerMaxInMemPutChunks * requestParallelism, requestInfos.size());
// There are routerMaxInMemPutChunks + 1 data chunks for this blob (and a metadata chunk).
// Once the first chunk is completely sent out, the first PutChunk will be reused. What the test verifies is that
// the buffer of the first PutChunk does not get reused. It does this as follows:
// For the first chunk,
// 1. use first request to succeed the chunk (the successTarget is set to 1).
// 2. read and store from the second for comparing later.
// 3. read from the third after the first PutChunk gets reused and ensure that the data from the third is the
// same as from what was saved off from the second. This means that the buffer was not reused by the first
// PutChunk.
// 1.
ResponseInfo responseInfo = getResponseInfo(requestInfos.get(0));
PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
op.handleResponse(responseInfo, putResponse);
requestInfos.get(0).getRequest().release();
responseInfo.release();
// 2.
PutRequest putRequest = (PutRequest) requestInfos.get(1).getRequest();
ByteBuffer buf = ByteBuffer.allocate((int) putRequest.sizeInBytes());
ByteBufferChannel bufChannel = new ByteBufferChannel(buf);
// read it out (which also marks this request as complete).
putRequest.writeTo(bufChannel);
putRequest.release();
byte[] expectedRequestContent = buf.array();
// 3.
// first save the third request
PutRequest savedRequest = (PutRequest) requestInfos.get(2).getRequest();
// succeed all the other requests.
for (int i = 3; i < requestInfos.size(); i++) {
responseInfo = getResponseInfo(requestInfos.get(i));
putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
op.handleResponse(responseInfo, putResponse);
requestInfos.get(i).getRequest().release();
responseInfo.release();
}
// fill the first PutChunk with the last chunk.
op.fillChunks();
// Verify that the last chunk was filled.
requestInfos.clear();
op.poll(requestRegistrationCallback);
Assert.assertEquals(1 * requestParallelism, requestInfos.size());
// Verify that the buffer of the third request is not affected.
buf = ByteBuffer.allocate((int) savedRequest.sizeInBytes());
bufChannel = new ByteBufferChannel(buf);
savedRequest.writeTo(bufChannel);
savedRequest.release();
byte[] savedRequestContent = buf.array();
// reset the correlation id as they will be different between the two requests.
resetCorrelationId(expectedRequestContent);
resetCorrelationId(savedRequestContent);
Assert.assertArrayEquals("Underlying buffer should not have be reused", expectedRequestContent, savedRequestContent);
// internal to the chunk (though this can be verified via coverage).
for (int i = 0; i < requestInfos.size(); i++) {
responseInfo = getResponseInfo(requestInfos.get(i));
putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
op.handleResponse(responseInfo, putResponse);
requestInfos.get(i).getRequest().release();
responseInfo.release();
}
requestInfos.clear();
// this should return requests for the metadata chunk
op.poll(requestRegistrationCallback);
Assert.assertEquals(1 * requestParallelism, requestInfos.size());
Assert.assertFalse("Operation should not be complete yet", op.isOperationComplete());
// once the metadata request succeeds, it should complete the operation.
responseInfo = getResponseInfo(requestInfos.get(0));
putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
op.handleResponse(responseInfo, putResponse);
responseInfo.release();
requestInfos.forEach(info -> info.getRequest().release());
Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
}
Aggregations