use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.
the class PutOperationTest method testSendIncomplete.
/**
* Ensure that if any of the requests associated with the buffer of a PutChunk is not completely read out even
* after the associated chunk is complete, the buffer is not reused even though the PutChunk is reused.
*/
@Test
public void testSendIncomplete() throws Exception {
int numChunks = NonBlockingRouter.MAX_IN_MEM_CHUNKS + 1;
BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false);
byte[] userMetadata = new byte[10];
byte[] content = new byte[chunkSize * numChunks];
random.nextBytes(content);
ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
FutureResult<String> future = new FutureResult<>();
MockNetworkClient mockNetworkClient = new MockNetworkClient();
PutOperation op = new PutOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, new LoggingNotificationSystem(), userMetadata, channel, future, null, new RouterCallback(mockNetworkClient, new ArrayList<BackgroundDeleteRequest>()), null, null, null, null, time, blobProperties);
op.startReadingFromChannel();
List<RequestInfo> requestInfos = new ArrayList<>();
requestRegistrationCallback.requestListToFill = requestInfos;
// Since this channel is in memory, one call to fill chunks would end up filling the maximum number of PutChunks.
op.fillChunks();
Assert.assertTrue("ReadyForPollCallback should have been invoked as chunks were fully filled", mockNetworkClient.getAndClearWokenUpStatus());
// A poll should therefore return requestParallelism number of requests from each chunk
op.poll(requestRegistrationCallback);
Assert.assertEquals(NonBlockingRouter.MAX_IN_MEM_CHUNKS * requestParallelism, requestInfos.size());
// There are MAX_IN_MEM_CHUNKS + 1 data chunks for this blob (and a metadata chunk).
// Once the first chunk is completely sent out, the first PutChunk will be reused. What the test verifies is that
// the buffer of the first PutChunk does not get reused. It does this as follows:
// For the first chunk,
// 1. use first request to succeed the chunk (the successTarget is set to 1).
// 2. read and store from the second for comparing later.
// 3. read from the third after the first PutChunk gets reused and ensure that the data from the third is the
// same as from what was saved off from the second. This means that the buffer was not reused by the first
// PutChunk.
// 1.
ResponseInfo responseInfo = getResponseInfo(requestInfos.get(0));
PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new DataInputStream(new ByteBufferInputStream(responseInfo.getResponse()))) : null;
op.handleResponse(responseInfo, putResponse);
// 2.
PutRequest putRequest = (PutRequest) requestInfos.get(1).getRequest();
ByteBuffer buf = ByteBuffer.allocate((int) putRequest.sizeInBytes());
ByteBufferChannel bufChannel = new ByteBufferChannel(buf);
// read it out (which also marks this request as complete).
putRequest.writeTo(bufChannel);
byte[] expectedRequestContent = buf.array();
// 3.
// first save the third request
PutRequest savedRequest = (PutRequest) requestInfos.get(2).getRequest();
// succeed all the other requests.
for (int i = 3; i < requestInfos.size(); i++) {
responseInfo = getResponseInfo(requestInfos.get(i));
putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new DataInputStream(new ByteBufferInputStream(responseInfo.getResponse()))) : null;
op.handleResponse(responseInfo, putResponse);
}
// fill the first PutChunk with the last chunk.
op.fillChunks();
// Verify that the last chunk was filled.
requestInfos.clear();
op.poll(requestRegistrationCallback);
Assert.assertEquals(1 * requestParallelism, requestInfos.size());
// Verify that the buffer of the third request is not affected.
buf = ByteBuffer.allocate((int) savedRequest.sizeInBytes());
bufChannel = new ByteBufferChannel(buf);
savedRequest.writeTo(bufChannel);
byte[] savedRequestContent = buf.array();
// reset the correlation id as they will be different between the two requests.
resetCorrelationId(expectedRequestContent);
resetCorrelationId(savedRequestContent);
Assert.assertArrayEquals("Underlying buffer should not have be reused", expectedRequestContent, savedRequestContent);
// internal to the chunk (though this can be verified via coverage).
for (int i = 0; i < requestInfos.size(); i++) {
responseInfo = getResponseInfo(requestInfos.get(i));
putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new DataInputStream(new ByteBufferInputStream(responseInfo.getResponse()))) : null;
op.handleResponse(responseInfo, putResponse);
}
requestInfos.clear();
// this should return requests for the metadata chunk
op.poll(requestRegistrationCallback);
Assert.assertEquals(1 * requestParallelism, requestInfos.size());
Assert.assertFalse("Operation should not be complete yet", op.isOperationComplete());
// once the metadata request succeeds, it should complete the operation.
responseInfo = getResponseInfo(requestInfos.get(0));
putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new DataInputStream(new ByteBufferInputStream(responseInfo.getResponse()))) : null;
op.handleResponse(responseInfo, putResponse);
Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
}
use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.
the class PutOperationTest method testSetOperationExceptionAndComplete.
/**
* Test the Errors {@link RouterErrorCode} received by Put Operation. The operation exception is set
* based on the priority of these errors.
* @throws Exception
*/
@Test
public void testSetOperationExceptionAndComplete() throws Exception {
int numChunks = NonBlockingRouter.MAX_IN_MEM_CHUNKS + 1;
BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false);
byte[] userMetadata = new byte[10];
byte[] content = new byte[chunkSize * numChunks];
random.nextBytes(content);
ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
FutureResult<String> future = new FutureResult<>();
MockNetworkClient mockNetworkClient = new MockNetworkClient();
PutOperation op = new PutOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, new LoggingNotificationSystem(), userMetadata, channel, future, null, new RouterCallback(mockNetworkClient, new ArrayList<BackgroundDeleteRequest>()), null, null, null, null, time, blobProperties);
RouterErrorCode[] routerErrorCodes = new RouterErrorCode[5];
routerErrorCodes[0] = RouterErrorCode.OperationTimedOut;
routerErrorCodes[1] = RouterErrorCode.UnexpectedInternalError;
routerErrorCodes[2] = RouterErrorCode.AmbryUnavailable;
routerErrorCodes[3] = RouterErrorCode.InsufficientCapacity;
routerErrorCodes[4] = RouterErrorCode.InvalidBlobId;
for (int i = 0; i < routerErrorCodes.length; ++i) {
op.setOperationExceptionAndComplete(new RouterException("RouterError", routerErrorCodes[i]));
Assert.assertEquals(((RouterException) op.getOperationException()).getErrorCode(), routerErrorCodes[i]);
}
for (int i = routerErrorCodes.length - 1; i >= 0; --i) {
op.setOperationExceptionAndComplete(new RouterException("RouterError", routerErrorCodes[i]));
Assert.assertEquals(((RouterException) op.getOperationException()).getErrorCode(), routerErrorCodes[routerErrorCodes.length - 1]);
}
Exception nonRouterException = new Exception();
op.setOperationExceptionAndComplete(nonRouterException);
Assert.assertEquals(nonRouterException, op.getOperationException());
// test edge case where current operationException is non RouterException
op.setOperationExceptionAndComplete(new RouterException("RouterError", RouterErrorCode.InsufficientCapacity));
Assert.assertEquals(((RouterException) op.getOperationException()).getErrorCode(), RouterErrorCode.InsufficientCapacity);
}
use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.
the class RouterFactoryTest method testRouterFactory.
/**
* Tests the instantiation of a {@link Router} implementation through its {@link RouterFactory} implementation
* @throws IOException
*/
@Test
public void testRouterFactory() throws Exception {
VerifiableProperties verifiableProperties = getVerifiableProperties();
List<FactoryAndRouter> factoryAndRouters = new ArrayList<FactoryAndRouter>();
factoryAndRouters.add(new FactoryAndRouter("com.github.ambry.router.NonBlockingRouterFactory", "com.github.ambry.router.NonBlockingRouter"));
for (FactoryAndRouter factoryAndRouter : factoryAndRouters) {
RouterFactory routerFactory = Utils.getObj(factoryAndRouter.factoryStr, verifiableProperties, new MockClusterMap(), new LoggingNotificationSystem(), null);
Router router = routerFactory.getRouter();
Assert.assertEquals("Did not receive expected Router instance", factoryAndRouter.routerStr, router.getClass().getCanonicalName());
router.close();
}
}
use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.
the class NonBlockingRouterTest method setRouter.
/**
* Initialize and set the router with the given {@link Properties} and {@link MockServerLayout}
* @param props the {@link Properties}
* @param mockServerLayout the {@link MockServerLayout}
*/
private void setRouter(Properties props, MockServerLayout mockServerLayout) throws IOException {
VerifiableProperties verifiableProperties = new VerifiableProperties((props));
routerMetrics = new NonBlockingRouterMetrics(mockClusterMap);
router = new NonBlockingRouter(new RouterConfig(verifiableProperties), routerMetrics, new MockNetworkClientFactory(verifiableProperties, null, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), new LoggingNotificationSystem(), mockClusterMap, kms, cryptoService, cryptoJobHandler, mockTime);
}
use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.
the class NonBlockingRouterTest method testCompositeBlobDataChunksDelete.
/**
* Test that if a composite blob is deleted, the data chunks are eventually deleted. Also check the service IDs used
* for delete operations.
*/
@Test
public void testCompositeBlobDataChunksDelete() throws Exception {
// Ensure there are 4 chunks.
maxPutChunkSize = PUT_CONTENT_SIZE / 4;
Properties props = getNonBlockingRouterProperties("DC1");
VerifiableProperties verifiableProperties = new VerifiableProperties((props));
RouterConfig routerConfig = new RouterConfig(verifiableProperties);
MockClusterMap mockClusterMap = new MockClusterMap();
MockTime mockTime = new MockTime();
MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
// metadata blob + data chunks.
final AtomicReference<CountDownLatch> deletesDoneLatch = new AtomicReference<>();
final Map<String, String> blobsThatAreDeleted = new HashMap<>();
LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {
@Override
public void onBlobDeleted(String blobId, String serviceId) {
blobsThatAreDeleted.put(blobId, serviceId);
deletesDoneLatch.get().countDown();
}
};
router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, mockTime);
setOperationParams();
String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel).get();
String deleteServiceId = "delete-service";
Set<String> blobsToBeDeleted = getBlobsInServers(mockServerLayout);
int getRequestCount = mockServerLayout.getCount(RequestOrResponseType.GetRequest);
// The third iteration is to test the case where the blob has expired.
for (int i = 0; i < 3; i++) {
if (i == 2) {
// Create a clean cluster and put another blob that immediate expires.
setOperationParams();
putBlobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false);
blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel).get();
Set<String> allBlobsInServer = getBlobsInServers(mockServerLayout);
allBlobsInServer.removeAll(blobsToBeDeleted);
blobsToBeDeleted = allBlobsInServer;
}
blobsThatAreDeleted.clear();
deletesDoneLatch.set(new CountDownLatch(5));
router.deleteBlob(blobId, deleteServiceId, null).get();
Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
Assert.assertTrue("All blobs in server are deleted", blobsThatAreDeleted.keySet().containsAll(blobsToBeDeleted));
Assert.assertTrue("Only blobs in server are deleted", blobsToBeDeleted.containsAll(blobsThatAreDeleted.keySet()));
for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
String expectedServiceId = blobIdAndServiceId.getKey().equals(blobId) ? deleteServiceId : BackgroundDeleteRequest.SERVICE_ID_PREFIX + deleteServiceId;
Assert.assertEquals("Unexpected service ID for deleted blob", expectedServiceId, blobIdAndServiceId.getValue());
}
// For 1 chunk deletion attempt, 1 background operation for Get is initiated which results in 2 Get Requests at
// the servers.
getRequestCount += 2;
Assert.assertEquals("Only one attempt of chunk deletion should have been done", getRequestCount, mockServerLayout.getCount(RequestOrResponseType.GetRequest));
}
deletesDoneLatch.set(new CountDownLatch(5));
router.deleteBlob(blobId, null, null).get();
Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
router.close();
assertClosed();
Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
}
Aggregations