Search in sources :

Example 21 with LoggingNotificationSystem

use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.

the class ChunkFillTest method testChunkNumAndSizeCalculations.

/**
 * Test the calculation of number of chunks and the size of each chunk, using a very large blob size. No content
 * comparison is done. This test does not consume memory more than chunkSize.
 */
@Test
public void testChunkNumAndSizeCalculations() throws Exception {
    chunkSize = 4 * 1024 * 1024;
    // a large blob greater than Integer.MAX_VALUE and not at chunk size boundary.
    final long blobSize = ((long) Integer.MAX_VALUE / chunkSize + 1) * chunkSize + random.nextInt(chunkSize - 1) + 1;
    VerifiableProperties vProps = getNonBlockingRouterProperties();
    MockClusterMap mockClusterMap = new MockClusterMap();
    RouterConfig routerConfig = new RouterConfig(vProps);
    NonBlockingRouterMetrics routerMetrics = new NonBlockingRouterMetrics(mockClusterMap);
    ResponseHandler responseHandler = new ResponseHandler(mockClusterMap);
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    BlobProperties putBlobProperties = new BlobProperties(blobSize, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, accountId, containerId, false);
    Random random = new Random();
    byte[] putUserMetadata = new byte[10];
    random.nextBytes(putUserMetadata);
    final MockReadableStreamChannel putChannel = new MockReadableStreamChannel(blobSize, false);
    FutureResult<String> futureResult = new FutureResult<String>();
    MockTime time = new MockTime();
    MockNetworkClientFactory networkClientFactory = new MockNetworkClientFactory(vProps, null, 0, 0, 0, null, time);
    PutOperation op = new PutOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, new LoggingNotificationSystem(), putUserMetadata, putChannel, futureResult, null, new RouterCallback(networkClientFactory.getNetworkClient(), new ArrayList<BackgroundDeleteRequest>()), null, null, null, null, new MockTime(), putBlobProperties);
    op.startReadingFromChannel();
    numChunks = RouterUtils.getNumChunksForBlobAndChunkSize(blobSize, chunkSize);
    // largeBlobSize is not a multiple of chunkSize
    int expectedNumChunks = (int) (blobSize / chunkSize + 1);
    Assert.assertEquals("numChunks should be as expected", expectedNumChunks, numChunks);
    int lastChunkSize = (int) (blobSize % chunkSize);
    final AtomicReference<Exception> channelException = new AtomicReference<Exception>(null);
    int chunkIndex = 0;
    // The write to the MockReadableStreamChannel blocks until the data is read as part fo the chunk filling,
    // so create a thread that fills the MockReadableStreamChannel.
    Utils.newThread(new Runnable() {

        @Override
        public void run() {
            try {
                byte[] writeBuf = new byte[chunkSize];
                long written = 0;
                while (written < blobSize) {
                    int toWrite = (int) Math.min(chunkSize, blobSize - written);
                    putChannel.write(ByteBuffer.wrap(writeBuf, 0, toWrite));
                    written += toWrite;
                }
            } catch (Exception e) {
                channelException.set(e);
            }
        }
    }, false).start();
    // Do the chunk filling.
    boolean fillingComplete = false;
    do {
        op.fillChunks();
        // since the channel is ByteBuffer based.
        for (PutOperation.PutChunk putChunk : op.putChunks) {
            Assert.assertNull("Mock channel write should not have caused an exception", channelException.get());
            if (putChunk.isFree()) {
                continue;
            }
            if (chunkIndex == numChunks - 1) {
                // last chunk may not be Ready as it is dependent on the completion callback to be called.
                Assert.assertTrue("Chunk should be Building or Ready.", putChunk.getState() == PutOperation.ChunkState.Ready || putChunk.getState() == PutOperation.ChunkState.Building);
                if (putChunk.getState() == PutOperation.ChunkState.Ready) {
                    Assert.assertEquals("Chunk size should be the last chunk size", lastChunkSize, putChunk.buf.remaining());
                    Assert.assertTrue("Chunk Filling should be complete at this time", op.isChunkFillingDone());
                    fillingComplete = true;
                }
            } else {
                // if not last chunk, then the chunk should be full and Ready.
                Assert.assertEquals("Chunk should be ready.", PutOperation.ChunkState.Ready, putChunk.getState());
                Assert.assertEquals("Chunk size should be maxChunkSize", chunkSize, putChunk.buf.remaining());
                chunkIndex++;
                putChunk.clear();
            }
        }
    } while (!fillingComplete);
}
Also used : ResponseHandler(com.github.ambry.commons.ResponseHandler) ArrayList(java.util.ArrayList) Random(java.util.Random) MockTime(com.github.ambry.utils.MockTime) VerifiableProperties(com.github.ambry.config.VerifiableProperties) AtomicReference(java.util.concurrent.atomic.AtomicReference) RouterConfig(com.github.ambry.config.RouterConfig) IOException(java.io.IOException) LoggingNotificationSystem(com.github.ambry.commons.LoggingNotificationSystem) BlobProperties(com.github.ambry.messageformat.BlobProperties) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Example 22 with LoggingNotificationSystem

use of com.github.ambry.commons.LoggingNotificationSystem in project ambry by linkedin.

the class FrontendIntegrationTest method setup.

/**
 * Sets up an Ambry frontend server.
 * @throws Exception
 */
@BeforeClass
public static void setup() throws Exception {
    ambryRestServer = new RestServer(FRONTEND_VERIFIABLE_PROPS, CLUSTER_MAP, new LoggingNotificationSystem(), new SSLFactory(new SSLConfig(FRONTEND_VERIFIABLE_PROPS)));
    ambryRestServer.start();
    plaintextNettyClient = new NettyClient("localhost", PLAINTEXT_SERVER_PORT, null);
    sslNettyClient = new NettyClient("localhost", SSL_SERVER_PORT, new SSLFactory(new SSLConfig(SSL_CLIENT_VERIFIABLE_PROPS)));
}
Also used : RestServer(com.github.ambry.rest.RestServer) SSLConfig(com.github.ambry.config.SSLConfig) NettyClient(com.github.ambry.rest.NettyClient) SSLFactory(com.github.ambry.commons.SSLFactory) LoggingNotificationSystem(com.github.ambry.commons.LoggingNotificationSystem) BeforeClass(org.junit.BeforeClass)

Aggregations

LoggingNotificationSystem (com.github.ambry.commons.LoggingNotificationSystem)22 VerifiableProperties (com.github.ambry.config.VerifiableProperties)19 Properties (java.util.Properties)14 Test (org.junit.Test)14 MockClusterMap (com.github.ambry.clustermap.MockClusterMap)13 RouterConfig (com.github.ambry.config.RouterConfig)11 BlobProperties (com.github.ambry.messageformat.BlobProperties)11 MockTime (com.github.ambry.utils.MockTime)7 ArrayList (java.util.ArrayList)6 ClusterMap (com.github.ambry.clustermap.ClusterMap)5 NotificationSystem (com.github.ambry.notification.NotificationSystem)4 IOException (java.io.IOException)4 AtomicReference (java.util.concurrent.atomic.AtomicReference)4 ByteBufferReadableStreamChannel (com.github.ambry.commons.ByteBufferReadableStreamChannel)3 ResponseHandler (com.github.ambry.commons.ResponseHandler)3 ServerErrorCode (com.github.ambry.commons.ServerErrorCode)3 CountDownLatch (java.util.concurrent.CountDownLatch)3 BlobId (com.github.ambry.commons.BlobId)2 SSLFactory (com.github.ambry.commons.SSLFactory)2 KMSConfig (com.github.ambry.config.KMSConfig)2