Examples with RouterConfig com.github.ambry.config.RouterConfig used on opensource projects

Example 31 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class NonBlockingRouterTest method testRequestResponseHandlerThreadExitFlow.

/**
 * Test RequestResponseHandler thread exit flow. If the RequestResponseHandlerThread exits on its own (due to a
 * Throwable), then the router gets closed immediately along with the completion of all the operations.
 */
@Test
public void testRequestResponseHandlerThreadExitFlow() throws Exception {
    nettyByteBufLeakHelper.setDisabled(true);
    Properties props = getNonBlockingRouterProperties("DC1");
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    MockClusterMap mockClusterMap = new MockClusterMap();
    MockTime mockTime = new MockTime();
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, new MockServerLayout(mockClusterMap), mockTime), new LoggingNotificationSystem(), mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    assertExpectedThreadCounts(2, 1);
    setOperationParams();
    mockSelectorState.set(MockSelectorState.ThrowExceptionOnAllPoll);
    Future future = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build());
    try {
        while (!future.isDone()) {
            mockTime.sleep(1000);
            Thread.yield();
        }
        future.get();
        Assert.fail("The operation should have failed");
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.OperationTimedOut, ((RouterException) e.getCause()).getErrorCode());
    }
    setOperationParams();
    mockSelectorState.set(MockSelectorState.ThrowThrowableOnSend);
    future = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build());
    Thread requestResponseHandlerThreadRegular = TestUtils.getThreadByThisName("RequestResponseHandlerThread-0");
    Thread requestResponseHandlerThreadBackground = TestUtils.getThreadByThisName("RequestResponseHandlerThread-backgroundDeleter");
    if (requestResponseHandlerThreadRegular != null) {
        requestResponseHandlerThreadRegular.join(NonBlockingRouter.SHUTDOWN_WAIT_MS);
    }
    if (requestResponseHandlerThreadBackground != null) {
        requestResponseHandlerThreadBackground.join(NonBlockingRouter.SHUTDOWN_WAIT_MS);
    }
    try {
        future.get();
        Assert.fail("The operation should have failed");
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.RouterClosed, ((RouterException) e.getCause()).getErrorCode());
    }
    assertClosed();
    // Ensure that both operations failed and with the right exceptions.
    Assert.assertEquals("No ChunkFiller Thread should be running after the router is closed", 0, TestUtils.numThreadsByThisName("ChunkFillerThread"));
    Assert.assertEquals("No RequestResponseHandler should be running after the router is closed", 0, TestUtils.numThreadsByThisName("RequestResponseHandlerThread"));
    Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
}

Example 32 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class CloudRouterTest method setRouter.

/**
 * Initialize and set the router with the given {@link Properties} and {@link MockServerLayout}
 * @param props the {@link Properties}
 * @param notificationSystem the {@link NotificationSystem} to use.
 */
@Override
protected void setRouter(Properties props, MockServerLayout mockServerLayout, NotificationSystem notificationSystem) throws Exception {
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, mockClusterMap.getMetricRegistry(), mockClusterMap);
    CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
    AccountService accountService = new InMemAccountService(false, true);
    CloudRouterFactory cloudRouterFactory = new CloudRouterFactory(verifiableProperties, mockClusterMap, new LoggingNotificationSystem(), null, accountService);
    RequestHandlerPool requestHandlerPool = cloudRouterFactory.getRequestHandlerPool(verifiableProperties, mockClusterMap, cloudDestination, cloudConfig);
    Map<ReplicaType, NetworkClientFactory> childFactories = new EnumMap<>(ReplicaType.class);
    childFactories.put(ReplicaType.CLOUD_BACKED, new LocalNetworkClientFactory((LocalRequestResponseChannel) requestHandlerPool.getChannel(), new NetworkConfig(verifiableProperties), new NetworkMetrics(routerMetrics.getMetricRegistry()), mockTime));
    childFactories.put(ReplicaType.DISK_BACKED, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime));
    NetworkClientFactory networkClientFactory = new CompositeNetworkClientFactory(childFactories);
    router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    router.addResourceToClose(requestHandlerPool);
}

Example 33 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class OperationTrackerTest method operationClassTest.

/**
 * Test that operation tracker can correctly populate parameters(i.e. successTarget) based on input {@link RouterOperation}.
 */
@Test
public void operationClassTest() {
    Properties props = new Properties();
    props.setProperty("router.hostname", "localhost");
    props.setProperty("router.datacenter.name", "dc-0");
    props.setProperty("router.get.success.target", "1");
    props.setProperty("router.put.success.target", "2");
    props.setProperty("router.delete.success.target", "2");
    props.setProperty("router.ttl.update.success.target", "2");
    RouterConfig routerConfig = new RouterConfig(new VerifiableProperties(props));
    initialize();
    NonBlockingRouterMetrics routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    Map<RouterOperation, Integer> operationAndSuccessTarget = new HashMap<>();
    operationAndSuccessTarget.put(RouterOperation.GetBlobOperation, 1);
    operationAndSuccessTarget.put(RouterOperation.GetBlobInfoOperation, 1);
    operationAndSuccessTarget.put(RouterOperation.PutOperation, 2);
    operationAndSuccessTarget.put(RouterOperation.DeleteOperation, 2);
    operationAndSuccessTarget.put(RouterOperation.TtlUpdateOperation, 2);
    for (Map.Entry<RouterOperation, Integer> entry : operationAndSuccessTarget.entrySet()) {
        SimpleOperationTracker operationTracker = null;
        switch(operationTrackerType) {
            case SIMPLE_OP_TRACKER:
                operationTracker = new SimpleOperationTracker(routerConfig, entry.getKey(), mockPartition, originatingDcName, true, routerMetrics);
                break;
            case ADAPTIVE_OP_TRACKER:
                try {
                    operationTracker = new AdaptiveOperationTracker(routerConfig, routerMetrics, entry.getKey(), mockPartition, originatingDcName, time);
                } catch (IllegalArgumentException e) {
                    assertTrue("Get operation shouldn't throw any exception in adaptive tracker", entry.getKey() != RouterOperation.GetBlobOperation && entry.getKey() != RouterOperation.GetBlobInfoOperation);
                }
                break;
        }
        // ensure the success target matches the number specified for each type of operaiton
        if (operationTracker != null) {
            assertEquals("The suggest target doesn't match", (long) entry.getValue(), operationTracker.getSuccessTarget(ReplicaType.DISK_BACKED));
        }
    }
}

Example 34 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class AdaptiveOperationTrackerTest method invalidOperationTrackerScopeTest.

/**
 * Test that if metric scope in router config is invalid, the {@link IllegalArgumentException} is thrown explicitly.
 */
@Test
public void invalidOperationTrackerScopeTest() {
    Properties props = new Properties();
    props.setProperty("router.hostname", "localhost");
    props.setProperty("router.datacenter.name", localDcName);
    props.setProperty("router.operation.tracker.metric.scope", "Invalid Scope");
    props.setProperty("router.get.success.target", Integer.toString(1));
    props.setProperty("router.get.request.parallelism", Integer.toString(1));
    RouterConfig routerConfig = null;
    try {
        routerConfig = new RouterConfig(new VerifiableProperties(props));
    } catch (IllegalArgumentException e) {
        // exception is expected and set valid metric scope to instantiate routerConfig for subsequent test.
        props.setProperty("router.operation.tracker.metric.scope", "Datacenter");
        routerConfig = new RouterConfig(new VerifiableProperties(props));
    }
    NonBlockingRouterMetrics routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    AdaptiveOperationTracker tracker = new AdaptiveOperationTracker(routerConfig, routerMetrics, RouterOperation.GetBlobInfoOperation, mockPartition, null, time);
    // test that operation tracker works correctly with default Datacenter scope
    sendRequests(tracker, 1);
    assertFalse("Operation should not be done", tracker.isDone());
    tracker.onResponse(partitionAndInflightReplicas.get(mockPartition).poll(), TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should have succeeded", tracker.hasSucceeded());
    // test that no other resource-level metrics have been instantiated.
    assertTrue("Partition histogram in RouterMetrics should be empty", routerMetrics.getBlobInfoLocalDcResourceToLatency.isEmpty());
    assertTrue("Partition histogram in RouterMetrics should be empty", routerMetrics.getBlobInfoCrossDcResourceToLatency.isEmpty());
    assertTrue("Partition histogram in OperationTracker should be empty", tracker.getResourceToLatencyMap(RouterOperation.GetBlobInfoOperation, true).isEmpty());
    assertTrue("Partition histogram in OperationTracker should be empty", tracker.getResourceToLatencyMap(RouterOperation.GetBlobInfoOperation, false).isEmpty());
    // extra test: invalid router operation
    try {
        tracker.getResourceToLatencyMap(RouterOperation.TtlUpdateOperation, true);
        fail("should fail due to invalid router operation");
    } catch (IllegalArgumentException e) {
    // expected
    }
}

Example 35 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class ChunkFillTest method testChunkNumAndSizeCalculations.

/**
 * Test the calculation of number of chunks and the size of each chunk, using a very large blob size. No content
 * comparison is done. This test does not consume memory more than chunkSize.
 */
@Test
public void testChunkNumAndSizeCalculations() throws Exception {
    chunkSize = 4 * 1024 * 1024;
    // a large blob greater than Integer.MAX_VALUE and not at chunk size boundary.
    final long blobSize = ((long) Integer.MAX_VALUE / chunkSize + 1) * chunkSize + random.nextInt(chunkSize - 1) + 1;
    VerifiableProperties vProps = getNonBlockingRouterProperties();
    MockClusterMap mockClusterMap = new MockClusterMap();
    RouterConfig routerConfig = new RouterConfig(vProps);
    NonBlockingRouterMetrics routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    BlobProperties putBlobProperties = new BlobProperties(blobSize, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, accountId, containerId, false, null, null, null);
    Random random = new Random();
    byte[] putUserMetadata = new byte[10];
    random.nextBytes(putUserMetadata);
    final MockReadableStreamChannel putChannel = new MockReadableStreamChannel(blobSize, false);
    FutureResult<String> futureResult = new FutureResult<String>();
    MockTime time = new MockTime();
    MockNetworkClientFactory networkClientFactory = new MockNetworkClientFactory(vProps, null, 0, 0, 0, null, time);
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), putUserMetadata, putChannel, PutBlobOptions.DEFAULT, futureResult, null, new RouterCallback(networkClientFactory.getNetworkClient(), new ArrayList<>()), null, null, null, null, new MockTime(), putBlobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    numChunks = RouterUtils.getNumChunksForBlobAndChunkSize(blobSize, chunkSize);
    // largeBlobSize is not a multiple of chunkSize
    int expectedNumChunks = (int) (blobSize / chunkSize + 1);
    Assert.assertEquals("numChunks should be as expected", expectedNumChunks, numChunks);
    int lastChunkSize = (int) (blobSize % chunkSize);
    final AtomicReference<Exception> channelException = new AtomicReference<Exception>(null);
    int chunkIndex = 0;
    // The write to the MockReadableStreamChannel blocks until the data is read as part fo the chunk filling,
    // so create a thread that fills the MockReadableStreamChannel.
    Utils.newThread(new Runnable() {

        @Override
        public void run() {
            try {
                byte[] writeBuf = new byte[chunkSize];
                long written = 0;
                while (written < blobSize) {
                    int toWrite = (int) Math.min(chunkSize, blobSize - written);
                    putChannel.write(ByteBuffer.wrap(writeBuf, 0, toWrite));
                    written += toWrite;
                }
            } catch (Exception e) {
                channelException.set(e);
            }
        }
    }, false).start();
    // Do the chunk filling.
    boolean fillingComplete = false;
    do {
        op.fillChunks();
        // since the channel is ByteBuffer based.
        for (PutOperation.PutChunk putChunk : op.putChunks) {
            Assert.assertNull("Mock channel write should not have caused an exception", channelException.get());
            if (putChunk.isFree()) {
                continue;
            }
            if (chunkIndex == numChunks - 1) {
                // last chunk may not be Ready as it is dependent on the completion callback to be called.
                Assert.assertTrue("Chunk should be Building or Ready.", putChunk.getState() == PutOperation.ChunkState.Ready || putChunk.getState() == PutOperation.ChunkState.Building);
                if (putChunk.getState() == PutOperation.ChunkState.Ready) {
                    Assert.assertEquals("Chunk size should be the last chunk size", lastChunkSize, putChunk.buf.readableBytes());
                    Assert.assertTrue("Chunk Filling should be complete at this time", op.isChunkFillingDone());
                    fillingComplete = true;
                }
            } else {
                // if not last chunk, then the chunk should be full and Ready.
                Assert.assertEquals("Chunk should be ready.", PutOperation.ChunkState.Ready, putChunk.getState());
                Assert.assertEquals("Chunk size should be maxChunkSize", chunkSize, putChunk.buf.readableBytes());
                chunkIndex++;
                putChunk.clear();
            }
        }
    } while (!fillingComplete);
}