Examples with RouterConfig com.github.ambry.config.RouterConfig used on opensource projects

Example 11 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class NonBlockingRouterTest method testUnsuccessfulPutDataChunkDelete.

/**
 * Test that if a composite blob put fails, the successfully put data chunks are deleted.
 */
@Test
public void testUnsuccessfulPutDataChunkDelete() throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
        // fails, it uses a notification system to track the deletions.
        final CountDownLatch deletesDoneLatch = new CountDownLatch(2);
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.countDown();
            }
        };
        router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
        List<ServerErrorCode> serverErrorList = new ArrayList<>();
        // There are 4 chunks for this blob.
        // All put operations make one request to each local server as there are 3 servers overall in the local DC.
        // Set the state of the mock servers so that they return success for the first 2 requests in order to succeed
        // the first two chunks.
        serverErrorList.add(ServerErrorCode.No_Error);
        serverErrorList.add(ServerErrorCode.No_Error);
        // fail requests for third and fourth data chunks including the slipped put attempts:
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        // all subsequent requests (no more puts, but there will be deletes) will succeed.
        for (DataNodeId dataNodeId : dataNodeIds) {
            MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
            server.setServerErrors(serverErrorList);
        }
        // Submit the put operation and wait for it to fail.
        try {
            router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        } catch (ExecutionException e) {
            Assert.assertEquals(RouterErrorCode.AmbryUnavailable, ((RouterException) e.getCause()).getErrorCode());
        }
        // Now, wait until the deletes of the successfully put blobs are complete.
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
            Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
        }
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}

Example 12 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class DeleteManagerTest method testVariousServerErrorCodesForThreeParallelism.

/**
 * The parallelism is set to 3 not 9.
 *
 * Test the case where servers return different {@link ServerErrorCode}, and the {@link DeleteOperation}
 * is able to resolve and conclude the correct {@link RouterErrorCode}. The {link ServerErrorCode} tested
 * are those could be mapped to {@link RouterErrorCode#AmbryUnavailable}. The order of received responses
 * is the same as defined in {@code serverErrorCodes}.
 */
@Test
public void testVariousServerErrorCodesForThreeParallelism() throws Exception {
    assertCloseCleanup(router);
    Properties props = getNonBlockingRouterProperties();
    props.setProperty("router.delete.request.parallelism", "3");
    VerifiableProperties vProps = new VerifiableProperties(props);
    RouterConfig routerConfig = new RouterConfig(vProps);
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(clusterMap, routerConfig), new MockNetworkClientFactory(vProps, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, serverLayout, mockTime), new LoggingNotificationSystem(), clusterMap, null, null, null, new InMemAccountService(false, true), mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    ServerErrorCode[] serverErrorCodes = new ServerErrorCode[9];
    serverErrorCodes[0] = ServerErrorCode.Blob_Not_Found;
    serverErrorCodes[1] = ServerErrorCode.Data_Corrupt;
    serverErrorCodes[2] = ServerErrorCode.IO_Error;
    serverErrorCodes[3] = ServerErrorCode.Partition_Unknown;
    serverErrorCodes[4] = ServerErrorCode.Disk_Unavailable;
    serverErrorCodes[5] = ServerErrorCode.No_Error;
    serverErrorCodes[6] = ServerErrorCode.Data_Corrupt;
    serverErrorCodes[7] = ServerErrorCode.Unknown_Error;
    serverErrorCodes[8] = ServerErrorCode.Disk_Unavailable;
    testWithErrorCodes(serverErrorCodes, partition, serverLayout, RouterErrorCode.AmbryUnavailable, deleteErrorCodeChecker);
}

Example 13 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class DeleteManagerTest method init.

/**
 * Initializes ClusterMap, Router, mock servers, and an {@code BlobId} to be deleted.
 */
@Before
public void init() throws Exception {
    VerifiableProperties vProps = new VerifiableProperties(getNonBlockingRouterProperties());
    mockTime = new MockTime();
    mockSelectorState = new AtomicReference<MockSelectorState>(MockSelectorState.Good);
    clusterMap = new MockClusterMap();
    serverLayout = new MockServerLayout(clusterMap);
    RouterConfig routerConfig = new RouterConfig(vProps);
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(clusterMap, routerConfig), new MockNetworkClientFactory(vProps, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, serverLayout, mockTime), new LoggingNotificationSystem(), clusterMap, null, null, null, new InMemAccountService(false, true), mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    List<PartitionId> mockPartitions = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
    partition = mockPartitions.get(ThreadLocalRandom.current().nextInt(mockPartitions.size()));
    blobId = new BlobId(routerConfig.routerBlobidCurrentVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), partition, false, BlobId.BlobDataType.DATACHUNK);
    blobIdString = blobId.getID();
}

Example 14 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class DeleteManagerTest method testOriginDcNotFoundError.

/**
 * Test the case when getting NOT_FOUND error from origin DC while termination on NOT_FOUND is enabled.
 */
@Test
public void testOriginDcNotFoundError() throws Exception {
    assertCloseCleanup(router);
    Properties props = getNonBlockingRouterProperties();
    props.setProperty("router.delete.request.parallelism", "1");
    props.setProperty("router.operation.tracker.terminate.on.not.found.enabled", "true");
    VerifiableProperties vProps = new VerifiableProperties(props);
    RouterConfig routerConfig = new RouterConfig(vProps);
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(clusterMap, routerConfig), new MockNetworkClientFactory(vProps, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, serverLayout, mockTime), new LoggingNotificationSystem(), clusterMap, null, null, null, new InMemAccountService(false, true), mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    blobId = new BlobId(routerConfig.routerBlobidCurrentVersion, BlobId.BlobIdType.NATIVE, (byte) 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), partition, false, BlobId.BlobDataType.DATACHUNK);
    blobIdString = blobId.getID();
    ServerErrorCode[] serverErrorCodes = new ServerErrorCode[9];
    Arrays.fill(serverErrorCodes, ServerErrorCode.No_Error);
    serverErrorCodes[0] = ServerErrorCode.Blob_Not_Found;
    serverErrorCodes[1] = ServerErrorCode.Blob_Not_Found;
    serverErrorCodes[2] = ServerErrorCode.Blob_Not_Found;
    // The first two responses are blob not found and they are from the local dc and originating dc.
    // So even if the rest of servers returns No_Error, router will not send any requests to them.
    testWithErrorCodes(serverErrorCodes, partition, serverLayout, RouterErrorCode.BlobDoesNotExist, deleteErrorCodeChecker);
}

Example 15 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class MockRouterCallback method testInstantiation.

/**
 * Test {@link GetBlobInfoOperation} instantiation and validate the get methods.
 */
@Test
public void testInstantiation() {
    BlobId blobId = new BlobId(routerConfig.routerBlobidCurrentVersion, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(random), Utils.getRandomShort(random), mockClusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0), false, BlobId.BlobDataType.DATACHUNK);
    Callback<GetBlobResultInternal> getOperationCallback = (result, exception) -> {
    // no op.
    };
    // test a good case
    GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, getOperationCallback, routerCallback, kms, cryptoService, cryptoJobHandler, time, false, quotaChargeCallback);
    Assert.assertEquals("Callback must match", getOperationCallback, op.getCallback());
    Assert.assertEquals("Blob ids must match", blobId.getID(), op.getBlobIdStr());
    // test the case where the tracker type is bad
    Properties properties = getNonBlockingRouterProperties(true);
    properties.setProperty("router.get.operation.tracker.type", "NonExistentTracker");
    RouterConfig badConfig = new RouterConfig(new VerifiableProperties(properties));
    try {
        new GetBlobInfoOperation(badConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, getOperationCallback, routerCallback, kms, cryptoService, cryptoJobHandler, time, false, quotaChargeCallback);
        Assert.fail("Instantiation of GetBlobInfoOperation with an invalid tracker type must fail");
    } catch (IllegalArgumentException e) {
    // expected. Nothing to do.
    }
}