Examples with VerifiableProperties com.github.ambry.config.VerifiableProperties used on opensource projects

Example 16 with VerifiableProperties

use of com.github.ambry.config.VerifiableProperties in project ambry by linkedin.

the class MockRouterCallback method testInstantiation.

/**
 * Test {@link GetBlobInfoOperation} instantiation and validate the get methods.
 * @throws Exception
 */
@Test
public void testInstantiation() throws Exception {
    BlobId blobId = new BlobId(routerConfig.routerBlobidCurrentVersion, BlobId.BlobIdType.NATIVE, ClusterMapUtils.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(random), Utils.getRandomShort(random), mockClusterMap.getWritablePartitionIds().get(0), false);
    Callback<GetBlobResultInternal> getOperationCallback = (result, exception) -> {
    // no op.
    };
    // test a good case
    GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, getOperationCallback, routerCallback, kms, cryptoService, cryptoJobHandler, time);
    Assert.assertEquals("Callback must match", getOperationCallback, op.getCallback());
    Assert.assertEquals("Blob ids must match", blobId.getID(), op.getBlobIdStr());
    // test the case where the tracker type is bad
    Properties properties = getNonBlockingRouterProperties();
    properties.setProperty("router.get.operation.tracker.type", "NonExistentTracker");
    RouterConfig badConfig = new RouterConfig(new VerifiableProperties(properties));
    try {
        new GetBlobInfoOperation(badConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, getOperationCallback, routerCallback, kms, cryptoService, cryptoJobHandler, time);
        Assert.fail("Instantiation of GetBlobInfoOperation with an invalid tracker type must fail");
    } catch (IllegalArgumentException e) {
    // expected. Nothing to do.
    }
}

Example 17 with VerifiableProperties

use of com.github.ambry.config.VerifiableProperties in project ambry by linkedin.

the class GetManagerTest method getNonBlockingRouter.

/**
 * @return Return a {@link NonBlockingRouter} created with default {@link VerifiableProperties}
 */
private NonBlockingRouter getNonBlockingRouter() throws IOException {
    Properties properties = new Properties();
    properties.setProperty("router.hostname", "localhost");
    properties.setProperty("router.datacenter.name", "DC1");
    properties.setProperty("router.max.put.chunk.size.bytes", Integer.toString(chunkSize));
    properties.setProperty("router.put.request.parallelism", Integer.toString(requestParallelism));
    properties.setProperty("router.put.success.target", Integer.toString(successTarget));
    VerifiableProperties vProps = new VerifiableProperties(properties);
    routerConfig = new RouterConfig(vProps);
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap), new MockNetworkClientFactory(vProps, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), new LoggingNotificationSystem(), mockClusterMap, kms, cryptoService, cryptoJobHandler, mockTime);
    return router;
}

Example 18 with VerifiableProperties

use of com.github.ambry.config.VerifiableProperties in project ambry by linkedin.

the class NonBlockingRouterTest method testResponseHandling.

/**
 * Response handling related tests for all operation managers.
 */
@Test
public void testResponseHandling() throws Exception {
    Properties props = getNonBlockingRouterProperties("DC1");
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    setOperationParams();
    final List<ReplicaId> failedReplicaIds = new ArrayList<>();
    final AtomicInteger successfulResponseCount = new AtomicInteger(0);
    final AtomicBoolean invalidResponse = new AtomicBoolean(false);
    ResponseHandler mockResponseHandler = new ResponseHandler(mockClusterMap) {

        @Override
        public void onEvent(ReplicaId replicaId, Object e) {
            if (e instanceof ServerErrorCode) {
                if (e == ServerErrorCode.No_Error) {
                    successfulResponseCount.incrementAndGet();
                } else {
                    invalidResponse.set(true);
                }
            } else {
                failedReplicaIds.add(replicaId);
            }
        }
    };
    // Instantiate a router just to put a blob successfully.
    MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
    setRouter(props, mockServerLayout);
    setOperationParams();
    // More extensive test for puts present elsewhere - these statements are here just to exercise the flow within the
    // NonBlockingRouter class, and to ensure that operations submitted to a router eventually completes.
    String blobIdStr = router.putBlob(putBlobProperties, putUserMetadata, putChannel).get();
    BlobId blobId = RouterUtils.getBlobIdFromString(blobIdStr, mockClusterMap);
    router.close();
    for (MockServer mockServer : mockServerLayout.getMockServers()) {
        mockServer.setServerErrorForAllRequests(ServerErrorCode.No_Error);
    }
    NetworkClient networkClient = new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime).getNetworkClient();
    cryptoJobHandler = new CryptoJobHandler(CryptoJobHandlerTest.DEFAULT_THREAD_COUNT);
    KeyManagementService localKMS = new MockKeyManagementService(new KMSConfig(verifiableProperties), singleKeyForKMS);
    putManager = new PutManager(mockClusterMap, mockResponseHandler, new LoggingNotificationSystem(), new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap), new RouterCallback(networkClient, new ArrayList<BackgroundDeleteRequest>()), "0", localKMS, cryptoService, cryptoJobHandler, mockTime);
    OperationHelper opHelper = new OperationHelper(OperationType.PUT);
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, null, successfulResponseCount, invalidResponse, -1);
    // Test that if a failed response comes before the operation is completed, failure detector is notified.
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, null, successfulResponseCount, invalidResponse, 0);
    // Test that if a failed response comes after the operation is completed, failure detector is notified.
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, null, successfulResponseCount, invalidResponse, PUT_REQUEST_PARALLELISM - 1);
    testNoResponseNoNotification(opHelper, failedReplicaIds, null, successfulResponseCount, invalidResponse);
    testResponseDeserializationError(opHelper, networkClient, null);
    opHelper = new OperationHelper(OperationType.GET);
    getManager = new GetManager(mockClusterMap, mockResponseHandler, new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap), new RouterCallback(networkClient, new ArrayList<BackgroundDeleteRequest>()), localKMS, cryptoService, cryptoJobHandler, mockTime);
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, -1);
    // Test that if a failed response comes before the operation is completed, failure detector is notified.
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, 0);
    // Test that if a failed response comes after the operation is completed, failure detector is notified.
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, GET_REQUEST_PARALLELISM - 1);
    testNoResponseNoNotification(opHelper, failedReplicaIds, blobId, successfulResponseCount, invalidResponse);
    testResponseDeserializationError(opHelper, networkClient, blobId);
    opHelper = new OperationHelper(OperationType.DELETE);
    deleteManager = new DeleteManager(mockClusterMap, mockResponseHandler, new LoggingNotificationSystem(), new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap), new RouterCallback(null, new ArrayList<BackgroundDeleteRequest>()), mockTime);
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, -1);
    // Test that if a failed response comes before the operation is completed, failure detector is notified.
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, 0);
    // Test that if a failed response comes after the operation is completed, failure detector is notified.
    testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, DELETE_REQUEST_PARALLELISM - 1);
    testNoResponseNoNotification(opHelper, failedReplicaIds, blobId, successfulResponseCount, invalidResponse);
    testResponseDeserializationError(opHelper, networkClient, blobId);
    putManager.close();
    getManager.close();
    deleteManager.close();
}

Example 19 with VerifiableProperties

use of com.github.ambry.config.VerifiableProperties in project ambry by linkedin.

the class NonBlockingRouterTest method testUnsuccessfulPutDataChunkDelete.

/**
 * Test that if a composite blob put fails, the successfully put data chunks are deleted.
 */
@Test
public void testUnsuccessfulPutDataChunkDelete() throws Exception {
    // Ensure there are 4 chunks.
    maxPutChunkSize = PUT_CONTENT_SIZE / 4;
    Properties props = getNonBlockingRouterProperties("DC1");
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    MockClusterMap mockClusterMap = new MockClusterMap();
    MockTime mockTime = new MockTime();
    MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
    // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
    // fails, it uses a notification system to track the deletions.
    final CountDownLatch deletesDoneLatch = new CountDownLatch(2);
    final Map<String, String> blobsThatAreDeleted = new HashMap<>();
    LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

        @Override
        public void onBlobDeleted(String blobId, String serviceId) {
            blobsThatAreDeleted.put(blobId, serviceId);
            deletesDoneLatch.countDown();
        }
    };
    router = new NonBlockingRouter(new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, mockTime);
    setOperationParams();
    List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
    List<ServerErrorCode> serverErrorList = new ArrayList<>();
    // There are 4 chunks for this blob.
    // All put operations make one request to each local server as there are 3 servers overall in the local DC.
    // Set the state of the mock servers so that they return success for the first 2 requests in order to succeed
    // the first two chunks.
    serverErrorList.add(ServerErrorCode.No_Error);
    serverErrorList.add(ServerErrorCode.No_Error);
    // fail requests for third and fourth data chunks including the slipped put attempts:
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    // all subsequent requests (no more puts, but there will be deletes) will succeed.
    for (DataNodeId dataNodeId : dataNodeIds) {
        MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
        server.setServerErrors(serverErrorList);
    }
    // Submit the put operation and wait for it to fail.
    try {
        router.putBlob(putBlobProperties, putUserMetadata, putChannel).get();
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.AmbryUnavailable, ((RouterException) e.getCause()).getErrorCode());
    }
    // Now, wait until the deletes of the successfully put blobs are complete.
    Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
    for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
        Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
    }
    router.close();
    assertClosed();
    Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
}

Example 20 with VerifiableProperties

use of com.github.ambry.config.VerifiableProperties in project ambry by linkedin.

the class NonBlockingRouterTest method testSimpleBlobDelete.

/**
 * Test to ensure that for simple blob deletions, no additional background delete operations
 * are initiated.
 */
@Test
public void testSimpleBlobDelete() throws Exception {
    // Ensure there are 4 chunks.
    maxPutChunkSize = PUT_CONTENT_SIZE;
    Properties props = getNonBlockingRouterProperties("DC1");
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    MockClusterMap mockClusterMap = new MockClusterMap();
    MockTime mockTime = new MockTime();
    MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
    String deleteServiceId = "delete-service";
    // metadata blob + data chunks.
    final AtomicInteger deletesInitiated = new AtomicInteger();
    final AtomicReference<String> receivedDeleteServiceId = new AtomicReference<>();
    LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

        @Override
        public void onBlobDeleted(String blobId, String serviceId) {
            deletesInitiated.incrementAndGet();
            receivedDeleteServiceId.set(serviceId);
        }
    };
    router = new NonBlockingRouter(new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, mockTime);
    setOperationParams();
    String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel).get();
    router.deleteBlob(blobId, deleteServiceId, null).get();
    long waitStart = SystemTime.getInstance().milliseconds();
    while (router.getBackgroundOperationsCount() != 0 && SystemTime.getInstance().milliseconds() < waitStart + AWAIT_TIMEOUT_MS) {
        Thread.sleep(1000);
    }
    Assert.assertEquals("All background operations should be complete ", 0, router.getBackgroundOperationsCount());
    Assert.assertEquals("Only the original blob deletion should have been initiated", 1, deletesInitiated.get());
    Assert.assertEquals("The delete service ID should match the expected value", deleteServiceId, receivedDeleteServiceId.get());
    router.close();
    assertClosed();
    Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
}