Examples with CloudDestinationFactory com.github.ambry.cloud.CloudDestinationFactory used on opensource projects

Example 6 with CloudDestinationFactory

use of com.github.ambry.cloud.CloudDestinationFactory in project ambry by linkedin.

the class AzureCloudDestinationTest method testInitClientException.

/**
 * Test constructor with invalid connection string.
 */
@Test
public void testInitClientException() throws IOException {
    CloudDestinationFactory factory = new AzureCloudDestinationFactory(new VerifiableProperties(configProps), new MetricRegistry(), clusterMap);
    CloudDestination cloudDestination = null;
    try {
        cloudDestination = factory.getCloudDestination();
        fail("Expected exception");
    } catch (IllegalStateException ex) {
    } finally {
        if (cloudDestination != null) {
            cloudDestination.close();
        }
    }
}

Example 7 with CloudDestinationFactory

use of com.github.ambry.cloud.CloudDestinationFactory in project ambry by linkedin.

the class CloudRouterFactory method getRouter.

/**
 * Construct and return a {@link NonBlockingRouter} that works with cloud storage.
 * @return a {@link NonBlockingRouter}
 */
@Override
public Router getRouter() throws InstantiationException {
    try {
        MetricRegistry registry = clusterMap.getMetricRegistry();
        CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
        CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, registry, clusterMap);
        CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
        RequestHandlerPool requestHandlerPool = getRequestHandlerPool(verifiableProperties, clusterMap, cloudDestination, cloudConfig);
        CompositeNetworkClientFactory networkClientFactory = getCompositeNetworkClientFactory(requestHandlerPool);
        NonBlockingRouter router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, clusterMap, kms, cryptoService, cryptoJobHandler, accountService, time, defaultPartitionClass);
        // Make sure requestHandlerPool is shut down properly
        router.addResourceToClose(requestHandlerPool);
        router.addResourceToClose(cloudDestination);
        logger.info("Instantiated NonBlockingRouter");
        return router;
    } catch (Exception e) {
        logger.error("Error instantiating NonBlocking Router", e);
        throw new InstantiationException("Error instantiating NonBlocking Router: " + e.toString());
    }
}

Example 8 with CloudDestinationFactory

use of com.github.ambry.cloud.CloudDestinationFactory in project ambry by linkedin.

the class CloudRouterTest method setRouter.

/**
 * Initialize and set the router with the given {@link Properties} and {@link MockServerLayout}
 * @param props the {@link Properties}
 * @param notificationSystem the {@link NotificationSystem} to use.
 */
@Override
protected void setRouter(Properties props, MockServerLayout mockServerLayout, NotificationSystem notificationSystem) throws Exception {
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, mockClusterMap.getMetricRegistry(), mockClusterMap);
    CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
    AccountService accountService = new InMemAccountService(false, true);
    CloudRouterFactory cloudRouterFactory = new CloudRouterFactory(verifiableProperties, mockClusterMap, new LoggingNotificationSystem(), null, accountService);
    RequestHandlerPool requestHandlerPool = cloudRouterFactory.getRequestHandlerPool(verifiableProperties, mockClusterMap, cloudDestination, cloudConfig);
    Map<ReplicaType, NetworkClientFactory> childFactories = new EnumMap<>(ReplicaType.class);
    childFactories.put(ReplicaType.CLOUD_BACKED, new LocalNetworkClientFactory((LocalRequestResponseChannel) requestHandlerPool.getChannel(), new NetworkConfig(verifiableProperties), new NetworkMetrics(routerMetrics.getMetricRegistry()), mockTime));
    childFactories.put(ReplicaType.DISK_BACKED, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime));
    NetworkClientFactory networkClientFactory = new CompositeNetworkClientFactory(childFactories);
    router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    router.addResourceToClose(requestHandlerPool);
}

Example 9 with CloudDestinationFactory

use of com.github.ambry.cloud.CloudDestinationFactory in project ambry by linkedin.

the class VcrRecoveryTest method setup.

/**
 * Create a cluster with a vcr node and a recovery (ambry data) node.
 * @throws Exception on {@link Exception}
 */
@Before
public void setup() throws Exception {
    String vcrMountPath = ClusterMapSnapshotConstants.CLOUD_REPLICA_MOUNT + "/1";
    recoveryProperties = new Properties();
    recoveryProperties.setProperty("replication.metadata.request.version", "2");
    // create vcr node
    List<Port> vcrPortList = new ArrayList<>(2);
    Port vcrClusterMapPort = new Port(12310, PortType.PLAINTEXT);
    Port vcrSslPort = new Port(12410, PortType.SSL);
    vcrPortList.add(vcrClusterMapPort);
    vcrPortList.add(vcrSslPort);
    MockDataNodeId vcrNode = new MockDataNodeId("localhost", vcrPortList, Collections.singletonList(vcrMountPath), dcName);
    // create recovery node
    recoveryNodePort = new Port(12311, PortType.PLAINTEXT);
    ArrayList<Port> recoveryPortList = new ArrayList<>(2);
    recoveryPortList.add(recoveryNodePort);
    recoveryNode = MockClusterMap.createDataNode(recoveryPortList, dcName, 1);
    // create cluster for recovery
    recoveryCluster = MockCluster.createOneNodeRecoveryCluster(vcrNode, recoveryNode, dcName);
    partitionId = recoveryCluster.getClusterMap().getWritablePartitionIds(null).get(0);
    // Start ZK Server and Helix Controller.
    if (!zkInfo.isZkServerStarted()) {
        zkInfo.startZkServer();
    }
    helixControllerManager = VcrTestUtil.populateZkInfoAndStartController(zkConnectString, vcrClusterName, recoveryCluster.getClusterMap());
    Properties vcrProperties = VcrTestUtil.createVcrProperties(vcrNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310, 12410, 12510, null);
    vcrProperties.putAll(recoveryProperties);
    NotificationSystem notificationSystem = new MockNotificationSystem(recoveryCluster.getClusterMap());
    // Create blobs and data for upload to vcr.
    int blobCount = 10;
    blobIds = ServerTestUtil.createBlobIds(blobCount, recoveryCluster.getClusterMap(), accountId, containerId, partitionId);
    // Create cloud destination and start vcr server.
    latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, recoveryCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(vcrProperties), recoveryCluster.getClusterAgentsFactory(), notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // start ambry server with data node
    recoveryCluster.initializeServers(notificationSystem, vcrNode, recoveryProperties);
    recoveryCluster.startServers();
}

Example 10 with CloudDestinationFactory

use of com.github.ambry.cloud.CloudDestinationFactory in project ambry by linkedin.

the class VcrBackupTest method singleNodeUpDownTestWithPersist.

/**
 * Test single VCR up and down with persisted token.
 */
@Test
public void singleNodeUpDownTestWithPersist() throws Exception {
    StrictMatchExternalViewVerifier helixBalanceVerifier = new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName, Collections.singleton(VcrTestUtil.helixResource), null);
    int numberOfBlobs = 20;
    sendBlobToDataNode(dataNode, numberOfBlobs);
    // Create in memory cloud destination.
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    // Start the VCR with token persistor on.
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, clusterMapPort, 12410, 12510, null, vcrHelixStateModelFactoryClass, true);
    props.setProperty("replication.persist.token.on.shutdown.or.replica.remove", "true");
    MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy = vcrNotificationSystem;
    assertTrue("Blob count is not correct.", TestUtils.checkAndSleep(numberOfBlobs, () -> vcrNotificationSystemCopy.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    assertTrue("Token is expected.", latchBasedInMemoryCloudDestination.getTokenMap().size() > 0);
    // Start VCR again with same cloud destination
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    // Because token is reloaded, back up number is 0.
    assertEquals("Number of blobs doesn't match", 0, vcrNotificationSystem.getBlobIds().size());
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    // Start VCR again with token.
    LatchBasedInMemoryCloudDestination newLatchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    for (Map.Entry<String, byte[]> entry : latchBasedInMemoryCloudDestination.getTokenMap().entrySet()) {
        newLatchBasedInMemoryCloudDestination.getTokenMap().put(entry.getKey(), entry.getValue());
    }
    cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy2 = vcrNotificationSystem;
    assertTrue("Blob count is not correct.", TestUtils.checkAndSleep(0, () -> vcrNotificationSystemCopy2.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
}