Examples with CloudDestination com.github.ambry.cloud.CloudDestination used on opensource projects

Example 1 with CloudDestination

use of com.github.ambry.cloud.CloudDestination in project ambry by linkedin.

the class VcrAutomationTest method testSimpleVcrAutomation.

/**
 * Test basic partition add and remove cases.
 */
@Test
public void testSimpleVcrAutomation() throws Exception {
    List<ZkInfo> zkInfoList = new ArrayList<>();
    String mainClusterStateModelDef = ClusterMapConfig.AMBRY_STATE_MODEL_DEF;
    DataNodeConfigSourceType dataNodeConfigSourceType = DataNodeConfigSourceType.INSTANCE_CONFIG;
    int zkPort = 2100;
    int numberOfDataNode = 3;
    int partitionCount = 9;
    int newPartitionCount = 13;
    String zkHostName = "localhost";
    String zkConnectString = zkHostName + ":" + zkPort;
    String clusterPrefix = "";
    String clusterName = "MainCluster";
    String vcrClusterName = "VcrCluster";
    String dcName = "DC0";
    TestHardwareLayout testHardwareLayout;
    TestPartitionLayout testPartitionLayout;
    String hardwareLayoutPath;
    String partitionLayoutPath;
    String zkLayoutPath;
    zkInfoList.add(new ZkInfo(TestUtils.getTempDir("tempZk"), "DC0", (byte) 0, zkPort, true));
    String tempDirPath = getTempDir(clusterName + "-");
    hardwareLayoutPath = tempDirPath + "/hardwareLayoutTest.json";
    partitionLayoutPath = tempDirPath + "/partitionLayoutTest.json";
    zkLayoutPath = tempDirPath + "/zkLayoutPath.json";
    testHardwareLayout = new TestHardwareLayout(clusterName, 1, 10737418240L, numberOfDataNode, 1, 18088, 20, false);
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, partitionCount, null);
    JSONObject zkJson = constructZkLayoutJSON(zkInfoList);
    Utils.writeJsonObjectToFile(zkJson, zkLayoutPath);
    Utils.writeJsonObjectToFile(testHardwareLayout.getHardwareLayout().toJSONObject(), hardwareLayoutPath);
    Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
    Properties props = VcrTestUtil.createVcrProperties("DC0", vcrClusterName, zkConnectString, 12300, 12400, 12510, null);
    props.setProperty("clustermap.host.name", "localhost");
    props.setProperty("clustermap.port", "1100");
    props.setProperty("clustermap.cluster.name", clusterName);
    props.setProperty("clustermap.datacenter.name", dcName);
    props.setProperty("clustermap.dcs.zk.connect.strings", zkJson.toString(2));
    props.setProperty("clustermap.state.model.definition", mainClusterStateModelDef);
    props.setProperty("clustermap.data.node.config.source.type", dataNodeConfigSourceType.name());
    props.setProperty("clustermap.cluster.change.handler.type", "DynamicClusterChangeHandler");
    props.setProperty("vcr.helix.updater.partition.id", "1");
    props.setProperty("vcr.helix.update.delay.time.in.seconds", "1");
    HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, clusterPrefix, dcName, 10, false, false, new HelixAdminFactory(), false, mainClusterStateModelDef, HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster, dataNodeConfigSourceType, false);
    HelixControllerManager helixControllerManager = new HelixControllerManager(zkConnectString, clusterPrefix + clusterName);
    helixControllerManager.syncStart();
    // Main cluster helix setup done.
    HelixVcrUtil.VcrHelixConfig vcrHelixConfig;
    String vcConfigData = CloudConfig.DEFAULT_VCR_HELIX_UPDATE_CONFIG;
    try (InputStream input = new ByteArrayInputStream(vcConfigData.getBytes())) {
        vcrHelixConfig = new ObjectMapper().readValue(input, HelixVcrUtil.VcrHelixConfig.class);
    } catch (IOException ex) {
        throw new IllegalStateException("Could not load config from config data: " + vcConfigData);
    }
    HelixVcrUtil.createCluster(zkConnectString, vcrClusterName, vcrHelixConfig);
    HelixVcrUtil.updateResourceAndPartition(zkConnectString, clusterPrefix + clusterName, zkConnectString, vcrClusterName, vcrHelixConfig, false);
    Assert.assertTrue("Dest and Src should be same", isSrcDestSync(zkConnectString, clusterPrefix + clusterName, zkConnectString, vcrClusterName));
    HelixControllerManager vcrHelixControllerManager = new HelixControllerManager(zkConnectString, vcrClusterName);
    vcrHelixControllerManager.syncStart();
    StrictMatchExternalViewVerifier helixBalanceVerifier = new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName, Collections.singleton(VcrTestUtil.helixResource), null);
    // VCR cluster helix setup done.
    VerifiableProperties verifiableProperties = new VerifiableProperties(props);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    HelixClusterAgentsFactory helixClusterAgentsFactory = new HelixClusterAgentsFactory(clusterMapConfig, null, null);
    VcrServer vcrServer = new VcrServer(verifiableProperties, helixClusterAgentsFactory, null, new CloudDestinationFactory() {

        @Override
        public CloudDestination getCloudDestination() throws IllegalStateException {
            return mock(CloudDestination.class);
        }
    }, null);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    Assert.assertTrue("Partition assignment is not correct.", TestUtils.checkAndSleep(partitionCount, () -> vcrServer.getVcrClusterParticipant().getAssignedPartitionIds().size(), 5000));
    // vcr server start up done.
    // Partition add case:
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, partitionCount + newPartitionCount, null);
    Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
    HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, clusterPrefix, dcName, 10, false, false, new HelixAdminFactory(), false, mainClusterStateModelDef, HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster, dataNodeConfigSourceType, false);
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    Assert.assertTrue("Partition assignment is not correct.", TestUtils.checkAndSleep(partitionCount + newPartitionCount, () -> vcrServer.getVcrClusterParticipant().getAssignedPartitionIds().size(), 5000));
    // Partition remove case:
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, partitionCount, null);
    Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
    HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, clusterPrefix, dcName, 10, false, true, new HelixAdminFactory(), false, mainClusterStateModelDef, HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster, dataNodeConfigSourceType, false);
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    Assert.assertTrue("Partition assignment is not correct.", TestUtils.checkAndSleep(partitionCount, () -> vcrServer.getVcrClusterParticipant().getAssignedPartitionIds().size(), 5000));
    helixControllerManager.syncStop();
    vcrHelixControllerManager.syncStop();
    zkInfoList.get(0).shutdown();
}

Example 2 with CloudDestination

use of com.github.ambry.cloud.CloudDestination in project ambry by linkedin.

the class AzureCloudDestinationTest method testInitClientException.

/**
 * Test constructor with invalid connection string.
 */
@Test
public void testInitClientException() throws IOException {
    CloudDestinationFactory factory = new AzureCloudDestinationFactory(new VerifiableProperties(configProps), new MetricRegistry(), clusterMap);
    CloudDestination cloudDestination = null;
    try {
        cloudDestination = factory.getCloudDestination();
        fail("Expected exception");
    } catch (IllegalStateException ex) {
    } finally {
        if (cloudDestination != null) {
            cloudDestination.close();
        }
    }
}

Example 3 with CloudDestination

use of com.github.ambry.cloud.CloudDestination in project ambry by linkedin.

the class CloudRouterFactory method getRouter.

/**
 * Construct and return a {@link NonBlockingRouter} that works with cloud storage.
 * @return a {@link NonBlockingRouter}
 */
@Override
public Router getRouter() throws InstantiationException {
    try {
        MetricRegistry registry = clusterMap.getMetricRegistry();
        CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
        CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, registry, clusterMap);
        CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
        RequestHandlerPool requestHandlerPool = getRequestHandlerPool(verifiableProperties, clusterMap, cloudDestination, cloudConfig);
        CompositeNetworkClientFactory networkClientFactory = getCompositeNetworkClientFactory(requestHandlerPool);
        NonBlockingRouter router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, clusterMap, kms, cryptoService, cryptoJobHandler, accountService, time, defaultPartitionClass);
        // Make sure requestHandlerPool is shut down properly
        router.addResourceToClose(requestHandlerPool);
        router.addResourceToClose(cloudDestination);
        logger.info("Instantiated NonBlockingRouter");
        return router;
    } catch (Exception e) {
        logger.error("Error instantiating NonBlocking Router", e);
        throw new InstantiationException("Error instantiating NonBlocking Router: " + e.toString());
    }
}

Example 4 with CloudDestination

use of com.github.ambry.cloud.CloudDestination in project ambry by linkedin.

the class CloudRouterTest method setRouter.

/**
 * Initialize and set the router with the given {@link Properties} and {@link MockServerLayout}
 * @param props the {@link Properties}
 * @param notificationSystem the {@link NotificationSystem} to use.
 */
@Override
protected void setRouter(Properties props, MockServerLayout mockServerLayout, NotificationSystem notificationSystem) throws Exception {
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, mockClusterMap.getMetricRegistry(), mockClusterMap);
    CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
    AccountService accountService = new InMemAccountService(false, true);
    CloudRouterFactory cloudRouterFactory = new CloudRouterFactory(verifiableProperties, mockClusterMap, new LoggingNotificationSystem(), null, accountService);
    RequestHandlerPool requestHandlerPool = cloudRouterFactory.getRequestHandlerPool(verifiableProperties, mockClusterMap, cloudDestination, cloudConfig);
    Map<ReplicaType, NetworkClientFactory> childFactories = new EnumMap<>(ReplicaType.class);
    childFactories.put(ReplicaType.CLOUD_BACKED, new LocalNetworkClientFactory((LocalRequestResponseChannel) requestHandlerPool.getChannel(), new NetworkConfig(verifiableProperties), new NetworkMetrics(routerMetrics.getMetricRegistry()), mockTime));
    childFactories.put(ReplicaType.DISK_BACKED, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime));
    NetworkClientFactory networkClientFactory = new CompositeNetworkClientFactory(childFactories);
    router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    router.addResourceToClose(requestHandlerPool);
}