Search in sources :

Example 1 with HelixAdminFactory

use of com.github.ambry.clustermap.HelixAdminFactory in project ambry by linkedin.

the class VcrAutomationTest method testSimpleVcrAutomation.

/**
 * Test basic partition add and remove cases.
 */
@Test
public void testSimpleVcrAutomation() throws Exception {
    List<ZkInfo> zkInfoList = new ArrayList<>();
    String mainClusterStateModelDef = ClusterMapConfig.AMBRY_STATE_MODEL_DEF;
    DataNodeConfigSourceType dataNodeConfigSourceType = DataNodeConfigSourceType.INSTANCE_CONFIG;
    int zkPort = 2100;
    int numberOfDataNode = 3;
    int partitionCount = 9;
    int newPartitionCount = 13;
    String zkHostName = "localhost";
    String zkConnectString = zkHostName + ":" + zkPort;
    String clusterPrefix = "";
    String clusterName = "MainCluster";
    String vcrClusterName = "VcrCluster";
    String dcName = "DC0";
    TestHardwareLayout testHardwareLayout;
    TestPartitionLayout testPartitionLayout;
    String hardwareLayoutPath;
    String partitionLayoutPath;
    String zkLayoutPath;
    zkInfoList.add(new ZkInfo(TestUtils.getTempDir("tempZk"), "DC0", (byte) 0, zkPort, true));
    String tempDirPath = getTempDir(clusterName + "-");
    hardwareLayoutPath = tempDirPath + "/hardwareLayoutTest.json";
    partitionLayoutPath = tempDirPath + "/partitionLayoutTest.json";
    zkLayoutPath = tempDirPath + "/zkLayoutPath.json";
    testHardwareLayout = new TestHardwareLayout(clusterName, 1, 10737418240L, numberOfDataNode, 1, 18088, 20, false);
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, partitionCount, null);
    JSONObject zkJson = constructZkLayoutJSON(zkInfoList);
    Utils.writeJsonObjectToFile(zkJson, zkLayoutPath);
    Utils.writeJsonObjectToFile(testHardwareLayout.getHardwareLayout().toJSONObject(), hardwareLayoutPath);
    Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
    Properties props = VcrTestUtil.createVcrProperties("DC0", vcrClusterName, zkConnectString, 12300, 12400, 12510, null);
    props.setProperty("clustermap.host.name", "localhost");
    props.setProperty("clustermap.port", "1100");
    props.setProperty("clustermap.cluster.name", clusterName);
    props.setProperty("clustermap.datacenter.name", dcName);
    props.setProperty("clustermap.dcs.zk.connect.strings", zkJson.toString(2));
    props.setProperty("clustermap.state.model.definition", mainClusterStateModelDef);
    props.setProperty("clustermap.data.node.config.source.type", dataNodeConfigSourceType.name());
    props.setProperty("clustermap.cluster.change.handler.type", "DynamicClusterChangeHandler");
    props.setProperty("vcr.helix.updater.partition.id", "1");
    props.setProperty("vcr.helix.update.delay.time.in.seconds", "1");
    HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, clusterPrefix, dcName, 10, false, false, new HelixAdminFactory(), false, mainClusterStateModelDef, HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster, dataNodeConfigSourceType, false);
    HelixControllerManager helixControllerManager = new HelixControllerManager(zkConnectString, clusterPrefix + clusterName);
    helixControllerManager.syncStart();
    // Main cluster helix setup done.
    HelixVcrUtil.VcrHelixConfig vcrHelixConfig;
    String vcConfigData = CloudConfig.DEFAULT_VCR_HELIX_UPDATE_CONFIG;
    try (InputStream input = new ByteArrayInputStream(vcConfigData.getBytes())) {
        vcrHelixConfig = new ObjectMapper().readValue(input, HelixVcrUtil.VcrHelixConfig.class);
    } catch (IOException ex) {
        throw new IllegalStateException("Could not load config from config data: " + vcConfigData);
    }
    HelixVcrUtil.createCluster(zkConnectString, vcrClusterName, vcrHelixConfig);
    HelixVcrUtil.updateResourceAndPartition(zkConnectString, clusterPrefix + clusterName, zkConnectString, vcrClusterName, vcrHelixConfig, false);
    Assert.assertTrue("Dest and Src should be same", isSrcDestSync(zkConnectString, clusterPrefix + clusterName, zkConnectString, vcrClusterName));
    HelixControllerManager vcrHelixControllerManager = new HelixControllerManager(zkConnectString, vcrClusterName);
    vcrHelixControllerManager.syncStart();
    StrictMatchExternalViewVerifier helixBalanceVerifier = new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName, Collections.singleton(VcrTestUtil.helixResource), null);
    // VCR cluster helix setup done.
    VerifiableProperties verifiableProperties = new VerifiableProperties(props);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    HelixClusterAgentsFactory helixClusterAgentsFactory = new HelixClusterAgentsFactory(clusterMapConfig, null, null);
    VcrServer vcrServer = new VcrServer(verifiableProperties, helixClusterAgentsFactory, null, new CloudDestinationFactory() {

        @Override
        public CloudDestination getCloudDestination() throws IllegalStateException {
            return mock(CloudDestination.class);
        }
    }, null);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    Assert.assertTrue("Partition assignment is not correct.", TestUtils.checkAndSleep(partitionCount, () -> vcrServer.getVcrClusterParticipant().getAssignedPartitionIds().size(), 5000));
    // vcr server start up done.
    // Partition add case:
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, partitionCount + newPartitionCount, null);
    Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
    HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, clusterPrefix, dcName, 10, false, false, new HelixAdminFactory(), false, mainClusterStateModelDef, HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster, dataNodeConfigSourceType, false);
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    Assert.assertTrue("Partition assignment is not correct.", TestUtils.checkAndSleep(partitionCount + newPartitionCount, () -> vcrServer.getVcrClusterParticipant().getAssignedPartitionIds().size(), 5000));
    // Partition remove case:
    testPartitionLayout = constructInitialPartitionLayoutJSON(testHardwareLayout, partitionCount, null);
    Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
    HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, clusterPrefix, dcName, 10, false, true, new HelixAdminFactory(), false, mainClusterStateModelDef, HelixBootstrapUpgradeUtil.HelixAdminOperation.BootstrapCluster, dataNodeConfigSourceType, false);
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    Assert.assertTrue("Partition assignment is not correct.", TestUtils.checkAndSleep(partitionCount, () -> vcrServer.getVcrClusterParticipant().getAssignedPartitionIds().size(), 5000));
    helixControllerManager.syncStop();
    vcrHelixControllerManager.syncStop();
    zkInfoList.get(0).shutdown();
}
Also used : ArrayList(java.util.ArrayList) HelixVcrUtil(com.github.ambry.clustermap.HelixVcrUtil) HelixAdminFactory(com.github.ambry.clustermap.HelixAdminFactory) CloudDestinationFactory(com.github.ambry.cloud.CloudDestinationFactory) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) VcrServer(com.github.ambry.cloud.VcrServer) DataNodeConfigSourceType(com.github.ambry.clustermap.DataNodeConfigSourceType) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) VerifiableProperties(com.github.ambry.config.VerifiableProperties) HelixClusterAgentsFactory(com.github.ambry.clustermap.HelixClusterAgentsFactory) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) CloudDestination(com.github.ambry.cloud.CloudDestination) HelixControllerManager(com.github.ambry.utils.HelixControllerManager) IOException(java.io.IOException) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) StrictMatchExternalViewVerifier(org.apache.helix.tools.ClusterVerifiers.StrictMatchExternalViewVerifier) JSONObject(org.json.JSONObject) ByteArrayInputStream(java.io.ByteArrayInputStream) VcrBackupTest(com.github.ambry.server.VcrBackupTest) Test(org.junit.Test)

Aggregations

ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 CloudDestination (com.github.ambry.cloud.CloudDestination)1 CloudDestinationFactory (com.github.ambry.cloud.CloudDestinationFactory)1 VcrServer (com.github.ambry.cloud.VcrServer)1 DataNodeConfigSourceType (com.github.ambry.clustermap.DataNodeConfigSourceType)1 HelixAdminFactory (com.github.ambry.clustermap.HelixAdminFactory)1 HelixClusterAgentsFactory (com.github.ambry.clustermap.HelixClusterAgentsFactory)1 HelixVcrUtil (com.github.ambry.clustermap.HelixVcrUtil)1 ClusterMapConfig (com.github.ambry.config.ClusterMapConfig)1 VerifiableProperties (com.github.ambry.config.VerifiableProperties)1 VcrBackupTest (com.github.ambry.server.VcrBackupTest)1 HelixControllerManager (com.github.ambry.utils.HelixControllerManager)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 ArrayList (java.util.ArrayList)1 Properties (java.util.Properties)1 StrictMatchExternalViewVerifier (org.apache.helix.tools.ClusterVerifiers.StrictMatchExternalViewVerifier)1 JSONObject (org.json.JSONObject)1 Test (org.junit.Test)1