Examples with ClusterSetup org.apache.helix.tools.ClusterSetup used on opensource projects

Example 61 with ClusterSetup

use of org.apache.helix.tools.ClusterSetup in project helix by apache.

the class TestStandAloneCMSessionExpiry method testStandAloneCMSessionExpiry.

@Test()
public void testStandAloneCMSessionExpiry() throws Exception {
    // Logger.getRootLogger().setLevel(Level.DEBUG);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, PARTICIPANT_PREFIX, "TestDB", 1, 20, 5, 3, "MasterSlave", true);
    MockParticipantManager[] participants = new MockParticipantManager[5];
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    boolean result;
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // participant session expiry
    MockParticipantManager participantToExpire = participants[1];
    System.out.println("Expire participant session");
    String oldSessionId = participantToExpire.getSessionId();
    ZkTestHelper.expireSession(participantToExpire.getZkClient());
    String newSessionId = participantToExpire.getSessionId();
    System.out.println("oldSessionId: " + oldSessionId + ", newSessionId: " + newSessionId);
    Assert.assertTrue(newSessionId.compareTo(oldSessionId) > 0, "Session id should be increased after expiry");
    ClusterSetup setupTool = new ClusterSetup(ZK_ADDR);
    setupTool.addResourceToCluster(clusterName, "TestDB1", 10, "MasterSlave");
    setupTool.rebalanceStorageCluster(clusterName, "TestDB1", 3);
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // controller session expiry
    System.out.println("Expire controller session");
    oldSessionId = controller.getSessionId();
    ZkTestHelper.expireSession(controller.getZkClient());
    newSessionId = controller.getSessionId();
    System.out.println("oldSessionId: " + oldSessionId + ", newSessionId: " + newSessionId);
    Assert.assertTrue(newSessionId.compareTo(oldSessionId) > 0, "Session id should be increased after expiry");
    setupTool.addResourceToCluster(clusterName, "TestDB2", 8, "MasterSlave");
    setupTool.rebalanceStorageCluster(clusterName, "TestDB2", 3);
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // clean up
    System.out.println("Clean up ...");
    // Logger.getRootLogger().setLevel(Level.DEBUG);
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 62 with ClusterSetup

use of org.apache.helix.tools.ClusterSetup in project helix by apache.

the class TestDisablePartition method testDisableFullAuto.

@Test(dataProvider = "rebalancer", enabled = true)
public void testDisableFullAuto(String rebalancerName) throws Exception {
    final int NUM_PARTITIONS = 8;
    final int NUM_PARTICIPANTS = 2;
    final int NUM_REPLICAS = 1;
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    ClusterSetup clusterSetup = new ClusterSetup(ZK_ADDR);
    clusterSetup.addCluster(clusterName, true);
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        String instanceName = "localhost_" + (11420 + i);
        clusterSetup.addInstanceToCluster(clusterName, instanceName);
    }
    // Create a known problematic scenario
    HelixAdmin admin = clusterSetup.getClusterManagementTool();
    String resourceName = "MailboxDB";
    IdealState idealState = new IdealState(resourceName + "DR");
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setStateModelDefRef("LeaderStandby");
    idealState.setReplicas(String.valueOf(NUM_REPLICAS));
    idealState.setNumPartitions(NUM_PARTITIONS);
    for (int i = 0; i < NUM_PARTITIONS; i++) {
        String partitionName = resourceName + '_' + i;
        List<String> assignmentList = Lists.newArrayList();
        if (i < NUM_PARTITIONS / 2) {
            assignmentList.add("localhost_11420");
        } else {
            assignmentList.add("localhost_11421");
        }
        Map<String, String> emptyMap = Maps.newHashMap();
        idealState.getRecord().setListField(partitionName, assignmentList);
        idealState.getRecord().setMapField(partitionName, emptyMap);
    }
    admin.addResource(clusterName, idealState.getResourceName(), idealState);
    // Start everything
    MockParticipantManager[] participants = new MockParticipantManager[NUM_PARTICIPANTS];
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        String instanceName = "localhost_" + (11420 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_1");
    controller.syncStart();
    Thread.sleep(1000);
    // Switch to full auto
    idealState.setRebalanceMode(RebalanceMode.FULL_AUTO);
    idealState.setRebalancerClassName(rebalancerName);
    for (int i = 0; i < NUM_PARTITIONS; i++) {
        List<String> emptyList = Collections.emptyList();
        idealState.getRecord().setListField(resourceName + '_' + i, emptyList);
    }
    admin.setResourceIdealState(clusterName, idealState.getResourceName(), idealState);
    Thread.sleep(1000);
    // Get the external view
    HelixDataAccessor accessor = controller.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    ExternalView externalView = accessor.getProperty(keyBuilder.externalView(idealState.getResourceName()));
    // Disable the partitions in an order known to cause problems
    int[] pid = { 0, 7 };
    for (int i = 0; i < pid.length; i++) {
        String partitionName = resourceName + '_' + pid[i];
        Map<String, String> stateMap = externalView.getStateMap(partitionName);
        String leader = null;
        for (String participantName : stateMap.keySet()) {
            String state = stateMap.get(participantName);
            if (state.equals("LEADER")) {
                leader = participantName;
            }
        }
        List<String> partitionNames = Lists.newArrayList(partitionName);
        admin.enablePartition(false, clusterName, leader, idealState.getResourceName(), partitionNames);
        Thread.sleep(1000);
    }
    // Ensure that nothing was reassigned and the disabled are offline
    externalView = accessor.getProperty(keyBuilder.externalView(idealState.getResourceName()));
    Map<String, String> p0StateMap = externalView.getStateMap(resourceName + "_0");
    Assert.assertEquals(p0StateMap.size(), 1);
    String p0Participant = p0StateMap.keySet().iterator().next();
    Assert.assertEquals(p0StateMap.get(p0Participant), "OFFLINE");
    Map<String, String> p7StateMap = externalView.getStateMap(resourceName + "_7");
    Assert.assertEquals(p7StateMap.size(), 1);
    String p7Participant = p7StateMap.keySet().iterator().next();
    Assert.assertEquals(p7StateMap.get(p7Participant), "OFFLINE");
    // Cleanup
    controller.syncStop();
    for (MockParticipantManager participant : participants) {
        participant.syncStop();
    }
}

Example 63 with ClusterSetup

use of org.apache.helix.tools.ClusterSetup in project helix by apache.

the class TestDriver method setupCluster.

public static void setupCluster(String uniqClusterName, String zkAddr, int numResources, int numPartitionsPerResource, int numInstances, int replica, boolean doRebalance) throws Exception {
    ZkClient zkClient = new ZkClient(zkAddr);
    zkClient.setZkSerializer(new ZNRecordSerializer());
    // String clusterName = CLUSTER_PREFIX + "_" + uniqClusterName;
    String clusterName = uniqClusterName;
    if (zkClient.exists("/" + clusterName)) {
        LOG.warn("test cluster already exists:" + clusterName + ", test name:" + uniqClusterName + " is not unique or test has been run without cleaning up zk; deleting it");
        zkClient.deleteRecursively("/" + clusterName);
    }
    if (_testInfoMap.containsKey(uniqClusterName)) {
        LOG.warn("test info already exists:" + uniqClusterName + " is not unique or test has been run without cleaning up test info map; removing it");
        _testInfoMap.remove(uniqClusterName);
    }
    TestInfo testInfo = new TestInfo(clusterName, zkClient, numResources, numPartitionsPerResource, numInstances, replica);
    _testInfoMap.put(uniqClusterName, testInfo);
    ClusterSetup setupTool = new ClusterSetup(zkAddr);
    setupTool.addCluster(clusterName, true);
    for (int i = 0; i < numInstances; i++) {
        int port = START_PORT + i;
        setupTool.addInstanceToCluster(clusterName, PARTICIPANT_PREFIX + "_" + port);
    }
    for (int i = 0; i < numResources; i++) {
        String dbName = TEST_DB_PREFIX + i;
        setupTool.addResourceToCluster(clusterName, dbName, numPartitionsPerResource, STATE_MODEL);
        if (doRebalance) {
            setupTool.rebalanceStorageCluster(clusterName, dbName, replica);
        // String idealStatePath = "/" + clusterName + "/" +
        // PropertyType.IDEALSTATES.toString() + "/"
        // + dbName;
        // ZNRecord idealState = zkClient.<ZNRecord> readData(idealStatePath);
        // testInfo._idealStateMap.put(dbName, idealState);
        }
    }
}

Example 64 with ClusterSetup

use of org.apache.helix.tools.ClusterSetup in project helix by apache.

the class ZkStandAloneCMTestBase method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    System.out.println("START " + CLASS_NAME + " at " + new Date(System.currentTimeMillis()));
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    _setupTool = new ClusterSetup(ZK_ADDR);
    // setup storage cluster
    _setupTool.addCluster(CLUSTER_NAME, true);
    _setupTool.addResourceToCluster(CLUSTER_NAME, TEST_DB, _PARTITIONS, STATE_MODEL);
    for (int i = 0; i < NODE_NR; i++) {
        String storageNodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
        _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
    }
    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, TEST_DB, _replica);
    // start dummy participants
    for (int i = 0; i < NODE_NR; i++) {
        String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
        _participants[i] = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName);
        _participants[i].syncStart();
    }
    // start controller
    String controllerName = CONTROLLER_PREFIX + "_0";
    _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName);
    _controller.syncStart();
    boolean result = ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, CLUSTER_NAME));
    Assert.assertTrue(result);
    result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
    Assert.assertTrue(result);
    // create cluster manager
    _manager = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "Admin", InstanceType.ADMINISTRATOR, ZK_ADDR);
    _manager.connect();
}

Example 65 with ClusterSetup

use of org.apache.helix.tools.ClusterSetup in project helix by apache.

the class TestClusterStartsup method setupCluster.

void setupCluster() throws HelixException {
    System.out.println("START " + CLASS_NAME + " at " + new Date(System.currentTimeMillis()));
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    _setupTool = new ClusterSetup(ZK_ADDR);
    // setup storage cluster
    _setupTool.addCluster(CLUSTER_NAME, true);
    _setupTool.addResourceToCluster(CLUSTER_NAME, TEST_DB, 20, STATE_MODEL);
    for (int i = 0; i < NODE_NR; i++) {
        String storageNodeName = "localhost_" + (START_PORT + i);
        _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
    }
    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, TEST_DB, 3);
}