Examples with ClusterConfig org.apache.helix.model.ClusterConfig used on opensource projects

Example 16 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestTopology method testCreateClusterTopology.

@Test
public void testCreateClusterTopology() {
    ClusterConfig clusterConfig = new ClusterConfig("Test_Cluster");
    String topology = "/Rack/Sub-Rack/Host/Instance";
    clusterConfig.setTopology(topology);
    clusterConfig.setFaultZoneType("Sub-Rack");
    clusterConfig.setTopologyAwareEnabled(true);
    List<String> allNodes = new ArrayList<String>();
    List<String> liveNodes = new ArrayList<String>();
    Map<String, InstanceConfig> instanceConfigMap = new HashMap<String, InstanceConfig>();
    Map<String, Integer> nodeToWeightMap = new HashMap<String, Integer>();
    for (int i = 0; i < 100; i++) {
        String instance = "localhost_" + i;
        InstanceConfig config = new InstanceConfig(instance);
        String rack_id = "rack_" + i / 25;
        String sub_rack_id = "subrack-" + i / 5;
        String domain = String.format("Rack=%s, Sub-Rack=%s, Host=%s", rack_id, sub_rack_id, instance);
        config.setDomain(domain);
        config.setHostName(instance);
        config.setPort("9000");
        allNodes.add(instance);
        int weight = 0;
        if (i % 10 != 0) {
            liveNodes.add(instance);
            weight = 1000;
            if (i % 3 == 0) {
                // set random instance weight.
                weight = (i + 1) * 100;
                config.setWeight(weight);
            }
        }
        instanceConfigMap.put(instance, config);
        if (!nodeToWeightMap.containsKey(rack_id)) {
            nodeToWeightMap.put(rack_id, 0);
        }
        nodeToWeightMap.put(rack_id, nodeToWeightMap.get(rack_id) + weight);
        if (!nodeToWeightMap.containsKey(sub_rack_id)) {
            nodeToWeightMap.put(sub_rack_id, 0);
        }
        nodeToWeightMap.put(sub_rack_id, nodeToWeightMap.get(sub_rack_id) + weight);
    }
    Topology topo = new Topology(allNodes, liveNodes, instanceConfigMap, clusterConfig);
    Assert.assertTrue(topo.getEndNodeType().equals("Instance"));
    Assert.assertTrue(topo.getFaultZoneType().equals("Sub-Rack"));
    List<Node> faultZones = topo.getFaultZones();
    Assert.assertEquals(faultZones.size(), 20);
    Node root = topo.getRootNode();
    Assert.assertEquals(root.getChildrenCount("Rack"), 4);
    Assert.assertEquals(root.getChildrenCount("Sub-Rack"), 20);
    Assert.assertEquals(root.getChildrenCount("Host"), 100);
    Assert.assertEquals(root.getChildrenCount("Instance"), 100);
    // validate weights.
    for (Node rack : root.getChildren()) {
        Assert.assertEquals(rack.getWeight(), (long) nodeToWeightMap.get(rack.getName()));
        for (Node subRack : rack.getChildren()) {
            Assert.assertEquals(subRack.getWeight(), (long) nodeToWeightMap.get(subRack.getName()));
        }
    }
}

Example 17 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestStateTransitionCancellation method testDisableCancellationWhenDisableResource.

@Test
public void testDisableCancellationWhenDisableResource() throws InterruptedException {
    // Disable cancellation
    ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.stateTransitionCancelEnabled(false);
    _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
    // Reenable resource
    stateCleanUp();
    _setupTool.getClusterManagementTool().enableResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, true);
    // Wait for assignment done
    Thread.sleep(2000);
    // Disable the resource
    _setupTool.getClusterManagementTool().enableResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, false);
    // Wait for pipeline reaching final stage
    Thread.sleep(2000L);
    ExternalView externalView = _setupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
    for (String partition : externalView.getPartitionSet()) {
        Assert.assertTrue(externalView.getStateMap(partition).values().contains("SLAVE"));
    }
}

Example 18 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestStateTransitionCancellation method testRebalancingCauseCancellation.

@Test
public void testRebalancingCauseCancellation() throws InterruptedException {
    // Enable cancellation
    ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.stateTransitionCancelEnabled(true);
    clusterConfig.setPersistBestPossibleAssignment(true);
    _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
    // Reenable resource
    stateCleanUp();
    _setupTool.getClusterManagementTool().enableResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, true);
    // Wait for assignment done
    Thread.sleep(2000);
    int numNodesToStart = 10;
    for (int i = 0; i < numNodesToStart; i++) {
        String storageNodeName = PARTICIPANT_PREFIX + "_" + (_startPort + _numNodes + i);
        _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
    }
    MockParticipantManager[] newParticipants = new MockParticipantManager[numNodesToStart];
    registerParticipants(newParticipants, numNodesToStart, _startPort + _numNodes, 1000, -3000000L);
    // Wait for pipeline reaching final stage
    Thread.sleep(2000L);
    int numOfMasters = 0;
    ExternalView externalView = _setupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
    for (String partition : externalView.getPartitionSet()) {
        if (externalView.getStateMap(partition).values().contains("MASTER")) {
            numOfMasters++;
        }
    }
    for (MockParticipantManager participant : newParticipants) {
        participant.syncStop();
    }
    // Either partial of state transitions have been cancelled or all the Slave -> Master
    // reassigned to other cluster
    Assert.assertTrue((numOfMasters > 0 && numOfMasters <= _numParitions));
}

Example 19 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestStateTransitionCancellation method testCancellationWhenDisableResource.

@Test
public void testCancellationWhenDisableResource() throws InterruptedException {
    // Enable cancellation
    ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.stateTransitionCancelEnabled(true);
    _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
    // Wait for assignment done
    Thread.sleep(2000);
    // Disable the resource
    _setupTool.getClusterManagementTool().enableResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, false);
    // Wait for pipeline reaching final stage
    Thread.sleep(2000L);
    ExternalView externalView = _setupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
    for (String partition : externalView.getPartitionSet()) {
        for (String currentState : externalView.getStateMap(partition).values()) {
            Assert.assertEquals(currentState, "OFFLINE");
        }
    }
}

Example 20 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestStateTransitionThrottle method testTransitionThrottleOnErrorPartition.

@Test
public void testTransitionThrottleOnErrorPartition() throws Exception {
    String clusterName = getShortClassName() + "testMaxErrorPartition";
    MockParticipantManager[] participants = new MockParticipantManager[participantCount];
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    final ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    setupCluster(clusterName, accessor);
    // Set throttle config to enable throttling
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    ClusterConfig clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    clusterConfig.setResourcePriorityField("Name");
    List<StateTransitionThrottleConfig> throttleConfigs = new ArrayList<>();
    throttleConfigs.add(new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 100));
    throttleConfigs.add(new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 100));
    clusterConfig.setStateTransitionThrottleConfigs(throttleConfigs);
    accessor.setProperty(keyBuilder.clusterConfig(), clusterConfig);
    // set one partition to be always Error, so load balance won't be triggered
    Map<String, Set<String>> errPartitions = new HashMap<>();
    errPartitions.put("OFFLINE-SLAVE", TestHelper.setOf(resourceName + "_0"));
    // start part of participants
    for (int i = 0; i < participantCount - 1; i++) {
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, "localhost_" + (12918 + i));
        if (i == 0) {
            participants[i].setTransition(new ErrTransition(errPartitions));
        }
        participants[i].syncStart();
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    BestPossibleExternalViewVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkClient(_gZkClient).build();
    Assert.assertTrue(verifier.verify(3000));
    // Adding one more participant.
    participants[participantCount - 1] = new MockParticipantManager(ZK_ADDR, clusterName, "localhost_" + (12918 + participantCount - 1));
    participants[participantCount - 1].syncStart();
    // Since error partition exists, no load balance transition will be done
    Assert.assertFalse(pollForPartitionAssignment(accessor, participants[participantCount - 1], resourceName, 5000));
    // Update cluster config to tolerate error partition, so load balance transition will be done
    clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    clusterConfig.setErrorPartitionThresholdForLoadBalance(1);
    accessor.setProperty(keyBuilder.clusterConfig(), clusterConfig);
    _gSetupTool.rebalanceResource(clusterName, resourceName, 3);
    Assert.assertTrue(pollForPartitionAssignment(accessor, participants[participantCount - 1], resourceName, 3000));
    // clean up
    controller.syncStop();
    for (int i = 0; i < participantCount; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}