Examples with ClusterConfig org.apache.helix.model.ClusterConfig used on opensource projects

Example 36 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class StrictMatchExternalViewVerifier method verifyExternalView.

private boolean verifyExternalView(ClusterDataCache dataCache, ExternalView externalView, IdealState idealState) {
    Map<String, Map<String, String>> mappingInExtview = externalView.getRecord().getMapFields();
    Map<String, Map<String, String>> idealPartitionState;
    switch(idealState.getRebalanceMode()) {
        case FULL_AUTO:
            ClusterConfig clusterConfig = new ConfigAccessor(_zkClient).getClusterConfig(dataCache.getClusterName());
            if (!clusterConfig.isPersistBestPossibleAssignment() && !clusterConfig.isPersistIntermediateAssignment()) {
                throw new HelixException(String.format("Full-Auto IdealState verifier requires " + "ClusterConfig.PERSIST_BEST_POSSIBLE_ASSIGNMENT or ClusterConfig.PERSIST_INTERMEDIATE_ASSIGNMENT " + "is enabled."));
            }
            for (String partition : idealState.getPartitionSet()) {
                if (idealState.getPreferenceList(partition) == null || idealState.getPreferenceList(partition).isEmpty()) {
                    return false;
                }
            }
            idealPartitionState = computeIdealPartitionState(dataCache, idealState);
            break;
        case SEMI_AUTO:
        case USER_DEFINED:
            idealPartitionState = computeIdealPartitionState(dataCache, idealState);
            break;
        case CUSTOMIZED:
            idealPartitionState = idealState.getRecord().getMapFields();
            break;
        case TASK:
        // ignore jobs
        default:
            return true;
    }
    return mappingInExtview.equals(idealPartitionState);
}

Example 37 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestIntermediateStateCalcStage method preSetup.

private void preSetup(StateTransitionThrottleConfig.RebalanceType rebalanceType, Set<String> resourceSet, int numOfLiveInstances, int numOfReplicas) {
    setupIdealState(numOfLiveInstances, resourceSet.toArray(new String[resourceSet.size()]), numOfLiveInstances, numOfReplicas, IdealState.RebalanceMode.FULL_AUTO, "OnlineOffline");
    setupStateModel();
    setupLiveInstances(numOfLiveInstances);
    // Set up cluster configs
    ClusterConfig clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    StateTransitionThrottleConfig throttleConfig = new StateTransitionThrottleConfig(rebalanceType, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, Integer.MAX_VALUE);
    clusterConfig.setStateTransitionThrottleConfigs(Collections.singletonList(throttleConfig));
    setClusterConfig(clusterConfig);
}

Example 38 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestRebalancePipeline method testChangeIdealStateWithPendingMsg.

@Test
public void testChangeIdealStateWithPendingMsg() {
    String clusterName = "CLUSTER_" + _className + "_pending";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    HelixManager manager = new DummyClusterManager(clusterName, accessor);
    ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    ClusterDataCache cache = new ClusterDataCache();
    event.addAttribute(AttributeName.ClusterDataCache.name(), cache);
    refreshClusterConfig(clusterName, accessor);
    final String resourceName = "testResource_pending";
    String[] resourceGroups = new String[] { resourceName };
    // ideal state: node0 is MASTER, node1 is SLAVE
    // replica=2 means 1 master and 1 slave
    setupIdealState(clusterName, new int[] { 0 }, resourceGroups, 1, 1);
    setupLiveInstances(clusterName, new int[] { 0 });
    setupStateModel(clusterName);
    // cluster data cache refresh pipeline
    Pipeline dataRefresh = new Pipeline();
    dataRefresh.addStage(new ReadClusterDataStage());
    // rebalance pipeline
    Pipeline rebalancePipeline = new Pipeline();
    rebalancePipeline.addStage(new ResourceComputationStage());
    rebalancePipeline.addStage(new CurrentStateComputationStage());
    rebalancePipeline.addStage(new BestPossibleStateCalcStage());
    rebalancePipeline.addStage(new IntermediateStateCalcStage());
    rebalancePipeline.addStage(new MessageGenerationPhase());
    rebalancePipeline.addStage(new MessageSelectionStage());
    rebalancePipeline.addStage(new MessageThrottleStage());
    rebalancePipeline.addStage(new TaskAssignmentStage());
    // round1: set node0 currentState to OFFLINE and node1 currentState to SLAVE
    setCurrentState(clusterName, "localhost_0", resourceName, resourceName + "_0", "session_0", "OFFLINE");
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    MessageSelectionStageOutput msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: OFFLINE-SLAVE for node0");
    Message message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "OFFLINE");
    Assert.assertEquals(message.getToState(), "SLAVE");
    Assert.assertEquals(message.getTgtName(), "localhost_0");
    // round2: drop resource, but keep the
    // message, make sure controller should not send O->DROPPED until O->S is done
    HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    admin.dropResource(clusterName, resourceName);
    List<IdealState> idealStates = accessor.getChildValues(accessor.keyBuilder().idealStates());
    cache.setIdealStates(idealStates);
    runPipeline(event, dataRefresh);
    cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.setClusterConfig(new ClusterConfig(clusterName));
    runPipeline(event, rebalancePipeline);
    msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 0, "Should not output only 1 message: OFFLINE->DROPPED for localhost_0");
    // round3: remove O->S message for localhost_0, localhost_0 still in OFFLINE
    // controller should now send O->DROPPED to localhost_0
    Builder keyBuilder = accessor.keyBuilder();
    List<String> msgIds = accessor.getChildNames(keyBuilder.messages("localhost_0"));
    accessor.removeProperty(keyBuilder.message("localhost_0", msgIds.get(0)));
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: OFFLINE->DROPPED for localhost_0");
    message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "OFFLINE");
    Assert.assertEquals(message.getToState(), "DROPPED");
    Assert.assertEquals(message.getTgtName(), "localhost_0");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 39 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestResourceValidationStage method testIdealStateValidity.

@Test
public void testIdealStateValidity() throws Exception {
    MockAccessor accessor = new MockAccessor();
    accessor.setProperty(accessor.keyBuilder().clusterConfig(), new ClusterConfig("TestCluster"));
    // create some ideal states
    String masterSlaveCustomResource = "masterSlaveCustomResource";
    String onlineOfflineFullAutoResource = "onlineOfflineFullAutoResource";
    String masterSlaveSemiAutoInvalidResource = "masterSlaveSemiAutoInvalidResource";
    createIS(accessor, masterSlaveCustomResource, "MasterSlave", RebalanceMode.CUSTOMIZED);
    createIS(accessor, onlineOfflineFullAutoResource, "OnlineOffline", RebalanceMode.FULL_AUTO);
    createIS(accessor, masterSlaveSemiAutoInvalidResource, "MasterSlave", RebalanceMode.SEMI_AUTO);
    // create some ideal state specs
    createISSpec(accessor, masterSlaveCustomResource + "_spec", "MasterSlave", RebalanceMode.CUSTOMIZED);
    createISSpec(accessor, onlineOfflineFullAutoResource + "_spec", "OnlineOffline", RebalanceMode.FULL_AUTO);
    // add some state models
    addStateModels(accessor);
    // refresh the cache
    ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
    ClusterDataCache cache = new ClusterDataCache();
    cache.refresh(accessor);
    event.addAttribute(AttributeName.ClusterDataCache.name(), cache);
    // run resource computation
    new ResourceComputationStage().process(event);
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    Assert.assertTrue(resourceMap.containsKey(masterSlaveCustomResource));
    Assert.assertTrue(resourceMap.containsKey(onlineOfflineFullAutoResource));
    Assert.assertTrue(resourceMap.containsKey(masterSlaveSemiAutoInvalidResource));
    // run resource validation
    new ResourceValidationStage().process(event);
    Map<String, Resource> finalResourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    Assert.assertTrue(finalResourceMap.containsKey(masterSlaveCustomResource));
    Assert.assertTrue(finalResourceMap.containsKey(onlineOfflineFullAutoResource));
    Assert.assertFalse(finalResourceMap.containsKey(masterSlaveSemiAutoInvalidResource));
}

Example 40 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestStateTransitionPrirority method testPartitionLevelPriority.

@Test(dataProvider = "PartitionLevelPriority")
public void testPartitionLevelPriority(String resourceName, Map<String, Map<String, String>> bestPossibleMap, Map<String, Map<String, String>> currentStateMap, List<String> preferenceList, List<String> expectedPriority) {
    preSetup(StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, new HashSet<String>(Arrays.asList(resourceName)), "no_field", 3, 3);
    // Add load rebalance throttle config
    ClusterConfig clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    StateTransitionThrottleConfig throttleConfigForLoadRebalance = new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 1);
    List<StateTransitionThrottleConfig> currentThrottleConfig = clusterConfig.getStateTransitionThrottleConfigs();
    currentThrottleConfig.add(throttleConfigForLoadRebalance);
    clusterConfig.setStateTransitionThrottleConfigs(currentThrottleConfig);
    setClusterConfig(clusterConfig);
    // Initialize best possible state, current state and resource map.
    Resource resource = new Resource(resourceName);
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (String partitionName : bestPossibleMap.keySet()) {
        Partition partition = new Partition(partitionName);
        bestPossibleStateOutput.setPreferenceList(resourceName, partitionName, preferenceList);
        for (String instanceName : bestPossibleMap.get(partitionName).keySet()) {
            bestPossibleStateOutput.setState(resourceName, partition, instanceName, bestPossibleMap.get(partitionName).get(instanceName));
            currentStateOutput.setCurrentState(resourceName, partition, instanceName, currentStateMap.get(partitionName).get(instanceName));
        }
        resource.addPartition(partitionName);
    }
    resource.setStateModelDefRef("MasterSlave");
    event.addAttribute(AttributeName.RESOURCES.name(), Collections.singletonMap(resourceName, resource));
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resource);
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    runStage(event, new ReadClusterDataStage());
    // Keep update the current state.
    List<String> partitionPriority = new ArrayList<String>();
    for (int i = 0; i < bestPossibleMap.size(); i++) {
        runStage(event, new IntermediateStateCalcStage());
        updateCurrentStateForPartitionLevelPriority(partitionPriority, currentStateOutput, resourceName, bestPossibleMap);
    }
    Assert.assertEquals(partitionPriority, expectedPriority);
}