Examples with Partition org.apache.helix.model.Partition used on opensource projects

Example 31 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestRebalancePipeline method testMasterXfer.

@Test
public void testMasterXfer() {
    String clusterName = "CLUSTER_" + _className + "_xfer";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    HelixManager manager = new DummyClusterManager(clusterName, accessor);
    ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    refreshClusterConfig(clusterName, accessor);
    final String resourceName = "testResource_xfer";
    String[] resourceGroups = new String[] { resourceName };
    // ideal state: node0 is MASTER, node1 is SLAVE
    // replica=2 means 1 master and 1 slave
    setupIdealState(clusterName, new int[] { 0, 1 }, resourceGroups, 1, 2);
    setupLiveInstances(clusterName, new int[] { 1 });
    setupStateModel(clusterName);
    // cluster data cache refresh pipeline
    Pipeline dataRefresh = new Pipeline();
    dataRefresh.addStage(new ReadClusterDataStage());
    // rebalance pipeline
    Pipeline rebalancePipeline = new Pipeline();
    rebalancePipeline.addStage(new ResourceComputationStage());
    rebalancePipeline.addStage(new CurrentStateComputationStage());
    rebalancePipeline.addStage(new BestPossibleStateCalcStage());
    rebalancePipeline.addStage(new IntermediateStateCalcStage());
    rebalancePipeline.addStage(new MessageGenerationPhase());
    rebalancePipeline.addStage(new MessageSelectionStage());
    rebalancePipeline.addStage(new MessageThrottleStage());
    rebalancePipeline.addStage(new TaskAssignmentStage());
    // round1: set node1 currentState to SLAVE
    setCurrentState(clusterName, "localhost_1", resourceName, resourceName + "_0", "session_1", "SLAVE");
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    MessageSelectionStageOutput msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: SLAVE-MASTER for node1");
    Message message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "SLAVE");
    Assert.assertEquals(message.getToState(), "MASTER");
    Assert.assertEquals(message.getTgtName(), "localhost_1");
    // round2: updates node0 currentState to SLAVE but keep the
    // message, make sure controller should not send S->M until removal is done
    setupLiveInstances(clusterName, new int[] { 0 });
    setCurrentState(clusterName, "localhost_0", resourceName, resourceName + "_0", "session_0", "SLAVE");
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 0, "Should NOT output 1 message: SLAVE-MASTER for node0");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 32 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestRebalancePipeline method testNoDuplicatedMaster.

@Test
public void testNoDuplicatedMaster() {
    String clusterName = "CLUSTER_" + _className + "_no_duplicated_master";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    HelixManager manager = new DummyClusterManager(clusterName, accessor);
    ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    refreshClusterConfig(clusterName, accessor);
    final String resourceName = "testResource_no_duplicated_master";
    String[] resourceGroups = new String[] { resourceName };
    // ideal state: node0 is SLAVE, node1 is MASTER
    // replica=2 means 1 master and 1 slave
    setupIdealState(clusterName, new int[] { 0, 1 }, resourceGroups, 1, 2);
    setupLiveInstances(clusterName, new int[] { 0, 1 });
    setupStateModel(clusterName);
    // cluster data cache refresh pipeline
    Pipeline dataRefresh = new Pipeline();
    dataRefresh.addStage(new ReadClusterDataStage());
    // rebalance pipeline
    Pipeline rebalancePipeline = new Pipeline();
    rebalancePipeline.addStage(new ResourceComputationStage());
    rebalancePipeline.addStage(new CurrentStateComputationStage());
    rebalancePipeline.addStage(new BestPossibleStateCalcStage());
    rebalancePipeline.addStage(new IntermediateStateCalcStage());
    rebalancePipeline.addStage(new MessageGenerationPhase());
    rebalancePipeline.addStage(new MessageSelectionStage());
    rebalancePipeline.addStage(new MessageThrottleStage());
    rebalancePipeline.addStage(new TaskAssignmentStage());
    // set node0 currentState to SLAVE, node1 currentState to MASTER
    // Helix will try to switch the state of the two instances, but it should not be two MASTER at the same time
    // so it should first transit M->S, then transit another instance S->M
    setCurrentState(clusterName, "localhost_0", resourceName, resourceName + "_0", "session_0", "SLAVE");
    setCurrentState(clusterName, "localhost_1", resourceName, resourceName + "_0", "session_1", "MASTER");
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    MessageSelectionStageOutput msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: MASTER-SLAVE for localhost_1");
    Message message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "MASTER");
    Assert.assertEquals(message.getToState(), "SLAVE");
    Assert.assertEquals(message.getTgtName(), "localhost_1");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 33 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestStateTransitionPrirority method updateCurrentStateForPartitionLevelPriority.

private void updateCurrentStateForPartitionLevelPriority(List<String> partitionPriority, CurrentStateOutput currentStateOutput, String resourceName, Map<String, Map<String, String>> bestPossibleMap) {
    IntermediateStateOutput output = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    PartitionStateMap partitionStateMap = output.getPartitionStateMap(resourceName);
    for (Partition partition : partitionStateMap.getStateMap().keySet()) {
        Map<String, String> instanceStateMap = bestPossibleMap.get(partition.getPartitionName());
        if (partitionStateMap.getPartitionMap(partition).equals(instanceStateMap) && !partitionPriority.contains(partition.getPartitionName())) {
            partitionPriority.add(partition.getPartitionName());
            for (String instanceName : instanceStateMap.keySet()) {
                currentStateOutput.setCurrentState(resourceName, partition, instanceName, instanceStateMap.get(instanceName));
            }
            break;
        }
    }
}

Example 34 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestStateTransitionPrirority method testPartitionLevelPriority.

@Test(dataProvider = "PartitionLevelPriority")
public void testPartitionLevelPriority(String resourceName, Map<String, Map<String, String>> bestPossibleMap, Map<String, Map<String, String>> currentStateMap, List<String> preferenceList, List<String> expectedPriority) {
    preSetup(StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, new HashSet<String>(Arrays.asList(resourceName)), "no_field", 3, 3);
    // Add load rebalance throttle config
    ClusterConfig clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    StateTransitionThrottleConfig throttleConfigForLoadRebalance = new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 1);
    List<StateTransitionThrottleConfig> currentThrottleConfig = clusterConfig.getStateTransitionThrottleConfigs();
    currentThrottleConfig.add(throttleConfigForLoadRebalance);
    clusterConfig.setStateTransitionThrottleConfigs(currentThrottleConfig);
    setClusterConfig(clusterConfig);
    // Initialize best possible state, current state and resource map.
    Resource resource = new Resource(resourceName);
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (String partitionName : bestPossibleMap.keySet()) {
        Partition partition = new Partition(partitionName);
        bestPossibleStateOutput.setPreferenceList(resourceName, partitionName, preferenceList);
        for (String instanceName : bestPossibleMap.get(partitionName).keySet()) {
            bestPossibleStateOutput.setState(resourceName, partition, instanceName, bestPossibleMap.get(partitionName).get(instanceName));
            currentStateOutput.setCurrentState(resourceName, partition, instanceName, currentStateMap.get(partitionName).get(instanceName));
        }
        resource.addPartition(partitionName);
    }
    resource.setStateModelDefRef("MasterSlave");
    event.addAttribute(AttributeName.RESOURCES.name(), Collections.singletonMap(resourceName, resource));
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resource);
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    runStage(event, new ReadClusterDataStage());
    // Keep update the current state.
    List<String> partitionPriority = new ArrayList<String>();
    for (int i = 0; i < bestPossibleMap.size(); i++) {
        runStage(event, new IntermediateStateCalcStage());
        updateCurrentStateForPartitionLevelPriority(partitionPriority, currentStateOutput, resourceName, bestPossibleMap);
    }
    Assert.assertEquals(partitionPriority, expectedPriority);
}

Example 35 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestStateTransitionPrirority method testResourceLevelPriorityForRecoveryBalance.

// TODO : Reenable this when throttling enabled for recovery rebalance
@Test(dataProvider = "ResourceLevelPriority", enabled = false)
public void testResourceLevelPriorityForRecoveryBalance(Map<String, String> resourceMap, String priorityField, List<String> expectedPriority) {
    preSetup(StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, resourceMap.keySet(), priorityField, 10, 1);
    event.addAttribute(AttributeName.RESOURCES.name(), getResourceMap(resourceMap.keySet().toArray(new String[resourceMap.keySet().size()]), 1, "MasterSlave"));
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), getResourceMap(resourceMap.keySet().toArray(new String[resourceMap.keySet().size()]), 1, "MasterSlave"));
    // Initialize bestpossible state and current state
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (String resource : resourceMap.keySet()) {
        IdealState is = accessor.getProperty(accessor.keyBuilder().idealStates(resource));
        is.getRecord().setSimpleField(priorityField, resourceMap.get(resource));
        setSingleIdealState(is);
        Map<String, List<String>> partitionMap = new HashMap<String, List<String>>();
        Partition partition = new Partition(resource + "_0");
        String instanceName = HOSTNAME_PREFIX + resource.split("_")[1];
        partitionMap.put(partition.getPartitionName(), Collections.singletonList(instanceName));
        bestPossibleStateOutput.setPreferenceLists(resource, partitionMap);
        bestPossibleStateOutput.setState(resource, partition, instanceName, "SLAVE");
        currentStateOutput.setCurrentState(resource, partition, instanceName, "OFFLINE");
    }
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    runStage(event, new ReadClusterDataStage());
    // Keep update the current state.
    List<String> resourcePriority = new ArrayList<String>();
    for (int i = 0; i < resourceMap.size(); i++) {
        runStage(event, new IntermediateStateCalcStage());
        updateCurrentStatesForRecoveryBalance(resourcePriority, currentStateOutput);
    }
    Assert.assertEquals(resourcePriority, expectedPriority);
}