Examples with Partition org.apache.helix.model.Partition used on opensource projects

Example 36 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestStateTransitionPrirority method testResourceLevelPriorityForLoadBalance.

@Test(dataProvider = "ResourceLevelPriority")
public void testResourceLevelPriorityForLoadBalance(Map<String, String> resourceMap, String priorityField, List<String> expectedPriority) {
    preSetup(StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, resourceMap.keySet(), priorityField, 10, 1);
    event.addAttribute(AttributeName.RESOURCES.name(), getResourceMap(resourceMap.keySet().toArray(new String[resourceMap.keySet().size()]), 1, "MasterSlave"));
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), getResourceMap(resourceMap.keySet().toArray(new String[resourceMap.keySet().size()]), 1, "MasterSlave"));
    // Initialize bestpossible state and current state
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (String resource : resourceMap.keySet()) {
        IdealState is = accessor.getProperty(accessor.keyBuilder().idealStates(resource));
        is.getRecord().setSimpleField(priorityField, resourceMap.get(resource));
        setSingleIdealState(is);
        Map<String, List<String>> partitionMap = new HashMap<String, List<String>>();
        Partition partition = new Partition(resource + "_0");
        String instanceName = HOSTNAME_PREFIX + resource.split("_")[1];
        String nextInstanceName = HOSTNAME_PREFIX + (Integer.parseInt(resource.split("_")[1]) + 1);
        partitionMap.put(partition.getPartitionName(), Collections.singletonList(nextInstanceName));
        bestPossibleStateOutput.setPreferenceLists(resource, partitionMap);
        bestPossibleStateOutput.setState(resource, partition, nextInstanceName, "MASTER");
        currentStateOutput.setCurrentState(resource, partition, instanceName, "MASTER");
    }
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    runStage(event, new ReadClusterDataStage());
    // Keep update the current state.
    List<String> resourcePriority = new ArrayList<String>();
    for (int i = 0; i < resourceMap.size(); i++) {
        runStage(event, new IntermediateStateCalcStage());
        updateCurrentStatesForLoadBalance(resourcePriority, currentStateOutput);
    }
    Assert.assertEquals(resourcePriority, expectedPriority);
}

Example 37 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestPersistAssignmentStage method testSimple.

/**
 * Case where we have one resource in IdealState
 * @throws Exception
 */
@Test
public void testSimple() throws Exception {
    int nodes = 2;
    List<String> instances = new ArrayList<String>();
    for (int i = 0; i < nodes; i++) {
        instances.add("localhost_" + i);
    }
    int partitions = 10;
    int replicas = 1;
    String resourceName = "testResource";
    ZNRecord record = DefaultIdealStateCalculator.calculateIdealState(instances, partitions, replicas, resourceName, "ONLINE", "OFFLINE");
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef("OnlineOffline");
    // Read and load current state into event
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
    runStage(_manager, event, new ReadClusterDataStage());
    runStage(_manager, event, new ResourceComputationStage());
    // Ensure persist best possible assignment is true
    ClusterConfig clusterConfig = new ClusterConfig(CLUSTER_NAME);
    clusterConfig.setPersistBestPossibleAssignment(true);
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.setClusterConfig(clusterConfig);
    // 1. Change best possible state (simulate a new rebalancer run)
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    for (String partition : idealState.getPartitionSet()) {
        bestPossibleStateOutput.setState(resourceName, new Partition(partition), "localhost_3", "OFFLINE");
    }
    // 2. At the same time, set DelayRebalanceEnabled = true (simulate a Admin operation at the same time)
    idealState.setDelayRebalanceEnabled(true);
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
    // Persist new assignment
    PersistAssignmentStage stage = new PersistAssignmentStage();
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    runStage(_manager, event, stage);
    IdealState newIdealState = accessor.getProperty(keyBuilder.idealStates(resourceName));
    // 1. New assignment should be set
    Assert.assertEquals(newIdealState.getPartitionSet().size(), idealState.getPartitionSet().size());
    for (String partition : idealState.getPartitionSet()) {
        Map<String, String> assignment = newIdealState.getInstanceStateMap(partition);
        Assert.assertNotNull(assignment);
        Assert.assertEquals(assignment.size(), 1);
        Assert.assertTrue(assignment.containsKey("localhost_3") && assignment.get("localhost_3").equals("OFFLINE"));
    }
    // 2. Admin config should be set
    Assert.assertTrue(newIdealState.isDelayRebalanceEnabled());
}

Example 38 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class DeprecatedTaskRebalancer method emptyAssignment.

private static ResourceAssignment emptyAssignment(String name, CurrentStateOutput currStateOutput) {
    ResourceAssignment assignment = new ResourceAssignment(name);
    Set<Partition> partitions = currStateOutput.getCurrentStateMappedPartitions(name);
    for (Partition partition : partitions) {
        Map<String, String> currentStateMap = currStateOutput.getCurrentStateMap(name, partition);
        Map<String, String> replicaMap = Maps.newHashMap();
        for (String instanceName : currentStateMap.keySet()) {
            replicaMap.put(instanceName, HelixDefinedState.DROPPED.toString());
        }
        assignment.addReplicaMap(partition, replicaMap);
    }
    return assignment;
}

Example 39 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class JobRebalancer method processTaskWithPendingMessage.

private void processTaskWithPendingMessage(ResourceAssignment prevAssignment, Integer pId, String pName, String instance, Message pendingMessage, TaskState jobState, TaskPartitionState currState, Map<Integer, PartitionAssignment> paMap, Set<Integer> assignedPartitions) {
    Map<String, String> stateMap = prevAssignment.getReplicaMap(new Partition(pName));
    if (stateMap != null) {
        String prevState = stateMap.get(instance);
        if (!pendingMessage.getToState().equals(prevState)) {
            LOG.warn(String.format("Task pending to-state is %s while previous assigned state is %s. This should not" + "heppen.", pendingMessage.getToState(), prevState));
        }
        if (jobState == TaskState.TIMING_OUT && currState == TaskPartitionState.INIT && prevState.equals(TaskPartitionState.RUNNING.name())) {
            // While job is timing out, if the task is pending on INIT->RUNNING, set it back to INIT,
            // so that Helix will cancel the transition.
            paMap.put(pId, new PartitionAssignment(instance, TaskPartitionState.INIT.name()));
            assignedPartitions.add(pId);
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("Task partition %s has a pending state transition on instance %s INIT->RUNNING. " + "Setting it back to INIT so that Helix can cancel the transition(if enabled).", pName, instance, prevState));
            }
        } else {
            // Otherwise, Just copy forward
            // the state assignment from the previous ideal state.
            paMap.put(pId, new PartitionAssignment(instance, prevState));
            assignedPartitions.add(pId);
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("Task partition %s has a pending state transition on instance %s. Using the previous ideal state which was %s.", pName, instance, prevState));
            }
        }
    }
}

Example 40 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class JobRebalancer method isJobFinished.

private boolean isJobFinished(JobContext jobContext, String jobResource, CurrentStateOutput currentStateOutput) {
    for (int pId : jobContext.getPartitionSet()) {
        TaskPartitionState state = jobContext.getPartitionState(pId);
        Partition partition = new Partition(pName(jobResource, pId));
        String instance = jobContext.getAssignedParticipant(pId);
        Message pendingMessage = currentStateOutput.getPendingState(jobResource, partition, instance);
        // If state is INIT but is pending INIT->RUNNING, it's not yet safe to say the job finished
        if (state == TaskPartitionState.RUNNING || (state == TaskPartitionState.INIT && pendingMessage != null)) {
            return false;
        }
    }
    return true;
}