Examples with Partition org.apache.helix.model.Partition used on opensource projects

Example 16 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class JobRebalancer method updateJobContextAndGetTaskCurrentState.

private TaskPartitionState updateJobContextAndGetTaskCurrentState(CurrentStateOutput currentStateOutput, String jobResource, Integer pId, String pName, String instance, JobContext jobCtx) {
    String currentStateString = currentStateOutput.getCurrentState(jobResource, new Partition(pName), instance);
    if (currentStateString == null) {
        // Task state is either DROPPED or INIT
        return jobCtx.getPartitionState(pId);
    }
    TaskPartitionState currentState = TaskPartitionState.valueOf(currentStateString);
    jobCtx.setPartitionState(pId, currentState);
    String taskMsg = currentStateOutput.getInfo(jobResource, new Partition(pName), instance);
    if (taskMsg != null) {
        jobCtx.setPartitionInfo(pId, taskMsg);
    }
    return currentState;
}

Example 17 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestRebalancePipeline method testDuplicateMsg.

@Test
public void testDuplicateMsg() {
    String clusterName = "CLUSTER_" + _className + "_dup";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    refreshClusterConfig(clusterName, accessor);
    HelixManager manager = new DummyClusterManager(clusterName, accessor);
    ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    final String resourceName = "testResource_dup";
    String[] resourceGroups = new String[] { resourceName };
    // ideal state: node0 is MASTER, node1 is SLAVE
    // replica=2 means 1 master and 1 slave
    setupIdealState(clusterName, new int[] { 0 }, resourceGroups, 1, 1);
    setupLiveInstances(clusterName, new int[] { 0 });
    setupStateModel(clusterName);
    // cluster data cache refresh pipeline
    Pipeline dataRefresh = new Pipeline();
    dataRefresh.addStage(new ReadClusterDataStage());
    // rebalance pipeline
    Pipeline rebalancePipeline = new Pipeline();
    rebalancePipeline.addStage(new ResourceComputationStage());
    rebalancePipeline.addStage(new CurrentStateComputationStage());
    rebalancePipeline.addStage(new BestPossibleStateCalcStage());
    rebalancePipeline.addStage(new IntermediateStateCalcStage());
    rebalancePipeline.addStage(new MessageGenerationPhase());
    rebalancePipeline.addStage(new MessageSelectionStage());
    rebalancePipeline.addStage(new MessageThrottleStage());
    rebalancePipeline.addStage(new TaskAssignmentStage());
    // round1: set node0 currentState to OFFLINE
    setCurrentState(clusterName, "localhost_0", resourceName, resourceName + "_0", "session_0", "OFFLINE");
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    MessageSelectionStageOutput msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: OFFLINE-SLAVE for node0");
    Message message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "OFFLINE");
    Assert.assertEquals(message.getToState(), "SLAVE");
    Assert.assertEquals(message.getTgtName(), "localhost_0");
    // round2: updates node0 currentState to SLAVE but keep the
    // message, make sure controller should not send S->M until removal is done
    setCurrentState(clusterName, "localhost_0", resourceName, resourceName + "_0", "session_1", "SLAVE");
    runPipeline(event, dataRefresh);
    refreshClusterConfig(clusterName, accessor);
    runPipeline(event, rebalancePipeline);
    msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 0, "Should NOT output 1 message: SLAVE-MASTER for node1");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 18 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestStateTransitionPrirority method updateCurrentStatesForRecoveryBalance.

private void updateCurrentStatesForRecoveryBalance(List<String> resourcePriority, CurrentStateOutput currentStateOutput) {
    IntermediateStateOutput output = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    for (PartitionStateMap partitionStateMap : output.getResourceStatesMap().values()) {
        String resourceName = partitionStateMap.getResourceName();
        Partition partition = new Partition(resourceName + "_0");
        String instanceName = HOSTNAME_PREFIX + resourceName.split("_")[1];
        if (partitionStateMap.getPartitionMap(partition).values().contains("SLAVE") && !resourcePriority.contains(resourceName)) {
            updateCurrentOutput(resourcePriority, currentStateOutput, resourceName, partition, instanceName, "SLAVE");
            break;
        }
    }
}

Example 19 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class TestStateTransitionPrirority method updateCurrentStatesForLoadBalance.

private void updateCurrentStatesForLoadBalance(List<String> resourcePriority, CurrentStateOutput currentStateOutput) {
    IntermediateStateOutput output = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    for (PartitionStateMap partitionStateMap : output.getResourceStatesMap().values()) {
        String resourceName = partitionStateMap.getResourceName();
        Partition partition = new Partition(resourceName + "_0");
        String oldInstance = HOSTNAME_PREFIX + resourceName.split("_")[1];
        String expectedInstance = HOSTNAME_PREFIX + (Integer.parseInt(resourceName.split("_")[1]) + 1);
        if (partitionStateMap.getPartitionMap(partition).containsKey(expectedInstance) && !resourcePriority.contains(resourceName)) {
            currentStateOutput.getCurrentStateMap(resourceName, partition).remove(oldInstance);
            updateCurrentOutput(resourcePriority, currentStateOutput, resourceName, partition, expectedInstance, "MASTER");
            break;
        }
    }
}

Example 20 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class BestPossibleExternalViewVerifier method verifyState.

@Override
protected synchronized boolean verifyState() {
    try {
        PropertyKey.Builder keyBuilder = _accessor.keyBuilder();
        // read cluster once and do verification
        ClusterDataCache cache = new ClusterDataCache();
        cache.refresh(_accessor);
        Map<String, IdealState> idealStates = cache.getIdealStates();
        if (idealStates == null) {
            // ideal state is null because ideal state is dropped
            idealStates = Collections.emptyMap();
        }
        // filter out all resources that use Task state model
        Iterator<Map.Entry<String, IdealState>> it = idealStates.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, IdealState> pair = it.next();
            if (pair.getValue().getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME)) {
                it.remove();
            }
        }
        // verify live instances.
        if (_expectLiveInstances != null && !_expectLiveInstances.isEmpty()) {
            Set<String> actualLiveNodes = cache.getLiveInstances().keySet();
            if (!_expectLiveInstances.equals(actualLiveNodes)) {
                LOG.warn("Live instances are not as expected. Actual live nodes: " + actualLiveNodes.toString());
                return false;
            }
        }
        Map<String, ExternalView> extViews = _accessor.getChildValuesMap(keyBuilder.externalViews());
        if (extViews == null) {
            extViews = Collections.emptyMap();
        }
        // Filter resources if requested
        if (_resources != null && !_resources.isEmpty()) {
            idealStates.keySet().retainAll(_resources);
            extViews.keySet().retainAll(_resources);
        }
        // add empty idealState for the resource
        for (String resource : extViews.keySet()) {
            if (!idealStates.containsKey(resource)) {
                ExternalView ev = extViews.get(resource);
                IdealState is = new IdealState(resource);
                is.getRecord().setSimpleFields(ev.getRecord().getSimpleFields());
                idealStates.put(resource, is);
            }
        }
        // calculate best possible state
        BestPossibleStateOutput bestPossOutput = calcBestPossState(cache);
        Map<String, Map<Partition, Map<String, String>>> bestPossStateMap = bestPossOutput.getStateMap();
        // set error states
        if (_errStates != null) {
            for (String resourceName : _errStates.keySet()) {
                Map<String, String> partErrStates = _errStates.get(resourceName);
                for (String partitionName : partErrStates.keySet()) {
                    String instanceName = partErrStates.get(partitionName);
                    if (!bestPossStateMap.containsKey(resourceName)) {
                        bestPossStateMap.put(resourceName, new HashMap<Partition, Map<String, String>>());
                    }
                    Partition partition = new Partition(partitionName);
                    if (!bestPossStateMap.get(resourceName).containsKey(partition)) {
                        bestPossStateMap.get(resourceName).put(partition, new HashMap<String, String>());
                    }
                    bestPossStateMap.get(resourceName).get(partition).put(instanceName, HelixDefinedState.ERROR.toString());
                }
            }
        }
        for (String resourceName : idealStates.keySet()) {
            ExternalView extView = extViews.get(resourceName);
            IdealState is = idealStates.get(resourceName);
            if (extView == null) {
                if (is.isExternalViewDisabled()) {
                    continue;
                } else {
                    LOG.error("externalView for " + resourceName + " is not available");
                    return false;
                }
            }
            // step 0: remove empty map and DROPPED state from best possible state
            PartitionStateMap bpStateMap = bestPossOutput.getPartitionStateMap(resourceName);
            StateModelDefinition stateModelDef = cache.getStateModelDef(is.getStateModelDefRef());
            if (stateModelDef == null) {
                LOG.error("State model definition " + is.getStateModelDefRef() + " for resource not found!" + is.getResourceName());
                return false;
            }
            boolean result = verifyExternalView(extView, bpStateMap, stateModelDef);
            if (!result) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("verifyExternalView fails for " + resourceName + "! ExternalView: " + extView + " BestPossibleState: " + bpStateMap);
                } else {
                    LOG.warn("verifyExternalView fails for " + resourceName + "! ExternalView does not match BestPossibleState");
                }
                return false;
            }
        }
        return true;
    } catch (Exception e) {
        LOG.error("exception in verification", e);
        return false;
    }
}