Examples with Resource org.apache.helix.model.Resource used on opensource projects

Example 6 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class CurrentStateComputationStage method updateMissingTopStateStatus.

private void updateMissingTopStateStatus(ClusterDataCache cache, ClusterStatusMonitor clusterStatusMonitor, Map<String, Resource> resourceMap, CurrentStateOutput currentStateOutput) {
    Map<String, Map<String, Long>> missingTopStateMap = cache.getMissingTopStateMap();
    long durationThreshold = Long.MAX_VALUE;
    if (cache.getClusterConfig() != null) {
        durationThreshold = cache.getClusterConfig().getMissTopStateDurationThreshold();
    }
    for (Resource resource : resourceMap.values()) {
        StateModelDefinition stateModelDef = cache.getStateModelDef(resource.getStateModelDefRef());
        if (stateModelDef == null || resource.getStateModelDefRef().equalsIgnoreCase(TASK_STATE_MODEL_NAME)) {
            // Resource does not have valid statemodel or it is task state model
            continue;
        }
        for (Partition partition : resource.getPartitions()) {
            Map<String, String> stateMap = currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
            // Missing top state need to record
            if (!stateMap.values().contains(stateModelDef.getTopState()) && (!missingTopStateMap.containsKey(resource.getResourceName()) || !missingTopStateMap.get(resource.getResourceName()).containsKey(partition.getPartitionName()))) {
                reportNewTopStateMissing(cache, stateMap, missingTopStateMap, resource, partition, stateModelDef.getTopState());
            }
            // The first time participant started or controller switched will be ignored
            if (missingTopStateMap.containsKey(resource.getResourceName()) && missingTopStateMap.get(resource.getResourceName()).containsKey(partition.getPartitionName()) && stateMap.values().contains(stateModelDef.getTopState())) {
                reportTopStateComesBack(cache, stateMap, missingTopStateMap, resource, partition, clusterStatusMonitor, durationThreshold, stateModelDef.getTopState());
            }
        }
    }
    // Check whether it is already passed threshold
    for (String resourceName : missingTopStateMap.keySet()) {
        for (String partitionName : missingTopStateMap.get(resourceName).keySet()) {
            long startTime = missingTopStateMap.get(resourceName).get(partitionName);
            if (startTime > 0 && System.currentTimeMillis() - startTime > durationThreshold) {
                missingTopStateMap.get(resourceName).put(partitionName, TRANSITION_FAILED);
                if (clusterStatusMonitor != null) {
                    clusterStatusMonitor.updateMissingTopStateDurationStats(resourceName, 0L, false);
                }
            }
        }
    }
    if (clusterStatusMonitor != null) {
        clusterStatusMonitor.resetMaxMissingTopStateGauge();
    }
}

Example 7 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class ResourceValidationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache");
    }
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (resourceMap == null) {
        throw new StageException("Resources must be computed prior to validation!");
    }
    Map<String, IdealState> idealStateMap = cache.getIdealStates();
    Map<String, Map<String, String>> idealStateRuleMap = cache.getIdealStateRules();
    for (String resourceName : idealStateMap.keySet()) {
        // check every ideal state against the ideal state rules
        // the pipeline should not process any resources that have an unsupported ideal state
        IdealState idealState = idealStateMap.get(resourceName);
        if (!idealStateRuleMap.isEmpty()) {
            boolean hasMatchingRule = false;
            for (String ruleName : idealStateRuleMap.keySet()) {
                Map<String, String> rule = idealStateRuleMap.get(ruleName);
                boolean matches = idealStateMatchesRule(idealState, rule);
                hasMatchingRule = hasMatchingRule || matches;
                if (matches) {
                    break;
                }
            }
            if (!hasMatchingRule) {
                LOG.warn("Resource " + resourceName + " does not have a valid ideal state!");
                resourceMap.remove(resourceName);
            }
        }
        // check that every resource to process has a live state model definition
        String stateModelDefRef = idealState.getStateModelDefRef();
        StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefRef);
        if (stateModelDef == null) {
            LOG.warn("Resource " + resourceName + " uses state model " + stateModelDefRef + ", but it is not on the cluster!");
            resourceMap.remove(resourceName);
        }
    }
}

Example 8 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TaskAssignmentStage method batchMessage.

List<Message> batchMessage(Builder keyBuilder, List<Message> messages, Map<String, Resource> resourceMap, Map<String, LiveInstance> liveInstanceMap, HelixManagerProperties properties) {
    // group messages by its CurrentState path + "/" + fromState + "/" + toState
    Map<String, Message> batchMessages = new HashMap<String, Message>();
    List<Message> outputMessages = new ArrayList<Message>();
    Iterator<Message> iter = messages.iterator();
    while (iter.hasNext()) {
        Message message = iter.next();
        String resourceName = message.getResourceName();
        Resource resource = resourceMap.get(resourceName);
        String instanceName = message.getTgtName();
        LiveInstance liveInstance = liveInstanceMap.get(instanceName);
        String participantVersion = null;
        if (liveInstance != null) {
            participantVersion = liveInstance.getHelixVersion();
        }
        if (resource == null || !resource.getBatchMessageMode() || participantVersion == null || !properties.isFeatureSupported("batch_message", participantVersion)) {
            outputMessages.add(message);
            continue;
        }
        String key = keyBuilder.currentState(message.getTgtName(), message.getTgtSessionId(), message.getResourceName()).getPath() + "/" + message.getFromState() + "/" + message.getToState();
        if (!batchMessages.containsKey(key)) {
            Message batchMessage = new Message(message.getRecord());
            batchMessage.setBatchMessageMode(true);
            outputMessages.add(batchMessage);
            batchMessages.put(key, batchMessage);
        }
        batchMessages.get(key).addPartitionName(message.getPartitionName());
    }
    return outputMessages;
}

Example 9 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TestJobStateOnCreation method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    _cache = new ClusterDataCache();
    _idealState = new IdealState(WORKFLOW_NAME);
    _resource = new Resource(WORKFLOW_NAME);
    _currStateOutput = new CurrentStateOutput();
    _participants = new MockParticipantManager[_numNodes];
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    _setupTool = new ClusterSetup(ZK_ADDR);
    _setupTool.addCluster(CLUSTER_NAME, true);
    createManagers();
}

Example 10 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TestRebalancerMetrics method testLoadBalanceMetrics.

@Test
public void testLoadBalanceMetrics() {
    System.out.println("START testLoadBalanceMetrics at " + new Date(System.currentTimeMillis()));
    String resource = "testResourceName";
    int numPartition = 100;
    int numReplica = 3;
    int maxPending = 3;
    setupIdealState(5, new String[] { resource }, numPartition, numReplica, IdealState.RebalanceMode.FULL_AUTO, BuiltInStateModelDefinitions.MasterSlave.name());
    setupInstances(5);
    setupLiveInstances(4);
    setupStateModel();
    Map<String, Resource> resourceMap = getResourceMap(new String[] { resource }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name());
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    ClusterStatusMonitor monitor = new ClusterStatusMonitor(_clusterName);
    monitor.active();
    event.addAttribute(AttributeName.clusterStatusMonitor.name(), monitor);
    runStage(event, new ReadClusterDataStage());
    runStage(event, new BestPossibleStateCalcStage());
    BestPossibleStateOutput bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    currentStateOutput = copyCurrentStateFromBestPossible(bestPossibleStateOutput, resource);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    setupLiveInstances(4);
    runStage(event, new ReadClusterDataStage());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    setupThrottleConfig(cache.getClusterConfig(), StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, maxPending);
    runStage(event, new BestPossibleStateCalcStage());
    runStage(event, new IntermediateStateCalcStage());
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    ResourceMonitor resourceMonitor = clusterStatusMonitor.getResourceMonitor(resource);
    long numPendingLoadBalance = resourceMonitor.getPendingLoadRebalancePartitionGauge();
    Assert.assertTrue(numPendingLoadBalance > 0);
    Assert.assertEquals(resourceMonitor.getLoadRebalanceThrottledPartitionGauge(), numPendingLoadBalance - maxPending);
    System.out.println("END testLoadBalanceMetrics at " + new Date(System.currentTimeMillis()));
}