Examples with LiveInstance org.apache.helix.model.LiveInstance used on opensource projects

Example 71 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class CurrentStateComputationStage method reportTopStateComesBack.

private void reportTopStateComesBack(ClusterDataCache cache, Map<String, String> stateMap, Map<String, Map<String, Long>> missingTopStateMap, Resource resource, Partition partition, ClusterStatusMonitor clusterStatusMonitor, long threshold, String topState) {
    long handOffStartTime = missingTopStateMap.get(resource.getResourceName()).get(partition.getPartitionName());
    // Find the earliest end time from the top states
    long handOffEndTime = System.currentTimeMillis();
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    for (String instanceName : stateMap.keySet()) {
        CurrentState currentState = cache.getCurrentState(instanceName, liveInstances.get(instanceName).getSessionId()).get(resource.getResourceName());
        if (currentState.getState(partition.getPartitionName()).equalsIgnoreCase(topState)) {
            handOffEndTime = Math.min(handOffEndTime, currentState.getEndTime(partition.getPartitionName()));
        }
    }
    if (handOffStartTime != TRANSITION_FAILED && handOffEndTime - handOffStartTime <= threshold) {
        LOG.info(String.format("Missing topstate duration is %d for partition %s", handOffEndTime - handOffStartTime, partition.getPartitionName()));
        if (clusterStatusMonitor != null) {
            clusterStatusMonitor.updateMissingTopStateDurationStats(resource.getResourceName(), handOffEndTime - handOffStartTime, true);
        }
    }
    removeFromStatsMap(missingTopStateMap, resource, partition);
}

Example 72 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class ResourceComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache");
    }
    Map<String, IdealState> idealStates = cache.getIdealStates();
    Map<String, Resource> resourceMap = new LinkedHashMap<String, Resource>();
    Map<String, Resource> resourceToRebalance = new LinkedHashMap<>();
    if (idealStates != null && idealStates.size() > 0) {
        for (IdealState idealState : idealStates.values()) {
            if (idealState == null) {
                continue;
            }
            Set<String> partitionSet = idealState.getPartitionSet();
            String resourceName = idealState.getResourceName();
            if (!resourceMap.containsKey(resourceName)) {
                Resource resource = new Resource(resourceName, cache.getClusterConfig(), cache.getResourceConfig(resourceName));
                resourceMap.put(resourceName, resource);
                if (!idealState.isValid() && !cache.isTaskCache() || idealState.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && cache.isTaskCache() || !idealState.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && !cache.isTaskCache()) {
                    resourceToRebalance.put(resourceName, resource);
                }
                resource.setStateModelDefRef(idealState.getStateModelDefRef());
                resource.setStateModelFactoryName(idealState.getStateModelFactoryName());
                resource.setBucketSize(idealState.getBucketSize());
                boolean batchMessageMode = idealState.getBatchMessageMode();
                ClusterConfig clusterConfig = cache.getClusterConfig();
                if (clusterConfig != null) {
                    batchMessageMode |= clusterConfig.getBatchMessageMode();
                }
                resource.setBatchMessageMode(batchMessageMode);
                resource.setResourceGroupName(idealState.getResourceGroupName());
                resource.setResourceTag(idealState.getInstanceGroupTag());
            }
            for (String partition : partitionSet) {
                addPartition(partition, resourceName, resourceMap);
            }
        }
    }
    // It's important to get partitions from CurrentState as well since the
    // idealState might be removed.
    Map<String, LiveInstance> availableInstances = cache.getLiveInstances();
    if (availableInstances != null && availableInstances.size() > 0) {
        for (LiveInstance instance : availableInstances.values()) {
            String instanceName = instance.getInstanceName();
            String clientSessionId = instance.getSessionId();
            Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, clientSessionId);
            if (currentStateMap == null || currentStateMap.size() == 0) {
                continue;
            }
            for (CurrentState currentState : currentStateMap.values()) {
                String resourceName = currentState.getResourceName();
                Map<String, String> resourceStateMap = currentState.getPartitionStateMap();
                if (resourceStateMap.keySet().isEmpty()) {
                    // don't include empty current state for dropped resource
                    continue;
                }
                // don't overwrite ideal state settings
                if (!resourceMap.containsKey(resourceName)) {
                    addResource(resourceName, resourceMap);
                    Resource resource = resourceMap.get(resourceName);
                    resource.setStateModelDefRef(currentState.getStateModelDefRef());
                    resource.setStateModelFactoryName(currentState.getStateModelFactoryName());
                    resource.setBucketSize(currentState.getBucketSize());
                    resource.setBatchMessageMode(currentState.getBatchMessageMode());
                    if (resource.getStateModelDefRef() == null && !cache.isTaskCache() || resource.getStateModelDefRef() != null && (resource.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && cache.isTaskCache() || !resource.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && !cache.isTaskCache())) {
                        resourceToRebalance.put(resourceName, resource);
                    }
                    IdealState idealState = idealStates.get(resourceName);
                    if (idealState != null) {
                        resource.setResourceGroupName(idealState.getResourceGroupName());
                        resource.setResourceTag(idealState.getInstanceGroupTag());
                    }
                }
                if (currentState.getStateModelDefRef() == null) {
                    LOG.error("state model def is null." + "resource:" + currentState.getResourceName() + ", partitions: " + currentState.getPartitionStateMap().keySet() + ", states: " + currentState.getPartitionStateMap().values());
                    throw new StageException("State model def is null for resource:" + currentState.getResourceName());
                }
                for (String partition : resourceStateMap.keySet()) {
                    addPartition(partition, resourceName, resourceMap);
                }
            }
        }
    }
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceToRebalance);
}

Example 73 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class TaskAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    MessageThrottleStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    if (manager == null || resourceMap == null || messageOutput == null || cache == null || liveInstanceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|RESOURCES|MESSAGES_THROTTLE|DataCache|liveInstanceMap");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    List<Message> messagesToSend = new ArrayList<Message>();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = messageOutput.getMessages(resourceName, partition);
            messagesToSend.addAll(messages);
        }
    }
    List<Message> outputMessages = batchMessage(dataAccessor.keyBuilder(), messagesToSend, resourceMap, liveInstanceMap, manager.getProperties());
    sendMessages(dataAccessor, outputMessages);
    // TODO: Need also count messages from task rebalancer
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        if (clusterStatusMonitor != null) {
            clusterStatusMonitor.increaseMessageReceived(outputMessages);
        }
    }
    long cacheStart = System.currentTimeMillis();
    cache.cacheMessages(outputMessages);
    long cacheEnd = System.currentTimeMillis();
    logger.debug("Caching messages took " + (cacheEnd - cacheStart) + " ms");
}

Example 74 with LiveInstance

use of org.apache.helix.model.LiveInstance in project incubator-gobblin by apache.

the class HelixUtils method getLeaderUrl.

private static String getLeaderUrl(HelixManager helixManager) {
    PropertyKey key = helixManager.getHelixDataAccessor().keyBuilder().controllerLeader();
    LiveInstance leader = helixManager.getHelixDataAccessor().getProperty(key);
    return getUrlFromHelixInstanceName(leader.getInstanceName());
}