Examples with LiveInstance org.apache.helix.model.LiveInstance used on opensource projects

Example 66 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class SchedulerTasksResource method post.

@Override
public Representation post(Representation entity) {
    try {
        String clusterName = (String) getRequest().getAttributes().get("clusterName");
        Form form = new Form(entity);
        ZkClient zkClient = (ZkClient) getContext().getAttributes().get(RestAdminApplication.ZKCLIENT);
        String msgTemplateString = ClusterRepresentationUtil.getFormJsonParameterString(form, MESSAGETEMPLATE);
        if (msgTemplateString == null) {
            throw new HelixException("SchedulerTasksResource need to have MessageTemplate specified.");
        }
        Map<String, String> messageTemplate = ClusterRepresentationUtil.getFormJsonParameters(form, MESSAGETEMPLATE);
        String criteriaString = ClusterRepresentationUtil.getFormJsonParameterString(form, CRITERIA);
        if (criteriaString == null) {
            throw new HelixException("SchedulerTasksResource need to have Criteria specified.");
        }
        HelixDataAccessor accessor = ClusterRepresentationUtil.getClusterDataAccessor(zkClient, clusterName);
        LiveInstance leader = accessor.getProperty(accessor.keyBuilder().controllerLeader());
        if (leader == null) {
            throw new HelixException("There is no leader for the cluster " + clusterName);
        }
        Message schedulerMessage = new Message(MessageType.SCHEDULER_MSG, UUID.randomUUID().toString());
        schedulerMessage.getRecord().getSimpleFields().put(CRITERIA, criteriaString);
        schedulerMessage.getRecord().getMapFields().put(MESSAGETEMPLATE, messageTemplate);
        schedulerMessage.setTgtSessionId(leader.getSessionId());
        schedulerMessage.setTgtName("CONTROLLER");
        schedulerMessage.setSrcInstanceType(InstanceType.CONTROLLER);
        String taskQueueName = ClusterRepresentationUtil.getFormJsonParameterString(form, TASKQUEUENAME);
        if (taskQueueName != null && taskQueueName.length() > 0) {
            schedulerMessage.getRecord().setSimpleField(DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE, taskQueueName);
        }
        accessor.setProperty(accessor.keyBuilder().controllerMessage(schedulerMessage.getMsgId()), schedulerMessage);
        Map<String, String> resultMap = new HashMap<String, String>();
        resultMap.put("StatusUpdatePath", PropertyPathBuilder.controllerStatusUpdate(clusterName, MessageType.SCHEDULER_MSG.name(), schedulerMessage.getMsgId()));
        resultMap.put("MessageType", Message.MessageType.SCHEDULER_MSG.name());
        resultMap.put("MsgId", schedulerMessage.getMsgId());
        // Assemble the rest URL for task status update
        String ipAddress = InetAddress.getLocalHost().getCanonicalHostName();
        String url = "http://" + ipAddress + ":" + getContext().getAttributes().get(RestAdminApplication.PORT) + "/clusters/" + clusterName + "/Controller/statusUpdates/SCHEDULER_MSG/" + schedulerMessage.getMsgId();
        resultMap.put("statusUpdateUrl", url);
        getResponse().setEntity(ClusterRepresentationUtil.ObjectToJson(resultMap), MediaType.APPLICATION_JSON);
        getResponse().setStatus(Status.SUCCESS_OK);
    } catch (Exception e) {
        getResponse().setEntity(ClusterRepresentationUtil.getErrorAsJsonStringFromException(e), MediaType.APPLICATION_JSON);
        getResponse().setStatus(Status.SUCCESS_OK);
        LOG.error("", e);
    }
    return null;
}

Example 67 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class ClusterDataCache method refresh.

/**
 * This refreshes the cluster data by re-fetching the data from zookeeper in
 * an efficient way
 * @param accessor
 * @return
 */
public synchronized boolean refresh(HelixDataAccessor accessor) {
    long startTime = System.currentTimeMillis();
    Builder keyBuilder = accessor.keyBuilder();
    if (_propertyDataChangedMap.get(ChangeType.IDEAL_STATE)) {
        _propertyDataChangedMap.put(ChangeType.IDEAL_STATE, Boolean.valueOf(false));
        clearCachedResourceAssignments();
        _idealStateCacheMap = refreshIdealStates(accessor);
    }
    if (_propertyDataChangedMap.get(ChangeType.LIVE_INSTANCE)) {
        long start = System.currentTimeMillis();
        _propertyDataChangedMap.put(ChangeType.LIVE_INSTANCE, Boolean.valueOf(false));
        clearCachedResourceAssignments();
        _liveInstanceCacheMap = accessor.getChildValuesMap(keyBuilder.liveInstances(), true);
        _updateInstanceOfflineTime = true;
        LOG.info("Refresh LiveInstances for cluster " + _clusterName + ", took " + (System.currentTimeMillis() - startTime) + " ms");
    }
    if (_propertyDataChangedMap.get(ChangeType.INSTANCE_CONFIG)) {
        _propertyDataChangedMap.put(ChangeType.INSTANCE_CONFIG, Boolean.valueOf(false));
        clearCachedResourceAssignments();
        _instanceConfigCacheMap = accessor.getChildValuesMap(keyBuilder.instanceConfigs(), true);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Reload InstanceConfig: " + _instanceConfigCacheMap.keySet());
        }
    }
    if (_propertyDataChangedMap.get(ChangeType.RESOURCE_CONFIG)) {
        _propertyDataChangedMap.put(ChangeType.RESOURCE_CONFIG, Boolean.valueOf(false));
        clearCachedResourceAssignments();
        _resourceConfigCacheMap = accessor.getChildValuesMap(accessor.keyBuilder().resourceConfigs(), true);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Reload ResourceConfigs: " + _resourceConfigCacheMap.size());
        }
    }
    _idealStateMap = new HashMap<>(_idealStateCacheMap);
    _liveInstanceMap = new HashMap(_liveInstanceCacheMap);
    _instanceConfigMap = new ConcurrentHashMap<>(_instanceConfigCacheMap);
    _resourceConfigMap = new HashMap(_resourceConfigCacheMap);
    if (_updateInstanceOfflineTime) {
        updateOfflineInstanceHistory(accessor);
    }
    if (_isTaskCache) {
        _taskDataCache.refresh(accessor, _resourceConfigMap);
    }
    Map<String, StateModelDefinition> stateDefMap = accessor.getChildValuesMap(keyBuilder.stateModelDefs(), true);
    _stateModelDefMap = new ConcurrentHashMap<>(stateDefMap);
    _constraintMap = accessor.getChildValuesMap(keyBuilder.constraints(), true);
    _clusterConfig = accessor.getProperty(keyBuilder.clusterConfig());
    _instanceMessagesCache.refresh(accessor, _liveInstanceMap);
    _currentStateCache.refresh(accessor, _liveInstanceMap);
    // current state must be refreshed before refreshing relay messages
    // because we need to use current state to validate all relay messages.
    _instanceMessagesCache.updateRelayMessages(_liveInstanceMap, _currentStateCache.getCurrentStatesMap());
    if (_clusterConfig != null) {
        _idealStateRuleMap = _clusterConfig.getIdealStateRules();
    } else {
        _idealStateRuleMap = new HashMap();
        LOG.warn("Cluster config is null!");
    }
    MaintenanceSignal maintenanceSignal = accessor.getProperty(keyBuilder.maintenance());
    _isMaintenanceModeEnabled = (maintenanceSignal != null) ? true : false;
    updateDisabledInstances();
    long endTime = System.currentTimeMillis();
    LOG.info("END: ClusterDataCache.refresh() for cluster " + getClusterName() + ", took " + (endTime - startTime) + " ms");
    if (LOG.isDebugEnabled()) {
        LOG.debug("# of StateModelDefinition read from zk: " + _stateModelDefMap.size());
        LOG.debug("# of ConstraintMap read from zk: " + _constraintMap.size());
        LOG.debug("LiveInstances: " + _liveInstanceMap.keySet());
        for (LiveInstance instance : _liveInstanceMap.values()) {
            LOG.debug("live instance: " + instance.getInstanceName() + " " + instance.getSessionId());
        }
        LOG.debug("IdealStates: " + _idealStateMap.keySet());
        LOG.debug("ResourceConfigs: " + _resourceConfigMap.keySet());
        LOG.debug("InstanceConfigs: " + _instanceConfigMap.keySet());
        LOG.debug("ClusterConfigs: " + _clusterConfig);
        LOG.debug("JobContexts: " + _taskDataCache.getContexts().keySet());
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("Cache content: " + toString());
    }
    return true;
}

Example 68 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class CompatibilityCheckStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (manager == null || cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager | DataCache");
    }
    HelixManagerProperties properties = manager.getProperties();
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    for (LiveInstance liveInstance : liveInstanceMap.values()) {
        String participantVersion = liveInstance.getHelixVersion();
        if (!properties.isParticipantCompatible(participantVersion)) {
            String errorMsg = "incompatible participant. pipeline will not continue. " + "controller: " + manager.getInstanceName() + ", controllerVersion: " + properties.getVersion() + ", minimumSupportedParticipantVersion: " + properties.getProperty("miminum_supported_version.participant") + ", participant: " + liveInstance.getInstanceName() + ", participantVersion: " + participantVersion;
            LOG.error(errorMsg);
            throw new StageException(errorMsg);
        }
    }
}

Example 69 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class CurrentStateComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (cache == null || resourceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache|RESOURCE");
    }
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (LiveInstance instance : liveInstances.values()) {
        String instanceName = instance.getInstanceName();
        String instanceSessionId = instance.getSessionId();
        // update pending messages
        Map<String, Message> messages = cache.getMessages(instanceName);
        updatePendingMessages(instance, messages.values(), currentStateOutput, resourceMap);
        // update current states.
        Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, instanceSessionId);
        updateCurrentStates(instance, currentStateMap.values(), currentStateOutput, resourceMap);
    }
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        updateMissingTopStateStatus(cache, clusterStatusMonitor, resourceMap, currentStateOutput);
    }
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
}

Example 70 with LiveInstance

use of org.apache.helix.model.LiveInstance in project helix by apache.

the class CurrentStateComputationStage method reportNewTopStateMissing.

private void reportNewTopStateMissing(ClusterDataCache cache, Map<String, String> stateMap, Map<String, Map<String, Long>> missingTopStateMap, Resource resource, Partition partition, String topState) {
    long startTime = NOT_RECORDED;
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    for (String instanceName : stateMap.keySet()) {
        if (liveInstances.containsKey(instanceName)) {
            CurrentState currentState = cache.getCurrentState(instanceName, liveInstances.get(instanceName).getSessionId()).get(resource.getResourceName());
            if (currentState.getPreviousState(partition.getPartitionName()) != null && currentState.getPreviousState(partition.getPartitionName()).equalsIgnoreCase(topState)) {
                // Update the latest start time only from top state to other state transition
                // At beginning, the start time should -1 (not recorded). If something happen either
                // instance not alive or the instance just started for that partition, Helix does not know
                // the previous start time or end time. So we count from current.
                // 
                // Previous state is top state does not mean that resource has only one top state
                // (i.e. Online/Offline). So Helix has to find the latest start time as the staring point.
                startTime = Math.max(startTime, currentState.getStartTime(partition.getPartitionName()));
            }
        }
    }
    if (startTime == NOT_RECORDED) {
        startTime = System.currentTimeMillis();
    }
    if (!missingTopStateMap.containsKey(resource.getResourceName())) {
        missingTopStateMap.put(resource.getResourceName(), new HashMap<String, Long>());
    }
    Map<String, Long> partitionMap = missingTopStateMap.get(resource.getResourceName());
    // Update the new partition without top state
    if (!partitionMap.containsKey(partition.getPartitionName())) {
        missingTopStateMap.get(resource.getResourceName()).put(partition.getPartitionName(), startTime);
    }
}