Examples with ResourceControllerDataProvider org.apache.helix.controller.dataproviders.ResourceControllerDataProvider used on opensource projects

Example 1 with ResourceControllerDataProvider

use of org.apache.helix.controller.dataproviders.ResourceControllerDataProvider in project helix by apache.

the class TestP2PNoDuplicatedMessage method verifyP2PEnabled.

private void verifyP2PEnabled(long startTime) {
    ResourceControllerDataProvider dataCache = new ResourceControllerDataProvider(CLUSTER_NAME);
    dataCache.refresh(_accessor);
    Map<String, LiveInstance> liveInstanceMap = dataCache.getLiveInstances();
    for (LiveInstance instance : liveInstanceMap.values()) {
        Map<String, CurrentState> currentStateMap = dataCache.getCurrentState(instance.getInstanceName(), instance.getEphemeralOwner());
        Assert.assertNotNull(currentStateMap);
        for (CurrentState currentState : currentStateMap.values()) {
            for (String partition : currentState.getPartitionStateMap().keySet()) {
                String state = currentState.getState(partition);
                long start = currentState.getStartTime(partition);
                if (state.equalsIgnoreCase("MASTER") && start > startTime) {
                    String triggerHost = currentState.getTriggerHost(partition);
                    if (!triggerHost.equals(_controllerName)) {
                        p2pTrigged++;
                    }
                    total++;
                }
            }
        }
    }
}

Example 2 with ResourceControllerDataProvider

use of org.apache.helix.controller.dataproviders.ResourceControllerDataProvider in project helix by apache.

the class TestHelixDataAccessor method testDataProviderRefresh.

@Test(expectedExceptions = { HelixMetaDataAccessException.class })
public void testDataProviderRefresh() {
    ResourceControllerDataProvider cache = new ResourceControllerDataProvider("MyCluster");
    cache.refresh(accessor);
}

Example 3 with ResourceControllerDataProvider

use of org.apache.helix.controller.dataproviders.ResourceControllerDataProvider in project helix by apache.

the class TestAbnormalStatesResolver method testConfigureResolver.

@Test
public void testConfigureResolver() {
    ResourceControllerDataProvider cache = new ResourceControllerDataProvider(CLUSTER_NAME);
    // Verify the initial setup.
    cache.refresh(_controller.getHelixDataAccessor());
    for (String stateModelDefName : cache.getStateModelDefMap().keySet()) {
        Assert.assertEquals(cache.getAbnormalStateResolver(stateModelDefName).getResolverClass(), MonitoredAbnormalResolver.DUMMY_STATE_RESOLVER.getResolverClass());
    }
    // Update the resolver configuration for MasterSlave state model.
    ConfigAccessor configAccessor = new ConfigAccessor.Builder().setZkAddress(ZK_ADDR).build();
    ClusterConfig clusterConfig = configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.setAbnormalStateResolverMap(ImmutableMap.of(MasterSlaveSMD.name, MockAbnormalStateResolver.class.getName()));
    configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
    cache.requireFullRefresh();
    cache.refresh(_controller.getHelixDataAccessor());
    for (String stateModelDefName : cache.getStateModelDefMap().keySet()) {
        Assert.assertEquals(cache.getAbnormalStateResolver(stateModelDefName).getResolverClass(), stateModelDefName.equals(MasterSlaveSMD.name) ? MockAbnormalStateResolver.class : MonitoredAbnormalResolver.DUMMY_STATE_RESOLVER.getResolverClass());
    }
    // Reset the resolver map
    clusterConfig = configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.setAbnormalStateResolverMap(Collections.emptyMap());
    configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
}

Example 4 with ResourceControllerDataProvider

use of org.apache.helix.controller.dataproviders.ResourceControllerDataProvider in project helix by apache.

the class ExternalViewComputeStage method execute.

@Override
public void execute(final ClusterEvent event) throws Exception {
    _eventId = event.getEventId();
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    ResourceControllerDataProvider cache = event.getAttribute(AttributeName.ControllerDataProvider.name());
    if (manager == null || resourceMap == null || cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires ClusterManager|RESOURCES|DataCache");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
    CurrentStateOutput currentStateOutput = event.getAttribute(AttributeName.CURRENT_STATE.name());
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    List<ExternalView> newExtViews = new ArrayList<>();
    Set<String> monitoringResources = new HashSet<>();
    Map<String, ExternalView> curExtViews = cache.getExternalViews();
    for (Resource resource : resourceMap.values()) {
        try {
            computeExternalView(resource, currentStateOutput, cache, clusterStatusMonitor, curExtViews, manager, monitoringResources, newExtViews);
        } catch (HelixException ex) {
            LogUtil.logError(LOG, _eventId, "Failed to calculate external view for resource " + resource.getResourceName(), ex);
        }
    }
    // Keep MBeans for existing resources and unregister MBeans for dropped resources
    if (clusterStatusMonitor != null) {
        clusterStatusMonitor.retainResourceMonitor(monitoringResources);
    }
    List<String> externalViewsToRemove = new ArrayList<>();
    // TODO: consider not setting the externalview of SCHEDULER_TASK_QUEUE at all.
    // Are there any entity that will be interested in its change?
    // For the resource with DisableExternalView option turned on in IdealState
    // We will not actually create or write the externalView to ZooKeeper.
    List<PropertyKey> keys = new ArrayList<>();
    for (Iterator<ExternalView> it = newExtViews.iterator(); it.hasNext(); ) {
        ExternalView view = it.next();
        String resourceName = view.getResourceName();
        IdealState idealState = cache.getIdealState(resourceName);
        if (idealState != null && idealState.isExternalViewDisabled()) {
            it.remove();
            // remove the external view if the external view exists
            if (curExtViews.containsKey(resourceName)) {
                LogUtil.logInfo(LOG, _eventId, "Remove externalView for resource: " + resourceName);
                dataAccessor.removeProperty(keyBuilder.externalView(resourceName));
                externalViewsToRemove.add(resourceName);
            }
        } else {
            keys.add(keyBuilder.externalView(resourceName));
        }
    }
    // add/update external-views
    if (newExtViews.size() > 0) {
        dataAccessor.setChildren(keys, newExtViews);
        cache.updateExternalViews(newExtViews);
    }
    // remove dead external-views
    for (String resourceName : curExtViews.keySet()) {
        if (!resourceMap.keySet().contains(resourceName)) {
            LogUtil.logInfo(LOG, _eventId, "Remove externalView for resource: " + resourceName);
            dataAccessor.removeProperty(keyBuilder.externalView(resourceName));
            externalViewsToRemove.add(resourceName);
        }
    }
    cache.removeExternalViews(externalViewsToRemove);
}

Example 5 with ResourceControllerDataProvider

use of org.apache.helix.controller.dataproviders.ResourceControllerDataProvider in project helix by apache.

the class IntermediateStateCalcStage method computeIntermediatePartitionState.

/**
 * Compute intermediate partition states for a prioritized resource.
 * @param cache
 * @param clusterStatusMonitor
 * @param idealState
 * @param resource
 * @param currentStateOutput
 * @param bestPossiblePartitionStateMap
 * @param preferenceLists
 * @param throttleController
 * @return
 */
private PartitionStateMap computeIntermediatePartitionState(ResourceControllerDataProvider cache, ClusterStatusMonitor clusterStatusMonitor, IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput, PartitionStateMap bestPossiblePartitionStateMap, Map<String, List<String>> preferenceLists, StateTransitionThrottleController throttleController, Map<Partition, List<Message>> resourceMessageMap) {
    String resourceName = resource.getResourceName();
    LogUtil.logDebug(logger, _eventId, String.format("Processing resource: %s", resourceName));
    // rebalance.
    if (!IdealState.RebalanceMode.FULL_AUTO.equals(idealState.getRebalanceMode()) || resourceMessageMap.isEmpty()) {
        return bestPossiblePartitionStateMap;
    }
    String stateModelDefName = idealState.getStateModelDefRef();
    StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
    // This require a deep copy of current state map because some of the states will be overwritten by applying
    // messages to it.
    Set<Partition> partitionsWithErrorStateReplica = new HashSet<>();
    Set<String> messagesForRecovery = new HashSet<>();
    Set<String> messagesForLoad = new HashSet<>();
    Set<String> messagesThrottledForRecovery = new HashSet<>();
    Set<String> messagesThrottledForLoad = new HashSet<>();
    ClusterConfig clusterConfig = cache.getClusterConfig();
    // If the threshold (ErrorOrRecovery) is set, then use it, if not, then check if the old
    // threshold (Error) is set. If the old threshold is set, use it. If not, use the default value
    // for the new one. This is for backward-compatibility
    // Default threshold for ErrorOrRecoveryPartitionThresholdForLoadBalance
    int threshold = 1;
    // Keep the error count as partition level. This logic only applies to downward state transition determination
    for (Partition partition : currentStateOutput.getCurrentStateMap(resourceName).keySet()) {
        Map<String, String> entry = currentStateOutput.getCurrentStateMap(resourceName).get(partition);
        if (entry.values().stream().anyMatch(x -> x.contains(HelixDefinedState.ERROR.name()))) {
            partitionsWithErrorStateReplica.add(partition);
        }
    }
    int numPartitionsWithErrorReplica = partitionsWithErrorStateReplica.size();
    if (clusterConfig.getErrorOrRecoveryPartitionThresholdForLoadBalance() != -1) {
        // ErrorOrRecovery is set
        threshold = clusterConfig.getErrorOrRecoveryPartitionThresholdForLoadBalance();
    } else {
        if (clusterConfig.getErrorPartitionThresholdForLoadBalance() != 0) {
            // 0 is the default value so the old threshold has been set
            threshold = clusterConfig.getErrorPartitionThresholdForLoadBalance();
        }
    }
    // Perform regular load balance only if the number of partitions in recovery and in error is
    // less than the threshold. Otherwise, only allow downward-transition load balance
    boolean onlyDownwardLoadBalance = numPartitionsWithErrorReplica > threshold;
    chargePendingTransition(resource, currentStateOutput, throttleController, cache, preferenceLists, stateModelDef);
    // Sort partitions in case of urgent partition need to take the quota first.
    List<Partition> partitions = new ArrayList<>(resource.getPartitions());
    Collections.sort(partitions, new PartitionPriorityComparator(bestPossiblePartitionStateMap.getStateMap(), currentStateOutput.getCurrentStateMap(resourceName), stateModelDef.getTopState()));
    for (Partition partition : partitions) {
        if (resourceMessageMap.get(partition) == null || resourceMessageMap.get(partition).isEmpty()) {
            continue;
        }
        List<Message> messagesToThrottle = new ArrayList<>(resourceMessageMap.get(partition));
        Map<String, String> derivedCurrentStateMap = currentStateOutput.getCurrentStateMap(resourceName, partition).entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue()));
        List<String> preferenceList = preferenceLists.get(partition.getPartitionName());
        Map<String, Integer> requiredState = getRequiredStates(resourceName, cache, preferenceList);
        Collections.sort(messagesToThrottle, new MessagePriorityComparator(preferenceList, stateModelDef.getStatePriorityMap()));
        for (Message message : messagesToThrottle) {
            RebalanceType rebalanceType = getRebalanceTypePerMessage(requiredState, message, derivedCurrentStateMap);
            // Number of states required by StateModelDefinition are not satisfied, need recovery
            if (rebalanceType.equals(RebalanceType.RECOVERY_BALANCE)) {
                messagesForRecovery.add(message.getId());
                recoveryRebalance(resource, partition, throttleController, message, cache, messagesThrottledForRecovery, resourceMessageMap);
            } else if (rebalanceType.equals(RebalanceType.LOAD_BALANCE)) {
                messagesForLoad.add(message.getId());
                loadRebalance(resource, partition, throttleController, message, cache, onlyDownwardLoadBalance, stateModelDef, messagesThrottledForLoad, resourceMessageMap);
            }
            // Apply the message to temporary current state map
            if (!messagesThrottledForRecovery.contains(message.getId()) && !messagesThrottledForLoad.contains(message.getId())) {
                derivedCurrentStateMap.put(message.getTgtName(), message.getToState());
            }
        }
    }
    // TODO: We may need to optimize it to be async compute for intermediate state output.
    PartitionStateMap intermediatePartitionStateMap = new PartitionStateMap(resourceName, currentStateOutput.getCurrentStateMap(resourceName));
    computeIntermediateMap(intermediatePartitionStateMap, currentStateOutput.getPendingMessageMap(resourceName), resourceMessageMap);
    if (!messagesForRecovery.isEmpty()) {
        LogUtil.logInfo(logger, _eventId, String.format("Recovery balance needed for %s with messages: %s", resourceName, messagesForRecovery));
    }
    if (!messagesForLoad.isEmpty()) {
        LogUtil.logInfo(logger, _eventId, String.format("Load balance needed for %s with messages: %s", resourceName, messagesForLoad));
    }
    if (!partitionsWithErrorStateReplica.isEmpty()) {
        LogUtil.logInfo(logger, _eventId, String.format("Partition currently has an ERROR replica in %s partitions: %s", resourceName, partitionsWithErrorStateReplica));
    }
    if (clusterStatusMonitor != null) {
        clusterStatusMonitor.updateRebalancerStats(resourceName, messagesForRecovery.size(), messagesForLoad.size(), messagesThrottledForRecovery.size(), messagesThrottledForLoad.size());
    }
    if (logger.isDebugEnabled()) {
        logPartitionMapState(resourceName, new HashSet<>(resource.getPartitions()), messagesForRecovery, messagesThrottledForRecovery, messagesForLoad, messagesThrottledForLoad, currentStateOutput, bestPossiblePartitionStateMap, intermediatePartitionStateMap);
    }
    LogUtil.logDebug(logger, _eventId, String.format("End processing resource: %s", resourceName));
    return intermediatePartitionStateMap;
}