Examples with ResourceConfig org.apache.helix.model.ResourceConfig used on opensource projects

Example 1 with ResourceConfig

use of org.apache.helix.model.ResourceConfig in project helix by apache.

the class DelayedAutoRebalancer method computeNewIdealState.

@Override
public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState, CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
    IdealState cachedIdealState = getCachedIdealState(resourceName, clusterData);
    if (cachedIdealState != null) {
        LOG.debug("Use cached IdealState for " + resourceName);
        return cachedIdealState;
    }
    LOG.info("Computing IdealState for " + resourceName);
    List<String> allPartitions = new ArrayList<>(currentIdealState.getPartitionSet());
    if (allPartitions.size() == 0) {
        LOG.info("Partition count is 0 for resource " + resourceName + ", stop calculate ideal mapping for the resource.");
        return generateNewIdealState(resourceName, currentIdealState, emptyMapping(currentIdealState));
    }
    Map<String, List<String>> userDefinedPreferenceList = new HashMap<>();
    ClusterConfig clusterConfig = clusterData.getClusterConfig();
    ResourceConfig resourceConfig = clusterData.getResourceConfig(resourceName);
    boolean delayRebalanceEnabled = isDelayRebalanceEnabled(currentIdealState, clusterConfig);
    if (resourceConfig != null) {
        userDefinedPreferenceList = resourceConfig.getPreferenceLists();
        if (!userDefinedPreferenceList.isEmpty()) {
            LOG.info("Using user defined preference list for partitions: " + userDefinedPreferenceList.keySet());
        }
    }
    Set<String> liveEnabledNodes;
    Set<String> allNodes;
    String instanceTag = currentIdealState.getInstanceGroupTag();
    if (instanceTag != null) {
        liveEnabledNodes = clusterData.getEnabledLiveInstancesWithTag(instanceTag);
        allNodes = clusterData.getInstancesWithTag(instanceTag);
        if (LOG.isInfoEnabled()) {
            LOG.info(String.format("Found the following participants with tag %s for %s: " + "instances: %s, liveEnabledInstances: %s", currentIdealState.getInstanceGroupTag(), resourceName, allNodes, liveEnabledNodes));
        }
    } else {
        liveEnabledNodes = clusterData.getEnabledLiveInstances();
        allNodes = clusterData.getAllInstances();
    }
    Set<String> activeNodes = liveEnabledNodes;
    if (delayRebalanceEnabled) {
        long delay = getRebalanceDelay(currentIdealState, clusterConfig);
        activeNodes = getActiveInstances(allNodes, currentIdealState, liveEnabledNodes, clusterData.getInstanceOfflineTimeMap(), clusterData.getLiveInstances().keySet(), clusterData.getInstanceConfigMap(), delay, clusterConfig);
        Set<String> offlineOrDisabledInstances = new HashSet<>(activeNodes);
        offlineOrDisabledInstances.removeAll(liveEnabledNodes);
        setRebalanceScheduler(currentIdealState, offlineOrDisabledInstances, clusterData.getInstanceOfflineTimeMap(), clusterData.getLiveInstances().keySet(), clusterData.getInstanceConfigMap(), delay, clusterConfig);
    }
    if (allNodes.isEmpty() || activeNodes.isEmpty()) {
        LOG.error(String.format("No instances or active instances available for resource %s, " + "allInstances: %s, liveInstances: %s, activeInstances: %s", resourceName, allNodes, liveEnabledNodes, activeNodes));
        return generateNewIdealState(resourceName, currentIdealState, emptyMapping(currentIdealState));
    }
    StateModelDefinition stateModelDef = clusterData.getStateModelDef(currentIdealState.getStateModelDefRef());
    int replicaCount = currentIdealState.getReplicaCount(activeNodes.size());
    if (replicaCount == 0) {
        LOG.error("Replica count is 0 for resource " + resourceName + ", stop calculate ideal mapping for the resource.");
        return generateNewIdealState(resourceName, currentIdealState, emptyMapping(currentIdealState));
    }
    LinkedHashMap<String, Integer> stateCountMap = stateModelDef.getStateCountMap(activeNodes.size(), replicaCount);
    Map<String, Map<String, String>> currentMapping = currentMapping(currentStateOutput, resourceName, allPartitions, stateCountMap);
    List<String> partitionsToAssign = new ArrayList<>(allPartitions);
    partitionsToAssign.removeAll(userDefinedPreferenceList.keySet());
    int maxPartition = currentIdealState.getMaxPartitionsPerInstance();
    _rebalanceStrategy = getRebalanceStrategy(currentIdealState.getRebalanceStrategy(), partitionsToAssign, resourceName, stateCountMap, maxPartition);
    // sort node lists to ensure consistent preferred assignments
    List<String> allNodeList = new ArrayList<>(allNodes);
    List<String> liveEnabledNodeList = new ArrayList<>(liveEnabledNodes);
    Collections.sort(allNodeList);
    Collections.sort(liveEnabledNodeList);
    ZNRecord newIdealMapping = _rebalanceStrategy.computePartitionAssignment(allNodeList, liveEnabledNodeList, currentMapping, clusterData);
    ZNRecord finalMapping = newIdealMapping;
    if (isDelayRebalanceEnabled(currentIdealState, clusterConfig)) {
        List<String> activeNodeList = new ArrayList<>(activeNodes);
        Collections.sort(activeNodeList);
        int minActiveReplicas = getMinActiveReplica(currentIdealState, replicaCount);
        ZNRecord newActiveMapping = _rebalanceStrategy.computePartitionAssignment(allNodeList, activeNodeList, currentMapping, clusterData);
        finalMapping = getFinalDelayedMapping(currentIdealState, newIdealMapping, newActiveMapping, liveEnabledNodes, replicaCount, minActiveReplicas);
        LOG.debug("newActiveMapping: " + newActiveMapping);
    }
    finalMapping.getListFields().putAll(userDefinedPreferenceList);
    if (LOG.isDebugEnabled()) {
        LOG.debug("currentMapping: " + currentMapping);
        LOG.debug("stateCountMap: " + stateCountMap);
        LOG.debug("liveEnabledNodes: " + liveEnabledNodes);
        LOG.debug("activeNodes: " + activeNodes);
        LOG.debug("allNodes: " + allNodes);
        LOG.debug("maxPartition: " + maxPartition);
        LOG.debug("newIdealMapping: " + newIdealMapping);
        LOG.debug("finalMapping: " + finalMapping);
    }
    IdealState idealState = generateNewIdealState(resourceName, currentIdealState, finalMapping);
    clusterData.setCachedIdealMapping(resourceName, idealState.getRecord());
    return idealState;
}

Example 2 with ResourceConfig

use of org.apache.helix.model.ResourceConfig in project helix by apache.

the class ConfigAccessor method getResourceConfig.

/**
 * Get resource config for given resource in given cluster.
 *
 * @param clusterName
 * @param resourceName
 *
 * @return
 */
public ResourceConfig getResourceConfig(String clusterName, String resourceName) {
    HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE).forCluster(clusterName).forResource(resourceName).build();
    ZNRecord record = getConfigZnRecord(scope);
    if (record == null) {
        LOG.warn("No config found at " + scope.getZkPath());
        return null;
    }
    return new ResourceConfig(record);
}

Example 3 with ResourceConfig

use of org.apache.helix.model.ResourceConfig in project helix by apache.

the class TestResourceAccessor method testResourceConfig.

@Test(dependsOnMethods = "testAddResources")
public void testResourceConfig() throws IOException {
    System.out.println("Start test :" + TestHelper.getTestMethodName());
    String body = get("clusters/" + CLUSTER_NAME + "/resources/" + RESOURCE_NAME + "/configs", Response.Status.OK.getStatusCode(), true);
    ResourceConfig resourceConfig = new ResourceConfig(toZNRecord(body));
    Assert.assertEquals(resourceConfig, _configAccessor.getResourceConfig(CLUSTER_NAME, RESOURCE_NAME));
}

Example 4 with ResourceConfig

use of org.apache.helix.model.ResourceConfig in project helix by apache.

the class ResourceAccessor method getResourceConfig.

@GET
@Path("{resourceName}/configs")
public Response getResourceConfig(@PathParam("clusterId") String clusterId, @PathParam("resourceName") String resourceName) {
    ConfigAccessor accessor = getConfigAccessor();
    ResourceConfig resourceConfig = accessor.getResourceConfig(clusterId, resourceName);
    if (resourceConfig != null) {
        return JSONRepresentation(resourceConfig.getRecord());
    }
    return notFound();
}

Example 5 with ResourceConfig

use of org.apache.helix.model.ResourceConfig in project helix by apache.

the class CallbackHandler method invoke.

public void invoke(NotificationContext changeContext) throws Exception {
    Type type = changeContext.getType();
    long start = System.currentTimeMillis();
    // This allows the listener to work with one change at a time
    synchronized (_manager) {
        if (logger.isInfoEnabled()) {
            logger.info(Thread.currentThread().getId() + " START:INVOKE " + _path + " listener:" + _listener + " type: " + type);
        }
        if (!_expectTypes.contains(type)) {
            logger.warn("Callback handler received event in wrong order. Listener: " + _listener + ", path: " + _path + ", expected types: " + _expectTypes + " but was " + type);
            return;
        }
        _expectTypes = nextNotificationType.get(type);
        if (type == Type.INIT || type == Type.FINALIZE) {
            subscribeForChanges(changeContext.getType(), _path, _watchChild);
        } else {
            // put SubscribeForChange run in async thread to reduce the latency of zk callback handling.
            subscribeForChangesAsyn(changeContext.getType(), _path, _watchChild);
        }
        _expectTypes = nextNotificationType.get(type);
        if (_changeType == IDEAL_STATE) {
            IdealStateChangeListener idealStateChangeListener = (IdealStateChangeListener) _listener;
            List<IdealState> idealStates = preFetch(_propertyKey);
            idealStateChangeListener.onIdealStateChange(idealStates, changeContext);
        } else if (_changeType == INSTANCE_CONFIG) {
            if (_listener instanceof ConfigChangeListener) {
                ConfigChangeListener configChangeListener = (ConfigChangeListener) _listener;
                List<InstanceConfig> configs = preFetch(_propertyKey);
                configChangeListener.onConfigChange(configs, changeContext);
            } else if (_listener instanceof InstanceConfigChangeListener) {
                InstanceConfigChangeListener listener = (InstanceConfigChangeListener) _listener;
                List<InstanceConfig> configs = preFetch(_propertyKey);
                listener.onInstanceConfigChange(configs, changeContext);
            }
        } else if (_changeType == RESOURCE_CONFIG) {
            ResourceConfigChangeListener listener = (ResourceConfigChangeListener) _listener;
            List<ResourceConfig> configs = preFetch(_propertyKey);
            listener.onResourceConfigChange(configs, changeContext);
        } else if (_changeType == CLUSTER_CONFIG) {
            ClusterConfigChangeListener listener = (ClusterConfigChangeListener) _listener;
            ClusterConfig config = null;
            if (_preFetchEnabled) {
                config = _accessor.getProperty(_propertyKey);
            }
            listener.onClusterConfigChange(config, changeContext);
        } else if (_changeType == CONFIG) {
            ScopedConfigChangeListener listener = (ScopedConfigChangeListener) _listener;
            List<HelixProperty> configs = preFetch(_propertyKey);
            listener.onConfigChange(configs, changeContext);
        } else if (_changeType == LIVE_INSTANCE) {
            LiveInstanceChangeListener liveInstanceChangeListener = (LiveInstanceChangeListener) _listener;
            List<LiveInstance> liveInstances = preFetch(_propertyKey);
            liveInstanceChangeListener.onLiveInstanceChange(liveInstances, changeContext);
        } else if (_changeType == CURRENT_STATE) {
            CurrentStateChangeListener currentStateChangeListener = (CurrentStateChangeListener) _listener;
            String instanceName = PropertyPathConfig.getInstanceNameFromPath(_path);
            List<CurrentState> currentStates = preFetch(_propertyKey);
            currentStateChangeListener.onStateChange(instanceName, currentStates, changeContext);
        } else if (_changeType == MESSAGE) {
            MessageListener messageListener = (MessageListener) _listener;
            String instanceName = PropertyPathConfig.getInstanceNameFromPath(_path);
            List<Message> messages = preFetch(_propertyKey);
            messageListener.onMessage(instanceName, messages, changeContext);
        } else if (_changeType == MESSAGES_CONTROLLER) {
            MessageListener messageListener = (MessageListener) _listener;
            List<Message> messages = preFetch(_propertyKey);
            messageListener.onMessage(_manager.getInstanceName(), messages, changeContext);
        } else if (_changeType == EXTERNAL_VIEW || _changeType == TARGET_EXTERNAL_VIEW) {
            ExternalViewChangeListener externalViewListener = (ExternalViewChangeListener) _listener;
            List<ExternalView> externalViewList = preFetch(_propertyKey);
            externalViewListener.onExternalViewChange(externalViewList, changeContext);
        } else if (_changeType == CONTROLLER) {
            ControllerChangeListener controllerChangelistener = (ControllerChangeListener) _listener;
            controllerChangelistener.onControllerChange(changeContext);
        } else {
            logger.warn("Unknown change type: " + _changeType);
        }
        long end = System.currentTimeMillis();
        if (logger.isInfoEnabled()) {
            logger.info(Thread.currentThread().getId() + " END:INVOKE " + _path + " listener:" + _listener + " type: " + type + " Took: " + (end - start) + "ms");
        }
        if (_monitor != null) {
            _monitor.increaseCallbackCounters(end - start);
        }
    }
}