Examples with ClusterConfig org.apache.helix.model.ClusterConfig used on opensource projects

Example 1 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class ReadClusterDataStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    if (manager == null) {
        throw new StageException("HelixManager attribute value is null");
    }
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null && _cache == null) {
        cache = new ClusterDataCache(event.getClusterName());
    }
    _cache = cache;
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    _cache.refresh(dataAccessor);
    final ClusterConfig clusterConfig = cache.getClusterConfig();
    if (!_cache.isTaskCache()) {
        final ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        asyncExecute(_cache.getAsyncTasksThreadPool(), new Callable<Object>() {

            @Override
            public Object call() {
                // Update the cluster status gauges
                if (clusterStatusMonitor != null) {
                    logger.debug("Update cluster status monitors");
                    Set<String> instanceSet = Sets.newHashSet();
                    Set<String> liveInstanceSet = Sets.newHashSet();
                    Set<String> disabledInstanceSet = Sets.newHashSet();
                    Map<String, Map<String, List<String>>> disabledPartitions = Maps.newHashMap();
                    Map<String, List<String>> oldDisabledPartitions = Maps.newHashMap();
                    Map<String, Set<String>> tags = Maps.newHashMap();
                    Map<String, LiveInstance> liveInstanceMap = _cache.getLiveInstances();
                    for (Map.Entry<String, InstanceConfig> e : _cache.getInstanceConfigMap().entrySet()) {
                        String instanceName = e.getKey();
                        InstanceConfig config = e.getValue();
                        instanceSet.add(instanceName);
                        if (liveInstanceMap.containsKey(instanceName)) {
                            liveInstanceSet.add(instanceName);
                        }
                        if (!config.getInstanceEnabled() || (clusterConfig.getDisabledInstances() != null && clusterConfig.getDisabledInstances().containsKey(instanceName))) {
                            disabledInstanceSet.add(instanceName);
                        }
                        // TODO : Get rid of this data structure once the API is removed.
                        oldDisabledPartitions.put(instanceName, config.getDisabledPartitions());
                        disabledPartitions.put(instanceName, config.getDisabledPartitionsMap());
                        Set<String> instanceTags = Sets.newHashSet(config.getTags());
                        tags.put(instanceName, instanceTags);
                    }
                    clusterStatusMonitor.setClusterInstanceStatus(liveInstanceSet, instanceSet, disabledInstanceSet, disabledPartitions, oldDisabledPartitions, tags);
                    logger.debug("Complete cluster status monitors update.");
                }
                return null;
            }
        });
    }
    event.addAttribute(AttributeName.ClusterDataCache.name(), _cache);
}

Example 2 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class DelayedAutoRebalancer method computeBestPossiblePartitionState.

/**
 * Compute the best state for all partitions.
 * This is the default implementation, subclasses should re-implement
 * this method if its logic to generate bestpossible map for each partition is different from the default one here.
 *
 * @param cache
 * @param idealState
 * @param resource
 * @param currentStateOutput Provides the current state and pending state transitions for all partitions
 * @return
 */
@Override
public ResourceAssignment computeBestPossiblePartitionState(ClusterDataCache cache, IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Processing resource:" + resource.getResourceName());
    }
    Set<String> allNodes = cache.getEnabledInstances();
    Set<String> liveNodes = cache.getLiveInstances().keySet();
    ClusterConfig clusterConfig = cache.getClusterConfig();
    long delayTime = getRebalanceDelay(idealState, clusterConfig);
    Set<String> activeNodes = getActiveInstances(allNodes, idealState, liveNodes, cache.getInstanceOfflineTimeMap(), cache.getLiveInstances().keySet(), cache.getInstanceConfigMap(), delayTime, clusterConfig);
    String stateModelDefName = idealState.getStateModelDefRef();
    StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
    ResourceAssignment partitionMapping = new ResourceAssignment(resource.getResourceName());
    for (Partition partition : resource.getPartitions()) {
        Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
        Set<String> disabledInstancesForPartition = cache.getDisabledInstancesForPartition(resource.getResourceName(), partition.toString());
        List<String> preferenceList = getPreferenceList(partition, idealState, activeNodes);
        Map<String, String> bestStateForPartition = computeBestPossibleStateForPartition(liveNodes, stateModelDef, preferenceList, currentStateMap, disabledInstancesForPartition, idealState);
        partitionMapping.addReplicaMap(partition, bestStateForPartition);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Best possible mapping for resource  " + resource.getResourceName() + ": " + partitionMapping);
    }
    return partitionMapping;
}

Example 3 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class DelayedAutoRebalancer method computeNewIdealState.

@Override
public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState, CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
    IdealState cachedIdealState = getCachedIdealState(resourceName, clusterData);
    if (cachedIdealState != null) {
        LOG.debug("Use cached IdealState for " + resourceName);
        return cachedIdealState;
    }
    LOG.info("Computing IdealState for " + resourceName);
    List<String> allPartitions = new ArrayList<>(currentIdealState.getPartitionSet());
    if (allPartitions.size() == 0) {
        LOG.info("Partition count is 0 for resource " + resourceName + ", stop calculate ideal mapping for the resource.");
        return generateNewIdealState(resourceName, currentIdealState, emptyMapping(currentIdealState));
    }
    Map<String, List<String>> userDefinedPreferenceList = new HashMap<>();
    ClusterConfig clusterConfig = clusterData.getClusterConfig();
    ResourceConfig resourceConfig = clusterData.getResourceConfig(resourceName);
    boolean delayRebalanceEnabled = isDelayRebalanceEnabled(currentIdealState, clusterConfig);
    if (resourceConfig != null) {
        userDefinedPreferenceList = resourceConfig.getPreferenceLists();
        if (!userDefinedPreferenceList.isEmpty()) {
            LOG.info("Using user defined preference list for partitions: " + userDefinedPreferenceList.keySet());
        }
    }
    Set<String> liveEnabledNodes;
    Set<String> allNodes;
    String instanceTag = currentIdealState.getInstanceGroupTag();
    if (instanceTag != null) {
        liveEnabledNodes = clusterData.getEnabledLiveInstancesWithTag(instanceTag);
        allNodes = clusterData.getInstancesWithTag(instanceTag);
        if (LOG.isInfoEnabled()) {
            LOG.info(String.format("Found the following participants with tag %s for %s: " + "instances: %s, liveEnabledInstances: %s", currentIdealState.getInstanceGroupTag(), resourceName, allNodes, liveEnabledNodes));
        }
    } else {
        liveEnabledNodes = clusterData.getEnabledLiveInstances();
        allNodes = clusterData.getAllInstances();
    }
    Set<String> activeNodes = liveEnabledNodes;
    if (delayRebalanceEnabled) {
        long delay = getRebalanceDelay(currentIdealState, clusterConfig);
        activeNodes = getActiveInstances(allNodes, currentIdealState, liveEnabledNodes, clusterData.getInstanceOfflineTimeMap(), clusterData.getLiveInstances().keySet(), clusterData.getInstanceConfigMap(), delay, clusterConfig);
        Set<String> offlineOrDisabledInstances = new HashSet<>(activeNodes);
        offlineOrDisabledInstances.removeAll(liveEnabledNodes);
        setRebalanceScheduler(currentIdealState, offlineOrDisabledInstances, clusterData.getInstanceOfflineTimeMap(), clusterData.getLiveInstances().keySet(), clusterData.getInstanceConfigMap(), delay, clusterConfig);
    }
    if (allNodes.isEmpty() || activeNodes.isEmpty()) {
        LOG.error(String.format("No instances or active instances available for resource %s, " + "allInstances: %s, liveInstances: %s, activeInstances: %s", resourceName, allNodes, liveEnabledNodes, activeNodes));
        return generateNewIdealState(resourceName, currentIdealState, emptyMapping(currentIdealState));
    }
    StateModelDefinition stateModelDef = clusterData.getStateModelDef(currentIdealState.getStateModelDefRef());
    int replicaCount = currentIdealState.getReplicaCount(activeNodes.size());
    if (replicaCount == 0) {
        LOG.error("Replica count is 0 for resource " + resourceName + ", stop calculate ideal mapping for the resource.");
        return generateNewIdealState(resourceName, currentIdealState, emptyMapping(currentIdealState));
    }
    LinkedHashMap<String, Integer> stateCountMap = stateModelDef.getStateCountMap(activeNodes.size(), replicaCount);
    Map<String, Map<String, String>> currentMapping = currentMapping(currentStateOutput, resourceName, allPartitions, stateCountMap);
    List<String> partitionsToAssign = new ArrayList<>(allPartitions);
    partitionsToAssign.removeAll(userDefinedPreferenceList.keySet());
    int maxPartition = currentIdealState.getMaxPartitionsPerInstance();
    _rebalanceStrategy = getRebalanceStrategy(currentIdealState.getRebalanceStrategy(), partitionsToAssign, resourceName, stateCountMap, maxPartition);
    // sort node lists to ensure consistent preferred assignments
    List<String> allNodeList = new ArrayList<>(allNodes);
    List<String> liveEnabledNodeList = new ArrayList<>(liveEnabledNodes);
    Collections.sort(allNodeList);
    Collections.sort(liveEnabledNodeList);
    ZNRecord newIdealMapping = _rebalanceStrategy.computePartitionAssignment(allNodeList, liveEnabledNodeList, currentMapping, clusterData);
    ZNRecord finalMapping = newIdealMapping;
    if (isDelayRebalanceEnabled(currentIdealState, clusterConfig)) {
        List<String> activeNodeList = new ArrayList<>(activeNodes);
        Collections.sort(activeNodeList);
        int minActiveReplicas = getMinActiveReplica(currentIdealState, replicaCount);
        ZNRecord newActiveMapping = _rebalanceStrategy.computePartitionAssignment(allNodeList, activeNodeList, currentMapping, clusterData);
        finalMapping = getFinalDelayedMapping(currentIdealState, newIdealMapping, newActiveMapping, liveEnabledNodes, replicaCount, minActiveReplicas);
        LOG.debug("newActiveMapping: " + newActiveMapping);
    }
    finalMapping.getListFields().putAll(userDefinedPreferenceList);
    if (LOG.isDebugEnabled()) {
        LOG.debug("currentMapping: " + currentMapping);
        LOG.debug("stateCountMap: " + stateCountMap);
        LOG.debug("liveEnabledNodes: " + liveEnabledNodes);
        LOG.debug("activeNodes: " + activeNodes);
        LOG.debug("allNodes: " + allNodes);
        LOG.debug("maxPartition: " + maxPartition);
        LOG.debug("newIdealMapping: " + newIdealMapping);
        LOG.debug("finalMapping: " + finalMapping);
    }
    IdealState idealState = generateNewIdealState(resourceName, currentIdealState, finalMapping);
    clusterData.setCachedIdealMapping(resourceName, idealState.getRecord());
    return idealState;
}

Example 4 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestClusterAccessor method createClusterConfig.

private ClusterConfig createClusterConfig(String cluster) {
    ClusterConfig clusterConfig = _configAccessor.getClusterConfig(cluster);
    clusterConfig.setPersistBestPossibleAssignment(true);
    clusterConfig.getRecord().setSimpleField("SimpleField1", "Value1");
    clusterConfig.getRecord().setSimpleField("SimpleField2", "Value2");
    clusterConfig.getRecord().setListField("ListField1", Arrays.asList("Value1", "Value2", "Value3"));
    clusterConfig.getRecord().setListField("ListField2", Arrays.asList("Value2", "Value1", "Value3"));
    clusterConfig.getRecord().setMapField("MapField1", new HashMap<String, String>() {

        {
            put("key1", "value1");
            put("key2", "value2");
        }
    });
    clusterConfig.getRecord().setMapField("MapField2", new HashMap<String, String>() {

        {
            put("key3", "value1");
            put("key4", "value2");
        }
    });
    return clusterConfig;
}

Example 5 with ClusterConfig

use of org.apache.helix.model.ClusterConfig in project helix by apache.

the class TestClusterAccessor method testDeleteConfigFields.

@Test(dependsOnMethods = "testUpdateConfigFields")
public void testDeleteConfigFields() throws IOException {
    System.out.println("Start test :" + TestHelper.getTestMethodName());
    String cluster = _clusters.iterator().next();
    ClusterConfig config = getClusterConfigFromRest(cluster);
    ZNRecord record = config.getRecord();
    String simpleKey = record.getSimpleFields().keySet().iterator().next();
    String value = record.getSimpleField(simpleKey);
    record.getSimpleFields().clear();
    record.setSimpleField(simpleKey, value);
    String listKey = record.getListFields().keySet().iterator().next();
    List<String> list = record.getListField(listKey);
    record.getListFields().clear();
    record.setListField(listKey, list);
    String mapKey = record.getMapFields().keySet().iterator().next();
    Map<String, String> map = record.getMapField(mapKey);
    record.getMapFields().clear();
    record.setMapField(mapKey, map);
    ClusterConfig prevConfig = getClusterConfigFromRest(cluster);
    updateClusterConfigFromRest(cluster, config, Command.delete);
    ClusterConfig newConfig = getClusterConfigFromRest(cluster);
    Assert.assertFalse(newConfig.getRecord().getSimpleFields().containsKey(simpleKey), "Failed to delete key " + simpleKey + " from cluster config");
    Assert.assertFalse(newConfig.getRecord().getListFields().containsKey(listKey), "Failed to delete key " + listKey + " from cluster config");
    Assert.assertFalse(newConfig.getRecord().getSimpleFields().containsKey(mapKey), "Failed to delete key " + mapKey + " from cluster config");
    prevConfig.getRecord().subtract(config.getRecord());
    Assert.assertEquals(newConfig, prevConfig, "cluster config from response: " + newConfig + " vs cluster config actually: " + prevConfig);
}