Examples with BestPossibleStateOutput org.apache.helix.controller.stages.BestPossibleStateOutput used on opensource projects

Example 1 with BestPossibleStateOutput

use of org.apache.helix.controller.stages.BestPossibleStateOutput in project helix by apache.

the class TestRebalancerMetrics method testLoadBalanceMetrics.

@Test
public void testLoadBalanceMetrics() {
    System.out.println("START testLoadBalanceMetrics at " + new Date(System.currentTimeMillis()));
    String resource = "testResourceName";
    int numPartition = 100;
    int numReplica = 3;
    int maxPending = 3;
    setupIdealState(5, new String[] { resource }, numPartition, numReplica, IdealState.RebalanceMode.FULL_AUTO, BuiltInStateModelDefinitions.MasterSlave.name());
    setupInstances(5);
    setupLiveInstances(4);
    setupStateModel();
    Map<String, Resource> resourceMap = getResourceMap(new String[] { resource }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name());
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    ClusterStatusMonitor monitor = new ClusterStatusMonitor(_clusterName);
    monitor.active();
    event.addAttribute(AttributeName.clusterStatusMonitor.name(), monitor);
    runStage(event, new ReadClusterDataStage());
    runStage(event, new BestPossibleStateCalcStage());
    BestPossibleStateOutput bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    currentStateOutput = copyCurrentStateFromBestPossible(bestPossibleStateOutput, resource);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    setupLiveInstances(4);
    runStage(event, new ReadClusterDataStage());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    setupThrottleConfig(cache.getClusterConfig(), StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, maxPending);
    runStage(event, new BestPossibleStateCalcStage());
    runStage(event, new IntermediateStateCalcStage());
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    ResourceMonitor resourceMonitor = clusterStatusMonitor.getResourceMonitor(resource);
    long numPendingLoadBalance = resourceMonitor.getPendingLoadRebalancePartitionGauge();
    Assert.assertTrue(numPendingLoadBalance > 0);
    Assert.assertEquals(resourceMonitor.getLoadRebalanceThrottledPartitionGauge(), numPendingLoadBalance - maxPending);
    System.out.println("END testLoadBalanceMetrics at " + new Date(System.currentTimeMillis()));
}

Example 2 with BestPossibleStateOutput

use of org.apache.helix.controller.stages.BestPossibleStateOutput in project helix by apache.

the class BestPossibleExternalViewVerifier method calcBestPossState.

/**
 * calculate the best possible state note that DROPPED states are not checked since when
 * kick off the BestPossibleStateCalcStage we are providing an empty current state map
 *
 * @param cache
 * @return
 * @throws Exception
 */
private BestPossibleStateOutput calcBestPossState(ClusterDataCache cache) throws Exception {
    ClusterEvent event = new ClusterEvent(ClusterEventType.StateVerifier);
    event.addAttribute(AttributeName.ClusterDataCache.name(), cache);
    runStage(event, new ResourceComputationStage());
    runStage(event, new CurrentStateComputationStage());
    // TODO: be caution here, should be handled statelessly.
    runStage(event, new BestPossibleStateCalcStage());
    BestPossibleStateOutput output = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    return output;
}

Example 3 with BestPossibleStateOutput

use of org.apache.helix.controller.stages.BestPossibleStateOutput in project helix by apache.

the class BestPossibleExternalViewVerifier method verifyState.

@Override
protected synchronized boolean verifyState() {
    try {
        PropertyKey.Builder keyBuilder = _accessor.keyBuilder();
        // read cluster once and do verification
        ClusterDataCache cache = new ClusterDataCache();
        cache.refresh(_accessor);
        Map<String, IdealState> idealStates = cache.getIdealStates();
        if (idealStates == null) {
            // ideal state is null because ideal state is dropped
            idealStates = Collections.emptyMap();
        }
        // filter out all resources that use Task state model
        Iterator<Map.Entry<String, IdealState>> it = idealStates.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, IdealState> pair = it.next();
            if (pair.getValue().getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME)) {
                it.remove();
            }
        }
        // verify live instances.
        if (_expectLiveInstances != null && !_expectLiveInstances.isEmpty()) {
            Set<String> actualLiveNodes = cache.getLiveInstances().keySet();
            if (!_expectLiveInstances.equals(actualLiveNodes)) {
                LOG.warn("Live instances are not as expected. Actual live nodes: " + actualLiveNodes.toString());
                return false;
            }
        }
        Map<String, ExternalView> extViews = _accessor.getChildValuesMap(keyBuilder.externalViews());
        if (extViews == null) {
            extViews = Collections.emptyMap();
        }
        // Filter resources if requested
        if (_resources != null && !_resources.isEmpty()) {
            idealStates.keySet().retainAll(_resources);
            extViews.keySet().retainAll(_resources);
        }
        // add empty idealState for the resource
        for (String resource : extViews.keySet()) {
            if (!idealStates.containsKey(resource)) {
                ExternalView ev = extViews.get(resource);
                IdealState is = new IdealState(resource);
                is.getRecord().setSimpleFields(ev.getRecord().getSimpleFields());
                idealStates.put(resource, is);
            }
        }
        // calculate best possible state
        BestPossibleStateOutput bestPossOutput = calcBestPossState(cache);
        Map<String, Map<Partition, Map<String, String>>> bestPossStateMap = bestPossOutput.getStateMap();
        // set error states
        if (_errStates != null) {
            for (String resourceName : _errStates.keySet()) {
                Map<String, String> partErrStates = _errStates.get(resourceName);
                for (String partitionName : partErrStates.keySet()) {
                    String instanceName = partErrStates.get(partitionName);
                    if (!bestPossStateMap.containsKey(resourceName)) {
                        bestPossStateMap.put(resourceName, new HashMap<Partition, Map<String, String>>());
                    }
                    Partition partition = new Partition(partitionName);
                    if (!bestPossStateMap.get(resourceName).containsKey(partition)) {
                        bestPossStateMap.get(resourceName).put(partition, new HashMap<String, String>());
                    }
                    bestPossStateMap.get(resourceName).get(partition).put(instanceName, HelixDefinedState.ERROR.toString());
                }
            }
        }
        for (String resourceName : idealStates.keySet()) {
            ExternalView extView = extViews.get(resourceName);
            IdealState is = idealStates.get(resourceName);
            if (extView == null) {
                if (is.isExternalViewDisabled()) {
                    continue;
                } else {
                    LOG.error("externalView for " + resourceName + " is not available");
                    return false;
                }
            }
            // step 0: remove empty map and DROPPED state from best possible state
            PartitionStateMap bpStateMap = bestPossOutput.getPartitionStateMap(resourceName);
            StateModelDefinition stateModelDef = cache.getStateModelDef(is.getStateModelDefRef());
            if (stateModelDef == null) {
                LOG.error("State model definition " + is.getStateModelDefRef() + " for resource not found!" + is.getResourceName());
                return false;
            }
            boolean result = verifyExternalView(extView, bpStateMap, stateModelDef);
            if (!result) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("verifyExternalView fails for " + resourceName + "! ExternalView: " + extView + " BestPossibleState: " + bpStateMap);
                } else {
                    LOG.warn("verifyExternalView fails for " + resourceName + "! ExternalView does not match BestPossibleState");
                }
                return false;
            }
        }
        return true;
    } catch (Exception e) {
        LOG.error("exception in verification", e);
        return false;
    }
}

Example 4 with BestPossibleStateOutput

use of org.apache.helix.controller.stages.BestPossibleStateOutput in project helix by apache.

the class TestPersistAssignmentStage method testSimple.

/**
 * Case where we have one resource in IdealState
 * @throws Exception
 */
@Test
public void testSimple() throws Exception {
    int nodes = 2;
    List<String> instances = new ArrayList<String>();
    for (int i = 0; i < nodes; i++) {
        instances.add("localhost_" + i);
    }
    int partitions = 10;
    int replicas = 1;
    String resourceName = "testResource";
    ZNRecord record = DefaultIdealStateCalculator.calculateIdealState(instances, partitions, replicas, resourceName, "ONLINE", "OFFLINE");
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef("OnlineOffline");
    // Read and load current state into event
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
    runStage(_manager, event, new ReadClusterDataStage());
    runStage(_manager, event, new ResourceComputationStage());
    // Ensure persist best possible assignment is true
    ClusterConfig clusterConfig = new ClusterConfig(CLUSTER_NAME);
    clusterConfig.setPersistBestPossibleAssignment(true);
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.setClusterConfig(clusterConfig);
    // 1. Change best possible state (simulate a new rebalancer run)
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    for (String partition : idealState.getPartitionSet()) {
        bestPossibleStateOutput.setState(resourceName, new Partition(partition), "localhost_3", "OFFLINE");
    }
    // 2. At the same time, set DelayRebalanceEnabled = true (simulate a Admin operation at the same time)
    idealState.setDelayRebalanceEnabled(true);
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
    // Persist new assignment
    PersistAssignmentStage stage = new PersistAssignmentStage();
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    runStage(_manager, event, stage);
    IdealState newIdealState = accessor.getProperty(keyBuilder.idealStates(resourceName));
    // 1. New assignment should be set
    Assert.assertEquals(newIdealState.getPartitionSet().size(), idealState.getPartitionSet().size());
    for (String partition : idealState.getPartitionSet()) {
        Map<String, String> assignment = newIdealState.getInstanceStateMap(partition);
        Assert.assertNotNull(assignment);
        Assert.assertEquals(assignment.size(), 1);
        Assert.assertTrue(assignment.containsKey("localhost_3") && assignment.get("localhost_3").equals("OFFLINE"));
    }
    // 2. Admin config should be set
    Assert.assertTrue(newIdealState.isDelayRebalanceEnabled());
}

Example 5 with BestPossibleStateOutput

use of org.apache.helix.controller.stages.BestPossibleStateOutput in project helix by apache.

the class ClusterExternalViewVerifier method verify.

@Override
public boolean verify() throws Exception {
    ClusterDataCache cache = new ClusterDataCache();
    cache.refresh(_accessor);
    List<String> liveInstances = new ArrayList<String>();
    liveInstances.addAll(cache.getLiveInstances().keySet());
    boolean success = verifyLiveNodes(liveInstances);
    if (!success) {
        LOG.info("liveNodes not match, expect: " + _expectSortedLiveNodes + ", actual: " + liveInstances);
        return false;
    }
    BestPossibleStateOutput bestPossbileStates = calculateBestPossibleState(cache);
    Map<String, ExternalView> externalViews = _accessor.getChildValuesMap(_keyBuilder.externalViews());
    for (String resourceName : externalViews.keySet()) {
        ExternalView externalView = externalViews.get(resourceName);
        Map<Partition, Map<String, String>> bestPossbileState = bestPossbileStates.getResourceMap(resourceName);
        success = verifyExternalView(externalView, bestPossbileState);
        if (!success) {
            LOG.info("external-view for resource: " + resourceName + " not match");
            return false;
        }
    }
    return true;
}