Examples with ClusterDataCache org.apache.helix.controller.stages.ClusterDataCache used on opensource projects

Example 1 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TestJobStateOnCreation method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    _cache = new ClusterDataCache();
    _idealState = new IdealState(WORKFLOW_NAME);
    _resource = new Resource(WORKFLOW_NAME);
    _currStateOutput = new CurrentStateOutput();
    _participants = new MockParticipantManager[_numNodes];
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    _setupTool = new ClusterSetup(ZK_ADDR);
    _setupTool.addCluster(CLUSTER_NAME, true);
    createManagers();
}

Example 2 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TestRebalancerMetrics method testLoadBalanceMetrics.

@Test
public void testLoadBalanceMetrics() {
    System.out.println("START testLoadBalanceMetrics at " + new Date(System.currentTimeMillis()));
    String resource = "testResourceName";
    int numPartition = 100;
    int numReplica = 3;
    int maxPending = 3;
    setupIdealState(5, new String[] { resource }, numPartition, numReplica, IdealState.RebalanceMode.FULL_AUTO, BuiltInStateModelDefinitions.MasterSlave.name());
    setupInstances(5);
    setupLiveInstances(4);
    setupStateModel();
    Map<String, Resource> resourceMap = getResourceMap(new String[] { resource }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name());
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    ClusterStatusMonitor monitor = new ClusterStatusMonitor(_clusterName);
    monitor.active();
    event.addAttribute(AttributeName.clusterStatusMonitor.name(), monitor);
    runStage(event, new ReadClusterDataStage());
    runStage(event, new BestPossibleStateCalcStage());
    BestPossibleStateOutput bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    currentStateOutput = copyCurrentStateFromBestPossible(bestPossibleStateOutput, resource);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    setupLiveInstances(4);
    runStage(event, new ReadClusterDataStage());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    setupThrottleConfig(cache.getClusterConfig(), StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, maxPending);
    runStage(event, new BestPossibleStateCalcStage());
    runStage(event, new IntermediateStateCalcStage());
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    ResourceMonitor resourceMonitor = clusterStatusMonitor.getResourceMonitor(resource);
    long numPendingLoadBalance = resourceMonitor.getPendingLoadRebalancePartitionGauge();
    Assert.assertTrue(numPendingLoadBalance > 0);
    Assert.assertEquals(resourceMonitor.getLoadRebalanceThrottledPartitionGauge(), numPendingLoadBalance - maxPending);
    System.out.println("END testLoadBalanceMetrics at " + new Date(System.currentTimeMillis()));
}

Example 3 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class StrictMatchExternalViewVerifier method verifyState.

@Override
protected boolean verifyState() {
    try {
        PropertyKey.Builder keyBuilder = _accessor.keyBuilder();
        // read cluster once and do verification
        ClusterDataCache cache = new ClusterDataCache();
        cache.refresh(_accessor);
        Map<String, IdealState> idealStates = cache.getIdealStates();
        if (idealStates == null) {
            // ideal state is null because ideal state is dropped
            idealStates = Collections.emptyMap();
        }
        // filter out all resources that use Task state model
        Iterator<Map.Entry<String, IdealState>> it = idealStates.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, IdealState> pair = it.next();
            if (pair.getValue().getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME)) {
                it.remove();
            }
        }
        // verify live instances.
        if (_expectLiveInstances != null && !_expectLiveInstances.isEmpty()) {
            Set<String> actualLiveNodes = cache.getLiveInstances().keySet();
            if (!_expectLiveInstances.equals(actualLiveNodes)) {
                return false;
            }
        }
        Map<String, ExternalView> extViews = _accessor.getChildValuesMap(keyBuilder.externalViews());
        if (extViews == null) {
            extViews = Collections.emptyMap();
        }
        // Filter resources if requested
        if (_resources != null && !_resources.isEmpty()) {
            idealStates.keySet().retainAll(_resources);
            extViews.keySet().retainAll(_resources);
        }
        // add empty idealState for the resource
        for (String resource : extViews.keySet()) {
            if (!idealStates.containsKey(resource)) {
                idealStates.put(resource, new IdealState(resource));
            }
        }
        for (String resourceName : idealStates.keySet()) {
            ExternalView extView = extViews.get(resourceName);
            IdealState idealState = idealStates.get(resourceName);
            if (extView == null) {
                if (idealState.isExternalViewDisabled()) {
                    continue;
                } else {
                    LOG.debug("externalView for " + resourceName + " is not available");
                    return false;
                }
            }
            boolean result = verifyExternalView(cache, extView, idealState);
            if (!result) {
                return false;
            }
        }
        return true;
    } catch (Exception e) {
        LOG.error("exception in verification", e);
        return false;
    }
}

Example 4 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TaskTestUtil method buildClusterDataCache.

public static ClusterDataCache buildClusterDataCache(HelixDataAccessor accessor, String clusterName) {
    ClusterDataCache cache = new ClusterDataCache(clusterName);
    cache.refresh(accessor);
    cache.setTaskCache(true);
    return cache;
}

Example 5 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TestClusterDataCacheSelectiveUpdate method testUpdateOnNotification.

@Test()
public void testUpdateOnNotification() throws Exception {
    MockZkHelixDataAccessor accessor = new MockZkHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    ClusterDataCache cache = new ClusterDataCache("CLUSTER_" + TestHelper.getTestClassName());
    cache.refresh(accessor);
    Assert.assertEquals(accessor.getReadCount(PropertyType.IDEALSTATES), 1);
    Assert.assertEquals(accessor.getReadCount(PropertyType.LIVEINSTANCES), NODE_NR);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CURRENTSTATES), NODE_NR);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), NODE_NR + 1);
    accessor.clearReadCounters();
    // refresh again should read nothing
    cache.refresh(accessor);
    Assert.assertEquals(accessor.getReadCount(PropertyType.IDEALSTATES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.LIVEINSTANCES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CURRENTSTATES), 0);
    // cluster config always get reloaded
    Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), 1);
    accessor.clearReadCounters();
    // refresh again should read nothing as ideal state is same
    cache.notifyDataChange(HelixConstants.ChangeType.IDEAL_STATE);
    cache.refresh(accessor);
    Assert.assertEquals(accessor.getReadCount(PropertyType.IDEALSTATES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.LIVEINSTANCES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CURRENTSTATES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), 1);
    accessor.clearReadCounters();
    cache.notifyDataChange(HelixConstants.ChangeType.LIVE_INSTANCE);
    cache.refresh(accessor);
    Assert.assertEquals(accessor.getReadCount(PropertyType.IDEALSTATES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.LIVEINSTANCES), NODE_NR);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CURRENTSTATES), 0);
    Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), 1);
}