Examples with ClusterDataCache org.apache.helix.controller.stages.ClusterDataCache used on opensource projects

Example 11 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class HelixUtil method getIdealAssignmentForFullAuto.

/**
 * This method provides the ideal state mapping with corresponding rebalance strategy
 * @param clusterConfig         The cluster config
 * @param instanceConfigs       List of instance configs
 * @param liveInstances         List of live instance names
 * @param idealState            The ideal state of current resource. If input is null, will be
 *                              treated as newly created resource.
 * @param partitions            The list of partition names
 * @param strategyClassName          The rebalance strategy. e.g. AutoRebalanceStrategy
 * @return A map of ideal state assignment as partition -> instance -> state
 */
public static Map<String, Map<String, String>> getIdealAssignmentForFullAuto(ClusterConfig clusterConfig, List<InstanceConfig> instanceConfigs, List<String> liveInstances, IdealState idealState, List<String> partitions, String strategyClassName) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
    List<String> allNodes = new ArrayList<>();
    Map<String, InstanceConfig> instanceConfigMap = new HashMap<>();
    for (InstanceConfig instanceConfig : instanceConfigs) {
        allNodes.add(instanceConfig.getInstanceName());
        instanceConfigMap.put(instanceConfig.getInstanceName(), instanceConfig);
    }
    ClusterDataCache cache = new ClusterDataCache();
    cache.setClusterConfig(clusterConfig);
    cache.setInstanceConfigMap(instanceConfigMap);
    StateModelDefinition stateModelDefinition = BuiltInStateModelDefinitions.valueOf(idealState.getStateModelDefRef()).getStateModelDefinition();
    RebalanceStrategy strategy = RebalanceStrategy.class.cast(loadClass(HelixUtil.class, strategyClassName).newInstance());
    strategy.init(idealState.getResourceName(), partitions, stateModelDefinition.getStateCountMap(liveInstances.size(), Integer.parseInt(idealState.getReplicas())), idealState.getMaxPartitionsPerInstance());
    Map<String, List<String>> preferenceLists = strategy.computePartitionAssignment(allNodes, liveInstances, new HashMap<String, Map<String, String>>(), cache).getListFields();
    Map<String, Map<String, String>> idealStateMapping = new HashMap<>();
    Set<String> liveInstanceSet = new HashSet<>(liveInstances);
    for (String partitionName : preferenceLists.keySet()) {
        idealStateMapping.put(partitionName, computeIdealMapping(preferenceLists.get(partitionName), stateModelDefinition, liveInstanceSet));
    }
    return idealStateMapping;
}

Example 12 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TestHelixDataAccessor method testClusterDataCache.

@Test(expectedExceptions = { HelixMetaDataAccessException.class })
public void testClusterDataCache() {
    ClusterDataCache cache = new ClusterDataCache("MyCluster");
    cache.refresh(accessor);
}

Example 13 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TestPersistAssignmentStage method testSimple.

/**
 * Case where we have one resource in IdealState
 * @throws Exception
 */
@Test
public void testSimple() throws Exception {
    int nodes = 2;
    List<String> instances = new ArrayList<String>();
    for (int i = 0; i < nodes; i++) {
        instances.add("localhost_" + i);
    }
    int partitions = 10;
    int replicas = 1;
    String resourceName = "testResource";
    ZNRecord record = DefaultIdealStateCalculator.calculateIdealState(instances, partitions, replicas, resourceName, "ONLINE", "OFFLINE");
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef("OnlineOffline");
    // Read and load current state into event
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
    runStage(_manager, event, new ReadClusterDataStage());
    runStage(_manager, event, new ResourceComputationStage());
    // Ensure persist best possible assignment is true
    ClusterConfig clusterConfig = new ClusterConfig(CLUSTER_NAME);
    clusterConfig.setPersistBestPossibleAssignment(true);
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.setClusterConfig(clusterConfig);
    // 1. Change best possible state (simulate a new rebalancer run)
    BestPossibleStateOutput bestPossibleStateOutput = new BestPossibleStateOutput();
    for (String partition : idealState.getPartitionSet()) {
        bestPossibleStateOutput.setState(resourceName, new Partition(partition), "localhost_3", "OFFLINE");
    }
    // 2. At the same time, set DelayRebalanceEnabled = true (simulate a Admin operation at the same time)
    idealState.setDelayRebalanceEnabled(true);
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
    // Persist new assignment
    PersistAssignmentStage stage = new PersistAssignmentStage();
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    runStage(_manager, event, stage);
    IdealState newIdealState = accessor.getProperty(keyBuilder.idealStates(resourceName));
    // 1. New assignment should be set
    Assert.assertEquals(newIdealState.getPartitionSet().size(), idealState.getPartitionSet().size());
    for (String partition : idealState.getPartitionSet()) {
        Map<String, String> assignment = newIdealState.getInstanceStateMap(partition);
        Assert.assertNotNull(assignment);
        Assert.assertEquals(assignment.size(), 1);
        Assert.assertTrue(assignment.containsKey("localhost_3") && assignment.get("localhost_3").equals("OFFLINE"));
    }
    // 2. Admin config should be set
    Assert.assertTrue(newIdealState.isDelayRebalanceEnabled());
}

Example 14 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class ClusterExternalViewVerifier method verify.

@Override
public boolean verify() throws Exception {
    ClusterDataCache cache = new ClusterDataCache();
    cache.refresh(_accessor);
    List<String> liveInstances = new ArrayList<String>();
    liveInstances.addAll(cache.getLiveInstances().keySet());
    boolean success = verifyLiveNodes(liveInstances);
    if (!success) {
        LOG.info("liveNodes not match, expect: " + _expectSortedLiveNodes + ", actual: " + liveInstances);
        return false;
    }
    BestPossibleStateOutput bestPossbileStates = calculateBestPossibleState(cache);
    Map<String, ExternalView> externalViews = _accessor.getChildValuesMap(_keyBuilder.externalViews());
    for (String resourceName : externalViews.keySet()) {
        ExternalView externalView = externalViews.get(resourceName);
        Map<Partition, Map<String, String>> bestPossbileState = bestPossbileStates.getResourceMap(resourceName);
        success = verifyExternalView(externalView, bestPossbileState);
        if (!success) {
            LOG.info("external-view for resource: " + resourceName + " not match");
            return false;
        }
    }
    return true;
}

Example 15 with ClusterDataCache

use of org.apache.helix.controller.stages.ClusterDataCache in project helix by apache.

the class TestP2PStateTransitionMessages method testP2PMessage.

private void testP2PMessage(ClusterConfig clusterConfig, Boolean p2pMessageEnabled) throws Exception {
    Map<String, Resource> resourceMap = getResourceMap(new String[] { db }, numPartition, BuiltInStateModelDefinitions.MasterSlave.name(), clusterConfig, null);
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), new CurrentStateOutput());
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    Pipeline pipeline = createPipeline();
    pipeline.handle(event);
    BestPossibleStateOutput bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    CurrentStateOutput currentStateOutput = populateCurrentStateFromBestPossible(bestPossibleStateOutput);
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
    Partition p = new Partition(db + "_0");
    String masterInstance = getTopStateInstance(bestPossibleStateOutput.getInstanceStateMap(db, p), MasterSlaveSMD.States.MASTER.name());
    Assert.assertNotNull(masterInstance);
    admin.enableInstance(_clusterName, masterInstance, false);
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.notifyDataChange(HelixConstants.ChangeType.INSTANCE_CONFIG);
    pipeline.handle(event);
    bestPossibleStateOutput = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    MessageSelectionStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = messageOutput.getMessages(db, p);
    Assert.assertEquals(messages.size(), 1);
    Message message = messages.get(0);
    Assert.assertEquals(message.getTgtName(), masterInstance);
    Assert.assertEquals(message.getFromState(), MasterSlaveSMD.States.MASTER.name());
    Assert.assertEquals(message.getToState(), MasterSlaveSMD.States.SLAVE.name());
    if (p2pMessageEnabled) {
        Assert.assertEquals(message.getRelayMessages().entrySet().size(), 1);
        String newMasterInstance = getTopStateInstance(bestPossibleStateOutput.getInstanceStateMap(db, p), MasterSlaveSMD.States.MASTER.name());
        Message relayMessage = message.getRelayMessage(newMasterInstance);
        Assert.assertNotNull(relayMessage);
        Assert.assertEquals(relayMessage.getMsgSubType(), Message.MessageType.RELAYED_MESSAGE.name());
        Assert.assertEquals(relayMessage.getTgtName(), newMasterInstance);
        Assert.assertEquals(relayMessage.getRelaySrcHost(), masterInstance);
        Assert.assertEquals(relayMessage.getFromState(), MasterSlaveSMD.States.SLAVE.name());
        Assert.assertEquals(relayMessage.getToState(), MasterSlaveSMD.States.MASTER.name());
    } else {
        Assert.assertTrue(message.getRelayMessages().entrySet().isEmpty());
    }
}