Examples with IdealState org.apache.helix.model.IdealState used on opensource projects

Example 76 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class IdealStateCalculatorForEspressoRelay method calculateRelayIdealState.

public static IdealState calculateRelayIdealState(List<String> partitions, List<String> instances, String resultRecordName, int replica, String firstValue, String restValue, String stateModelName) {
    Collections.sort(partitions);
    Collections.sort(instances);
    if (instances.size() % replica != 0) {
        throw new HelixException("Instances must be divided by replica");
    }
    IdealState result = new IdealState(resultRecordName);
    result.setNumPartitions(partitions.size());
    result.setReplicas("" + replica);
    result.setStateModelDefRef(stateModelName);
    int groups = instances.size() / replica;
    int remainder = instances.size() % replica;
    int remainder2 = partitions.size() % groups;
    int storageNodeGroupSize = partitions.size() / groups;
    for (int i = 0; i < groups; i++) {
        int relayStart = 0, relayEnd = 0, storageNodeStart = 0, storageNodeEnd = 0;
        if (i < remainder) {
            relayStart = (replica + 1) * i;
            relayEnd = (replica + 1) * (i + 1);
        } else {
            relayStart = (replica + 1) * remainder + replica * (i - remainder);
            relayEnd = relayStart + replica;
        }
        // System.out.println("relay start :" + relayStart + " relayEnd:" + relayEnd);
        if (i < remainder2) {
            storageNodeStart = (storageNodeGroupSize + 1) * i;
            storageNodeEnd = (storageNodeGroupSize + 1) * (i + 1);
        } else {
            storageNodeStart = (storageNodeGroupSize + 1) * remainder2 + storageNodeGroupSize * (i - remainder2);
            storageNodeEnd = storageNodeStart + storageNodeGroupSize;
        }
        // System.out.println("storageNodeStart :" + storageNodeStart + " storageNodeEnd:" +
        // storageNodeEnd);
        List<String> snBatch = partitions.subList(storageNodeStart, storageNodeEnd);
        List<String> relayBatch = instances.subList(relayStart, relayEnd);
        Map<String, List<String>> sublistFields = calculateSubIdealState(snBatch, relayBatch, replica);
        result.getRecord().getListFields().putAll(sublistFields);
    }
    for (String snName : result.getRecord().getListFields().keySet()) {
        Map<String, String> mapField = new TreeMap<String, String>();
        List<String> relayCandidates = result.getRecord().getListField(snName);
        mapField.put(relayCandidates.get(0), firstValue);
        for (int i = 1; i < relayCandidates.size(); i++) {
            mapField.put(relayCandidates.get(i), restValue);
        }
        result.getRecord().getMapFields().put(snName, mapField);
    }
    System.out.println();
    return result;
}

Example 77 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestRelayIdealStateCalculator method testEspressoStorageClusterIdealState.

public void testEspressoStorageClusterIdealState(int partitions, int nodes, int replica) throws Exception {
    List<String> storageNodes = new ArrayList<String>();
    for (int i = 0; i < partitions; i++) {
        storageNodes.add("localhost:123" + i);
    }
    List<String> relays = new ArrayList<String>();
    for (int i = 0; i < nodes; i++) {
        relays.add("relay:123" + i);
    }
    IdealState idealstate = IdealStateCalculatorForEspressoRelay.calculateRelayIdealState(storageNodes, relays, "TEST", replica, "Leader", "Standby", "LeaderStandby");
    Assert.assertEquals(idealstate.getRecord().getListFields().size(), idealstate.getRecord().getMapFields().size());
    Map<String, Integer> countMap = new TreeMap<String, Integer>();
    for (String key : idealstate.getRecord().getListFields().keySet()) {
        Assert.assertEquals(idealstate.getRecord().getListFields().get(key).size(), idealstate.getRecord().getMapFields().get(key).size());
        List<String> list = idealstate.getRecord().getListFields().get(key);
        Map<String, String> map = idealstate.getRecord().getMapFields().get(key);
        Assert.assertEquals(list.size(), replica);
        for (String val : list) {
            if (!countMap.containsKey(val)) {
                countMap.put(val, 1);
            } else {
                countMap.put(val, countMap.get(val) + 1);
            }
            Assert.assertTrue(map.containsKey(val));
        }
    }
    for (String nodeName : countMap.keySet()) {
        Assert.assertTrue(countMap.get(nodeName) <= partitions * replica / nodes + 1);
    // System.out.println(nodeName + " " + countMap.get(nodeName));
    }
    System.out.println();
}

Example 78 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class TestZKCallback method testInvocation.

@Test()
public void testInvocation() throws Exception {
    HelixManager testHelixManager = HelixManagerFactory.getZKHelixManager(clusterName, "localhost_8900", InstanceType.PARTICIPANT, ZK_ADDR);
    testHelixManager.connect();
    TestZKCallback test = new TestZKCallback();
    TestZKCallback.TestCallbackListener testListener = test.new TestCallbackListener();
    testHelixManager.addMessageListener(testListener, "localhost_8900");
    testHelixManager.addCurrentStateChangeListener(testListener, "localhost_8900", testHelixManager.getSessionId());
    testHelixManager.addConfigChangeListener(testListener);
    testHelixManager.addIdealStateChangeListener(testListener);
    testHelixManager.addExternalViewChangeListener(testListener);
    testHelixManager.addLiveInstanceChangeListener(testListener);
    // Initial add listener should trigger the first execution of the
    // listener callbacks
    AssertJUnit.assertTrue(testListener.configChangeReceived & testListener.currentStateChangeReceived & testListener.externalViewChangeReceived & testListener.idealStateChangeReceived & testListener.liveInstanceChangeReceived & testListener.messageChangeReceived);
    testListener.Reset();
    HelixDataAccessor accessor = testHelixManager.getHelixDataAccessor();
    Builder keyBuilder = accessor.keyBuilder();
    ExternalView extView = new ExternalView("db-12345");
    accessor.setProperty(keyBuilder.externalView("db-12345"), extView);
    Thread.sleep(100);
    AssertJUnit.assertTrue(testListener.externalViewChangeReceived);
    testListener.Reset();
    CurrentState curState = new CurrentState("db-12345");
    curState.setSessionId("sessionId");
    curState.setStateModelDefRef("StateModelDef");
    accessor.setProperty(keyBuilder.currentState("localhost_8900", testHelixManager.getSessionId(), curState.getId()), curState);
    Thread.sleep(100);
    AssertJUnit.assertTrue(testListener.currentStateChangeReceived);
    testListener.Reset();
    IdealState idealState = new IdealState("db-1234");
    idealState.setNumPartitions(400);
    idealState.setReplicas(Integer.toString(2));
    idealState.setStateModelDefRef("StateModeldef");
    accessor.setProperty(keyBuilder.idealStates("db-1234"), idealState);
    Thread.sleep(100);
    AssertJUnit.assertTrue(testListener.idealStateChangeReceived);
    testListener.Reset();
    // dummyRecord = new ZNRecord("db-12345");
    // dataAccessor.setProperty(PropertyType.IDEALSTATES, idealState, "db-12345"
    // );
    // Thread.sleep(100);
    // AssertJUnit.assertTrue(testListener.idealStateChangeReceived);
    // testListener.Reset();
    // dummyRecord = new ZNRecord("localhost:8900");
    // List<ZNRecord> recList = new ArrayList<ZNRecord>();
    // recList.add(dummyRecord);
    testListener.Reset();
    Message message = new Message(MessageType.STATE_TRANSITION, UUID.randomUUID().toString());
    message.setTgtSessionId("*");
    message.setResourceName("testResource");
    message.setPartitionName("testPartitionKey");
    message.setStateModelDef("MasterSlave");
    message.setToState("toState");
    message.setFromState("fromState");
    message.setTgtName("testTarget");
    message.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);
    accessor.setProperty(keyBuilder.message("localhost_8900", message.getId()), message);
    Thread.sleep(500);
    AssertJUnit.assertTrue(testListener.messageChangeReceived);
    // dummyRecord = new ZNRecord("localhost_9801");
    LiveInstance liveInstance = new LiveInstance("localhost_9801");
    liveInstance.setSessionId(UUID.randomUUID().toString());
    liveInstance.setHelixVersion(UUID.randomUUID().toString());
    accessor.setProperty(keyBuilder.liveInstance("localhost_9801"), liveInstance);
    Thread.sleep(500);
    AssertJUnit.assertTrue(testListener.liveInstanceChangeReceived);
    testListener.Reset();
// dataAccessor.setNodeConfigs(recList); Thread.sleep(100);
// AssertJUnit.assertTrue(testListener.configChangeReceived);
// testListener.Reset();
}

Example 79 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class WorkflowRebalancer method scheduleSingleJob.

/**
 * Posts new job to cluster
 */
private void scheduleSingleJob(String jobResource, JobConfig jobConfig) {
    HelixAdmin admin = _manager.getClusterManagmentTool();
    IdealState jobIS = admin.getResourceIdealState(_manager.getClusterName(), jobResource);
    if (jobIS != null) {
        LOG.info("Job " + jobResource + " idealstate already exists!");
        return;
    }
    // Set up job resource based on partitions from target resource
    TaskUtil.createUserContent(_manager.getHelixPropertyStore(), jobResource, new ZNRecord(TaskUtil.USER_CONTENT_NODE));
    int numIndependentTasks = jobConfig.getTaskConfigMap().size();
    int numPartitions = numIndependentTasks;
    if (numPartitions == 0) {
        IdealState targetIs = admin.getResourceIdealState(_manager.getClusterName(), jobConfig.getTargetResource());
        if (targetIs == null) {
            LOG.warn("Target resource does not exist for job " + jobResource);
        // do not need to fail here, the job will be marked as failure immediately when job starts running.
        } else {
            numPartitions = targetIs.getPartitionSet().size();
        }
    }
    admin.addResource(_manager.getClusterName(), jobResource, numPartitions, TaskConstants.STATE_MODEL_NAME);
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    // Set the job configuration
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    HelixProperty resourceConfig = new HelixProperty(jobResource);
    resourceConfig.getRecord().getSimpleFields().putAll(jobConfig.getResourceConfigMap());
    Map<String, TaskConfig> taskConfigMap = jobConfig.getTaskConfigMap();
    if (taskConfigMap != null) {
        for (TaskConfig taskConfig : taskConfigMap.values()) {
            resourceConfig.getRecord().setMapField(taskConfig.getId(), taskConfig.getConfigMap());
        }
    }
    accessor.setProperty(keyBuilder.resourceConfig(jobResource), resourceConfig);
    // Push out new ideal state based on number of target partitions
    IdealStateBuilder builder = new CustomModeISBuilder(jobResource);
    builder.setRebalancerMode(IdealState.RebalanceMode.TASK);
    builder.setNumReplica(1);
    builder.setNumPartitions(numPartitions);
    builder.setStateModel(TaskConstants.STATE_MODEL_NAME);
    if (jobConfig.getInstanceGroupTag() != null) {
        builder.setNodeGroup(jobConfig.getInstanceGroupTag());
    }
    if (jobConfig.isDisableExternalView()) {
        builder.disableExternalView();
    }
    jobIS = builder.build();
    for (int i = 0; i < numPartitions; i++) {
        jobIS.getRecord().setListField(jobResource + "_" + i, new ArrayList<String>());
        jobIS.getRecord().setMapField(jobResource + "_" + i, new HashMap<String, String>());
    }
    jobIS.setRebalancerClassName(JobRebalancer.class.getName());
    admin.setResourceIdealState(_manager.getClusterName(), jobResource, jobIS);
}

Example 80 with IdealState

use of org.apache.helix.model.IdealState in project helix by apache.

the class ZKHelixAdmin method rebalance.

/**
 * Takes the existing idealstate as input and computes newIdealState such that the partition
 * movement is minimized. The partitions are redistributed among the instances provided.
 *
 * @param clusterName
 * @param currentIdealState
 * @param instanceNames
 *
 * @return
 */
@Override
public void rebalance(String clusterName, IdealState currentIdealState, List<String> instanceNames) {
    Set<String> activeInstances = new HashSet<String>();
    for (String partition : currentIdealState.getPartitionSet()) {
        activeInstances.addAll(currentIdealState.getRecord().getListField(partition));
    }
    instanceNames.removeAll(activeInstances);
    Map<String, Object> previousIdealState = RebalanceUtil.buildInternalIdealState(currentIdealState);
    Map<String, Object> balancedRecord = DefaultIdealStateCalculator.calculateNextIdealState(instanceNames, previousIdealState);
    StateModelDefinition stateModDef = this.getStateModelDef(clusterName, currentIdealState.getStateModelDefRef());
    if (stateModDef == null) {
        throw new HelixException("cannot find state model: " + currentIdealState.getStateModelDefRef());
    }
    String[] states = RebalanceUtil.parseStates(clusterName, stateModDef);
    ZNRecord newIdealStateRecord = DefaultIdealStateCalculator.convertToZNRecord(balancedRecord, currentIdealState.getResourceName(), states[0], states[1]);
    Set<String> partitionSet = new HashSet<String>();
    partitionSet.addAll(newIdealStateRecord.getMapFields().keySet());
    partitionSet.addAll(newIdealStateRecord.getListFields().keySet());
    Map<String, String> reversePartitionIndex = (Map<String, String>) balancedRecord.get("reversePartitionIndex");
    for (String partition : partitionSet) {
        if (reversePartitionIndex.containsKey(partition)) {
            String originPartitionName = reversePartitionIndex.get(partition);
            if (partition.equals(originPartitionName)) {
                continue;
            }
            newIdealStateRecord.getMapFields().put(originPartitionName, newIdealStateRecord.getMapField(partition));
            newIdealStateRecord.getMapFields().remove(partition);
            newIdealStateRecord.getListFields().put(originPartitionName, newIdealStateRecord.getListField(partition));
            newIdealStateRecord.getListFields().remove(partition);
        }
    }
    newIdealStateRecord.getSimpleFields().putAll(currentIdealState.getRecord().getSimpleFields());
    IdealState newIdealState = new IdealState(newIdealStateRecord);
    setResourceIdealState(clusterName, newIdealStateRecord.getId(), newIdealState);
}