Examples with ZNRecord org.apache.helix.ZNRecord used on opensource projects

Example 46 with ZNRecord

use of org.apache.helix.ZNRecord in project helix by apache.

the class AutoRebalanceStrategy method computePartitionAssignment.

@Override
public ZNRecord computePartitionAssignment(final List<String> allNodes, final List<String> liveNodes, final Map<String, Map<String, String>> currentMapping, ClusterDataCache clusterData) {
    int numReplicas = countStateReplicas();
    ZNRecord znRecord = new ZNRecord(_resourceName);
    if (liveNodes.size() == 0) {
        return znRecord;
    }
    List<String> sortedAllNodes = new ArrayList<String>(allNodes);
    Collections.sort(sortedAllNodes);
    Comparator<String> currentStateNodeComparator = new CurrentStateNodeComparator(currentMapping);
    List<String> sortedLiveNodes = new ArrayList<String>(liveNodes);
    Collections.sort(sortedLiveNodes, currentStateNodeComparator);
    int distRemainder = (numReplicas * _partitions.size()) % sortedLiveNodes.size();
    int distFloor = (numReplicas * _partitions.size()) / sortedLiveNodes.size();
    _nodeMap = new HashMap<String, Node>();
    _liveNodesList = new ArrayList<Node>();
    for (String id : sortedAllNodes) {
        Node node = new Node(id);
        node.capacity = 0;
        node.hasCeilingCapacity = false;
        _nodeMap.put(id, node);
    }
    for (int i = 0; i < sortedLiveNodes.size(); i++) {
        boolean usingCeiling = false;
        int targetSize = (_maximumPerNode > 0) ? Math.min(distFloor, _maximumPerNode) : distFloor;
        if (distRemainder > 0 && targetSize < _maximumPerNode) {
            targetSize += 1;
            distRemainder = distRemainder - 1;
            usingCeiling = true;
        }
        Node node = _nodeMap.get(sortedLiveNodes.get(i));
        node.isAlive = true;
        node.capacity = targetSize;
        node.hasCeilingCapacity = usingCeiling;
        _liveNodesList.add(node);
    }
    // compute states for all replica ids
    _stateMap = generateStateMap();
    // compute the preferred mapping if all nodes were up
    _preferredAssignment = computePreferredPlacement(sortedAllNodes);
    // logger.info("preferred mapping:"+ preferredAssignment);
    // from current mapping derive the ones in preferred location
    // this will update the nodes with their current fill status
    _existingPreferredAssignment = computeExistingPreferredPlacement(currentMapping);
    // from current mapping derive the ones not in preferred location
    _existingNonPreferredAssignment = computeExistingNonPreferredPlacement(currentMapping);
    // compute orphaned replicas that are not assigned to any node
    _orphaned = computeOrphaned();
    if (_orphaned.size() > 0 && logger.isInfoEnabled()) {
        logger.info("orphan = " + _orphaned);
    }
    assignOrphans();
    moveNonPreferredReplicasToPreferred();
    moveExcessReplicas();
    if (_orphaned.size() > 0) {
        forceToAssignOrphans();
    }
    prepareResult(znRecord);
    return znRecord;
}

Example 47 with ZNRecord

use of org.apache.helix.ZNRecord in project helix by apache.

the class MultiRoundCrushRebalanceStrategy method generateZNRecord.

private ZNRecord generateZNRecord(String resource, List<String> partitions, Map<String, Map<String, List<Node>>> partitionStateMapping) {
    Map<String, List<String>> newPreferences = new HashMap<String, List<String>>();
    for (int i = 0; i < partitions.size(); i++) {
        String partitionName = partitions.get(i);
        Map<String, List<Node>> stateNodeMap = partitionStateMapping.get(partitionName);
        for (String state : _stateCountMap.keySet()) {
            List<Node> nodes = stateNodeMap.get(state);
            List<String> nodeList = new ArrayList<String>();
            for (int j = 0; j < nodes.size(); j++) {
                nodeList.add(nodes.get(j).getName());
            }
            if (!newPreferences.containsKey(partitionName)) {
                newPreferences.put(partitionName, new ArrayList<String>());
            }
            newPreferences.get(partitionName).addAll(nodeList);
        }
    }
    ZNRecord result = new ZNRecord(resource);
    result.setListFields(newPreferences);
    return result;
}

Example 48 with ZNRecord

use of org.apache.helix.ZNRecord in project helix by apache.

the class Replicator method main.

public static void main(String[] args) throws Exception {
    InstanceConfig localInstanceConfig = new InstanceConfig("localhost_12001");
    ZNRecord record = localInstanceConfig.getRecord();
    record.setSimpleField("change_log_dir", "data/localhost_12001/translog");
    record.setSimpleField("file_store_dir", "data/localhost_12001/filestore");
    record.setSimpleField("check_point_dir", "data/localhost_12001/checkpoint");
    InstanceConfig masterInstanceConfig = new InstanceConfig("localhost_12001");
    record = masterInstanceConfig.getRecord();
    record.setSimpleField("change_log_dir", "data/localhost_12000/translog");
    record.setSimpleField("file_store_dir", "data/localhost_12000/filestore");
    record.setSimpleField("check_point_dir", "data/localhost_12000/checkpoint");
    Replicator replicator = new Replicator(localInstanceConfig, "resource", "partition");
    replicator.startReplication(masterInstanceConfig);
}

Example 49 with ZNRecord

use of org.apache.helix.ZNRecord in project helix by apache.

the class ServiceDiscovery method register.

public boolean register(final String serviceId, final ServiceMetadata serviceMetadata) throws Exception {
    HelixManager helixManager = HelixManagerFactory.getZKHelixManager(cluster, serviceId, InstanceType.PARTICIPANT, zkAddress);
    LiveInstanceInfoProvider liveInstanceInfoProvider = new LiveInstanceInfoProvider() {

        @Override
        public ZNRecord getAdditionalLiveInstanceInfo() {
            // serialize serviceMetadata to ZNRecord
            ZNRecord rec = new ZNRecord(serviceId);
            rec.setSimpleField("HOST", serviceMetadata.getHost());
            rec.setSimpleField("PORT", String.valueOf(serviceMetadata.getPort()));
            rec.setSimpleField("SERVICE_NAME", serviceMetadata.getServiceName());
            return rec;
        }
    };
    helixManager.setLiveInstanceInfoProvider(liveInstanceInfoProvider);
    helixManager.connect();
    serviceMap.put(serviceId, helixManager);
    refreshCache();
    return true;
}

Example 50 with ZNRecord

use of org.apache.helix.ZNRecord in project helix by apache.

the class ServiceDiscovery method refreshCache.

private void refreshCache(List<LiveInstance> liveInstances) {
    List<ServiceMetadata> services = new ArrayList<ServiceMetadata>();
    for (LiveInstance liveInstance : liveInstances) {
        ServiceMetadata metadata = new ServiceMetadata();
        ZNRecord rec = liveInstance.getRecord();
        metadata.setPort(Integer.parseInt(rec.getSimpleField("PORT")));
        metadata.setHost(rec.getSimpleField("HOST"));
        metadata.setServiceName(rec.getSimpleField("SERVICE_NAME"));
        services.add(metadata);
    }
    // protect against multiple threads updating this
    synchronized (this) {
        cache = services;
    }
}