Examples with PropertyKey org.apache.helix.PropertyKey used on opensource projects

Example 31 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class ZKHelixDataAccessor method setChildren.

@Override
public <T extends HelixProperty> boolean[] setChildren(List<PropertyKey> keys, List<T> children) {
    int options = -1;
    List<String> paths = new ArrayList<String>();
    List<ZNRecord> records = new ArrayList<ZNRecord>();
    List<List<String>> bucketizedPaths = new ArrayList<List<String>>(Collections.<List<String>>nCopies(keys.size(), null));
    List<List<ZNRecord>> bucketizedRecords = new ArrayList<List<ZNRecord>>(Collections.<List<ZNRecord>>nCopies(keys.size(), null));
    for (int i = 0; i < keys.size(); i++) {
        PropertyKey key = keys.get(i);
        PropertyType type = key.getType();
        String path = key.getPath();
        paths.add(path);
        options = constructOptions(type);
        HelixProperty value = children.get(i);
        switch(type) {
            case EXTERNALVIEW:
                if (value.getBucketSize() == 0) {
                    records.add(value.getRecord());
                } else {
                    ZNRecord metaRecord = new ZNRecord(value.getId());
                    metaRecord.setSimpleFields(value.getRecord().getSimpleFields());
                    records.add(metaRecord);
                    ZNRecordBucketizer bucketizer = new ZNRecordBucketizer(value.getBucketSize());
                    Map<String, ZNRecord> map = bucketizer.bucketize(value.getRecord());
                    List<String> childBucketizedPaths = new ArrayList<String>();
                    List<ZNRecord> childBucketizedRecords = new ArrayList<ZNRecord>();
                    for (String bucketName : map.keySet()) {
                        childBucketizedPaths.add(path + "/" + bucketName);
                        childBucketizedRecords.add(map.get(bucketName));
                    }
                    bucketizedPaths.set(i, childBucketizedPaths);
                    bucketizedRecords.set(i, childBucketizedRecords);
                }
                break;
            case STATEMODELDEFS:
                if (value.isValid()) {
                    records.add(value.getRecord());
                }
                break;
            default:
                records.add(value.getRecord());
                break;
        }
    }
    // set non-bucketized nodes or parent nodes of bucketized nodes
    boolean[] success = _baseDataAccessor.setChildren(paths, records, options);
    // set bucketized nodes
    List<String> allBucketizedPaths = new ArrayList<String>();
    List<ZNRecord> allBucketizedRecords = new ArrayList<ZNRecord>();
    for (int i = 0; i < keys.size(); i++) {
        if (success[i] && bucketizedPaths.get(i) != null) {
            allBucketizedPaths.addAll(bucketizedPaths.get(i));
            allBucketizedRecords.addAll(bucketizedRecords.get(i));
        }
    }
    // TODO: set success accordingly
    _baseDataAccessor.setChildren(allBucketizedPaths, allBucketizedRecords, options);
    return success;
}

Example 32 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class ExternalViewComputeStage method execute.

@Override
public void execute(final ClusterEvent event) throws Exception {
    _eventId = event.getEventId();
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    ResourceControllerDataProvider cache = event.getAttribute(AttributeName.ControllerDataProvider.name());
    if (manager == null || resourceMap == null || cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires ClusterManager|RESOURCES|DataCache");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
    CurrentStateOutput currentStateOutput = event.getAttribute(AttributeName.CURRENT_STATE.name());
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    List<ExternalView> newExtViews = new ArrayList<>();
    Set<String> monitoringResources = new HashSet<>();
    Map<String, ExternalView> curExtViews = cache.getExternalViews();
    for (Resource resource : resourceMap.values()) {
        try {
            computeExternalView(resource, currentStateOutput, cache, clusterStatusMonitor, curExtViews, manager, monitoringResources, newExtViews);
        } catch (HelixException ex) {
            LogUtil.logError(LOG, _eventId, "Failed to calculate external view for resource " + resource.getResourceName(), ex);
        }
    }
    // Keep MBeans for existing resources and unregister MBeans for dropped resources
    if (clusterStatusMonitor != null) {
        clusterStatusMonitor.retainResourceMonitor(monitoringResources);
    }
    List<String> externalViewsToRemove = new ArrayList<>();
    // TODO: consider not setting the externalview of SCHEDULER_TASK_QUEUE at all.
    // Are there any entity that will be interested in its change?
    // For the resource with DisableExternalView option turned on in IdealState
    // We will not actually create or write the externalView to ZooKeeper.
    List<PropertyKey> keys = new ArrayList<>();
    for (Iterator<ExternalView> it = newExtViews.iterator(); it.hasNext(); ) {
        ExternalView view = it.next();
        String resourceName = view.getResourceName();
        IdealState idealState = cache.getIdealState(resourceName);
        if (idealState != null && idealState.isExternalViewDisabled()) {
            it.remove();
            // remove the external view if the external view exists
            if (curExtViews.containsKey(resourceName)) {
                LogUtil.logInfo(LOG, _eventId, "Remove externalView for resource: " + resourceName);
                dataAccessor.removeProperty(keyBuilder.externalView(resourceName));
                externalViewsToRemove.add(resourceName);
            }
        } else {
            keys.add(keyBuilder.externalView(resourceName));
        }
    }
    // add/update external-views
    if (newExtViews.size() > 0) {
        dataAccessor.setChildren(keys, newExtViews);
        cache.updateExternalViews(newExtViews);
    }
    // remove dead external-views
    for (String resourceName : curExtViews.keySet()) {
        if (!resourceMap.keySet().contains(resourceName)) {
            LogUtil.logInfo(LOG, _eventId, "Remove externalView for resource: " + resourceName);
            dataAccessor.removeProperty(keyBuilder.externalView(resourceName));
            externalViewsToRemove.add(resourceName);
        }
    }
    cache.removeExternalViews(externalViewsToRemove);
}

Example 33 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class MessageDispatchStage method sendMessages.

// return the messages actually sent
protected List<Message> sendMessages(HelixDataAccessor dataAccessor, List<Message> messages) {
    List<Message> messageSent = new ArrayList<>();
    if (messages == null || messages.isEmpty()) {
        return messageSent;
    }
    Builder keyBuilder = dataAccessor.keyBuilder();
    List<PropertyKey> keys = new ArrayList<PropertyKey>();
    for (Message message : messages) {
        LogUtil.logInfo(logger, _eventId, "Sending Message " + message.getMsgId() + " to " + message.getTgtName() + " transit " + message.getResourceName() + "." + message.getPartitionName() + "|" + message.getPartitionNames() + " from:" + message.getFromState() + " to:" + message.getToState() + ", relayMessages: " + message.getRelayMessages().size());
        if (message.hasRelayMessages()) {
            for (Message msg : message.getRelayMessages().values()) {
                LogUtil.logInfo(logger, _eventId, "Sending Relay Message " + msg.getMsgId() + " to " + msg.getTgtName() + " transit " + msg.getResourceName() + "." + msg.getPartitionName() + "|" + msg.getPartitionNames() + " from:" + msg.getFromState() + " to:" + msg.getToState() + ", relayFrom: " + msg.getRelaySrcHost() + ", attached to message: " + message.getMsgId());
            }
        }
        keys.add(keyBuilder.message(message.getTgtName(), message.getId()));
    }
    boolean[] results = dataAccessor.createChildren(keys, new ArrayList<>(messages));
    for (int i = 0; i < results.length; i++) {
        if (!results[i]) {
            LogUtil.logError(logger, _eventId, "Failed to send message: " + keys.get(i));
        } else {
            messageSent.add(messages.get(i));
        }
    }
    return messageSent;
}

Example 34 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class TargetExteralViewCalcStage method execute.

@Override
public void execute(final ClusterEvent event) throws Exception {
    ResourceControllerDataProvider cache = event.getAttribute(AttributeName.ControllerDataProvider.name());
    ClusterConfig clusterConfig = cache.getClusterConfig();
    if (!clusterConfig.isTargetExternalViewEnabled()) {
        return;
    }
    HelixManager helixManager = event.getAttribute(AttributeName.helixmanager.name());
    HelixDataAccessor accessor = helixManager.getHelixDataAccessor();
    BestPossibleStateOutput bestPossibleAssignments = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    IntermediateStateOutput intermediateAssignments = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (!accessor.getBaseDataAccessor().exists(accessor.keyBuilder().targetExternalViews().getPath(), AccessOption.PERSISTENT)) {
        accessor.getBaseDataAccessor().create(accessor.keyBuilder().targetExternalViews().getPath(), null, AccessOption.PERSISTENT);
    }
    List<PropertyKey> keys = new ArrayList<>();
    List<ExternalView> targetExternalViews = new ArrayList<>();
    for (String resourceName : bestPossibleAssignments.resourceSet()) {
        if (cache.getIdealState(resourceName) == null || cache.getIdealState(resourceName).isExternalViewDisabled()) {
            continue;
        }
        Resource resource = resourceMap.get(resourceName);
        if (resource != null) {
            PartitionStateMap partitionStateMap = intermediateAssignments.getPartitionStateMap(resourceName);
            Map<String, Map<String, String>> intermediateAssignment = convertToMapFields(partitionStateMap.getStateMap());
            Map<String, List<String>> preferenceLists = bestPossibleAssignments.getPreferenceLists(resourceName);
            boolean needPersist = false;
            ExternalView targetExternalView = cache.getTargetExternalView(resourceName);
            if (targetExternalView == null) {
                targetExternalView = new ExternalView(resourceName);
                targetExternalView.getRecord().getSimpleFields().putAll(cache.getIdealState(resourceName).getRecord().getSimpleFields());
                needPersist = true;
            }
            if (preferenceLists != null && !targetExternalView.getRecord().getListFields().equals(preferenceLists)) {
                targetExternalView.getRecord().setListFields(preferenceLists);
                needPersist = true;
            }
            if (intermediateAssignment != null && !targetExternalView.getRecord().getMapFields().equals(intermediateAssignment)) {
                targetExternalView.getRecord().setMapFields(intermediateAssignment);
                needPersist = true;
            }
            if (needPersist) {
                keys.add(accessor.keyBuilder().targetExternalView(resourceName));
                targetExternalViews.add(targetExternalView);
                cache.updateTargetExternalView(resourceName, targetExternalView);
            }
        }
    }
    // TODO (HELIX-964): remove TEV when idealstate is removed
    accessor.setChildren(keys, targetExternalViews);
}

Example 35 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class CustomizedStateProvider method deletePerPartitionCustomizedState.

/**
 * Delete the customized state for a specified resource and a specified partition
 */
public void deletePerPartitionCustomizedState(String customizedStateName, String resourceName, String partitionName) {
    HelixDataAccessor accessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    PropertyKey propertyKey = keyBuilder.customizedState(_instanceName, customizedStateName, resourceName);
    CustomizedState existingState = getCustomizedState(customizedStateName, resourceName);
    ZNRecord rec = new ZNRecord(existingState.getId());
    rec.getMapFields().put(partitionName, null);
    ZNRecordDelta delta = new ZNRecordDelta(rec, ZNRecordDelta.MergeOperation.SUBTRACT);
    List<ZNRecordDelta> deltaList = new ArrayList<ZNRecordDelta>();
    deltaList.add(delta);
    existingState.setDeltaList(deltaList);
    if (!accessor.updateProperty(propertyKey, existingState)) {
        throw new HelixException(String.format("Failed to delete customized state %s to zk for instance %s, resource %s, " + "partition %s", customizedStateName, _instanceName, resourceName, partitionName));
    }
}