Examples with HelixProperty org.apache.helix.HelixProperty used on opensource projects

Example 6 with HelixProperty

use of org.apache.helix.HelixProperty in project helix by apache.

the class WorkflowRebalancer method scheduleSingleJob.

/**
 * Posts new job to cluster
 */
private void scheduleSingleJob(String jobResource, JobConfig jobConfig) {
    HelixAdmin admin = _manager.getClusterManagmentTool();
    IdealState jobIS = admin.getResourceIdealState(_manager.getClusterName(), jobResource);
    if (jobIS != null) {
        LOG.info("Job " + jobResource + " idealstate already exists!");
        return;
    }
    // Set up job resource based on partitions from target resource
    TaskUtil.createUserContent(_manager.getHelixPropertyStore(), jobResource, new ZNRecord(TaskUtil.USER_CONTENT_NODE));
    int numIndependentTasks = jobConfig.getTaskConfigMap().size();
    int numPartitions = numIndependentTasks;
    if (numPartitions == 0) {
        IdealState targetIs = admin.getResourceIdealState(_manager.getClusterName(), jobConfig.getTargetResource());
        if (targetIs == null) {
            LOG.warn("Target resource does not exist for job " + jobResource);
        // do not need to fail here, the job will be marked as failure immediately when job starts running.
        } else {
            numPartitions = targetIs.getPartitionSet().size();
        }
    }
    admin.addResource(_manager.getClusterName(), jobResource, numPartitions, TaskConstants.STATE_MODEL_NAME);
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    // Set the job configuration
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    HelixProperty resourceConfig = new HelixProperty(jobResource);
    resourceConfig.getRecord().getSimpleFields().putAll(jobConfig.getResourceConfigMap());
    Map<String, TaskConfig> taskConfigMap = jobConfig.getTaskConfigMap();
    if (taskConfigMap != null) {
        for (TaskConfig taskConfig : taskConfigMap.values()) {
            resourceConfig.getRecord().setMapField(taskConfig.getId(), taskConfig.getConfigMap());
        }
    }
    accessor.setProperty(keyBuilder.resourceConfig(jobResource), resourceConfig);
    // Push out new ideal state based on number of target partitions
    IdealStateBuilder builder = new CustomModeISBuilder(jobResource);
    builder.setRebalancerMode(IdealState.RebalanceMode.TASK);
    builder.setNumReplica(1);
    builder.setNumPartitions(numPartitions);
    builder.setStateModel(TaskConstants.STATE_MODEL_NAME);
    if (jobConfig.getInstanceGroupTag() != null) {
        builder.setNodeGroup(jobConfig.getInstanceGroupTag());
    }
    if (jobConfig.isDisableExternalView()) {
        builder.disableExternalView();
    }
    jobIS = builder.build();
    for (int i = 0; i < numPartitions; i++) {
        jobIS.getRecord().setListField(jobResource + "_" + i, new ArrayList<String>());
        jobIS.getRecord().setMapField(jobResource + "_" + i, new HashMap<String, String>());
    }
    jobIS.setRebalancerClassName(JobRebalancer.class.getName());
    admin.setResourceIdealState(_manager.getClusterName(), jobResource, jobIS);
}

Example 7 with HelixProperty

use of org.apache.helix.HelixProperty in project helix by apache.

the class WorkflowRebalancer method cloneWorkflow.

/**
 * Create a new workflow based on an existing one
 *
 * @param manager          connection to Helix
 * @param origWorkflowName the name of the existing workflow
 * @param newWorkflowName  the name of the new workflow
 * @param newStartTime     a provided start time that deviates from the desired start time
 * @return the cloned workflow, or null if there was a problem cloning the existing one
 */
public static Workflow cloneWorkflow(HelixManager manager, String origWorkflowName, String newWorkflowName, Date newStartTime) {
    // Read all resources, including the workflow and jobs of interest
    HelixDataAccessor accessor = manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Map<String, HelixProperty> resourceConfigMap = accessor.getChildValuesMap(keyBuilder.resourceConfigs());
    if (!resourceConfigMap.containsKey(origWorkflowName)) {
        LOG.error("No such workflow named " + origWorkflowName);
        return null;
    }
    if (resourceConfigMap.containsKey(newWorkflowName)) {
        LOG.error("Workflow with name " + newWorkflowName + " already exists!");
        return null;
    }
    // Create a new workflow with a new name
    Map<String, String> workflowConfigsMap = resourceConfigMap.get(origWorkflowName).getRecord().getSimpleFields();
    WorkflowConfig.Builder workflowConfigBlder = WorkflowConfig.Builder.fromMap(workflowConfigsMap);
    // Set the schedule, if applicable
    if (newStartTime != null) {
        ScheduleConfig scheduleConfig = ScheduleConfig.oneTimeDelayedStart(newStartTime);
        workflowConfigBlder.setScheduleConfig(scheduleConfig);
    }
    workflowConfigBlder.setTerminable(true);
    WorkflowConfig workflowConfig = workflowConfigBlder.build();
    JobDag jobDag = workflowConfig.getJobDag();
    Map<String, Set<String>> parentsToChildren = jobDag.getParentsToChildren();
    Workflow.Builder workflowBuilder = new Workflow.Builder(newWorkflowName);
    workflowBuilder.setWorkflowConfig(workflowConfig);
    // Add each job back as long as the original exists
    Set<String> namespacedJobs = jobDag.getAllNodes();
    for (String namespacedJob : namespacedJobs) {
        if (resourceConfigMap.containsKey(namespacedJob)) {
            // Copy over job-level and task-level configs
            String job = TaskUtil.getDenamespacedJobName(origWorkflowName, namespacedJob);
            HelixProperty jobConfig = resourceConfigMap.get(namespacedJob);
            Map<String, String> jobSimpleFields = jobConfig.getRecord().getSimpleFields();
            JobConfig.Builder jobCfgBuilder = JobConfig.Builder.fromMap(jobSimpleFields);
            // overwrite workflow name
            jobCfgBuilder.setWorkflow(newWorkflowName);
            Map<String, Map<String, String>> rawTaskConfigMap = jobConfig.getRecord().getMapFields();
            List<TaskConfig> taskConfigs = Lists.newLinkedList();
            for (Map<String, String> rawTaskConfig : rawTaskConfigMap.values()) {
                TaskConfig taskConfig = TaskConfig.Builder.from(rawTaskConfig);
                taskConfigs.add(taskConfig);
            }
            jobCfgBuilder.addTaskConfigs(taskConfigs);
            workflowBuilder.addJob(job, jobCfgBuilder);
            // Add dag dependencies
            Set<String> children = parentsToChildren.get(namespacedJob);
            if (children != null) {
                for (String namespacedChild : children) {
                    String child = TaskUtil.getDenamespacedJobName(origWorkflowName, namespacedChild);
                    workflowBuilder.addParentChildDependency(job, child);
                }
            }
        }
    }
    return workflowBuilder.build();
}

Example 8 with HelixProperty

use of org.apache.helix.HelixProperty in project helix by apache.

the class ZKHelixDataAccessor method setChildren.

@Override
public <T extends HelixProperty> boolean[] setChildren(List<PropertyKey> keys, List<T> children) {
    int options = -1;
    List<String> paths = new ArrayList<String>();
    List<ZNRecord> records = new ArrayList<ZNRecord>();
    List<List<String>> bucketizedPaths = new ArrayList<List<String>>(Collections.<List<String>>nCopies(keys.size(), null));
    List<List<ZNRecord>> bucketizedRecords = new ArrayList<List<ZNRecord>>(Collections.<List<ZNRecord>>nCopies(keys.size(), null));
    for (int i = 0; i < keys.size(); i++) {
        PropertyKey key = keys.get(i);
        PropertyType type = key.getType();
        String path = key.getPath();
        paths.add(path);
        options = constructOptions(type);
        HelixProperty value = children.get(i);
        switch(type) {
            case EXTERNALVIEW:
                if (value.getBucketSize() == 0) {
                    records.add(value.getRecord());
                } else {
                    ZNRecord metaRecord = new ZNRecord(value.getId());
                    metaRecord.setSimpleFields(value.getRecord().getSimpleFields());
                    records.add(metaRecord);
                    ZNRecordBucketizer bucketizer = new ZNRecordBucketizer(value.getBucketSize());
                    Map<String, ZNRecord> map = bucketizer.bucketize(value.getRecord());
                    List<String> childBucketizedPaths = new ArrayList<String>();
                    List<ZNRecord> childBucketizedRecords = new ArrayList<ZNRecord>();
                    for (String bucketName : map.keySet()) {
                        childBucketizedPaths.add(path + "/" + bucketName);
                        childBucketizedRecords.add(map.get(bucketName));
                    }
                    bucketizedPaths.set(i, childBucketizedPaths);
                    bucketizedRecords.set(i, childBucketizedRecords);
                }
                break;
            case STATEMODELDEFS:
                if (value.isValid()) {
                    records.add(value.getRecord());
                }
                break;
            default:
                records.add(value.getRecord());
                break;
        }
    }
    // set non-bucketized nodes or parent nodes of bucketized nodes
    boolean[] success = _baseDataAccessor.setChildren(paths, records, options);
    // set bucketized nodes
    List<String> allBucketizedPaths = new ArrayList<String>();
    List<ZNRecord> allBucketizedRecords = new ArrayList<ZNRecord>();
    for (int i = 0; i < keys.size(); i++) {
        if (success[i] && bucketizedPaths.get(i) != null) {
            allBucketizedPaths.addAll(bucketizedPaths.get(i));
            allBucketizedRecords.addAll(bucketizedRecords.get(i));
        }
    }
    // TODO: set success accordingly
    _baseDataAccessor.setChildren(allBucketizedPaths, allBucketizedRecords, options);
    return success;
}

Example 9 with HelixProperty

use of org.apache.helix.HelixProperty in project helix by apache.

the class ZKHelixDataAccessor method getProperty.

@Override
public <T extends HelixProperty> List<T> getProperty(List<PropertyKey> keys, boolean throwException) throws HelixMetaDataAccessException {
    if (keys == null || keys.size() == 0) {
        return Collections.emptyList();
    }
    List<T> childValues = new ArrayList<T>();
    // read all records
    List<String> paths = new ArrayList<>();
    List<Stat> stats = new ArrayList<>();
    for (PropertyKey key : keys) {
        paths.add(key.getPath());
        stats.add(new Stat());
    }
    List<ZNRecord> children = _baseDataAccessor.get(paths, stats, 0, throwException);
    // check if bucketized
    for (int i = 0; i < keys.size(); i++) {
        PropertyKey key = keys.get(i);
        ZNRecord record = children.get(i);
        Stat stat = stats.get(i);
        PropertyType type = key.getType();
        String path = key.getPath();
        int options = constructOptions(type);
        if (record != null) {
            record.setCreationTime(stat.getCtime());
            record.setModifiedTime(stat.getMtime());
            record.setVersion(stat.getVersion());
        }
        switch(type) {
            case CURRENTSTATES:
            case IDEALSTATES:
            case EXTERNALVIEW:
                // check if bucketized
                if (record != null) {
                    HelixProperty property = new HelixProperty(record);
                    int bucketSize = property.getBucketSize();
                    if (bucketSize > 0) {
                        // @see HELIX-574
                        // clean up list and map fields in case we write to parent node by mistake
                        property.getRecord().getMapFields().clear();
                        property.getRecord().getListFields().clear();
                        List<ZNRecord> childRecords = _baseDataAccessor.getChildren(path, null, options, 1, 0);
                        ZNRecord assembledRecord = new ZNRecordAssembler().assemble(childRecords);
                        // merge with parent node value
                        if (assembledRecord != null) {
                            record.getSimpleFields().putAll(assembledRecord.getSimpleFields());
                            record.getListFields().putAll(assembledRecord.getListFields());
                            record.getMapFields().putAll(assembledRecord.getMapFields());
                        }
                    }
                }
                break;
            default:
                break;
        }
        @SuppressWarnings("unchecked") T t = (T) HelixProperty.convertToTypedInstance(key.getTypeClass(), record);
        childValues.add(t);
    }
    return childValues;
}

Example 10 with HelixProperty

use of org.apache.helix.HelixProperty in project helix by apache.

the class CriteriaEvaluator method getProperty.

private List<HelixProperty> getProperty(HelixDataAccessor accessor, String dataSpec, PropertyKey propertyKeys, PropertyKey propertyKey, String dataType) {
    List<HelixProperty> properties;
    if (Strings.isNullOrEmpty(dataSpec) || dataSpec.equals(MATCH_ALL_SYM)) {
        properties = accessor.getChildValues(propertyKeys);
    } else {
        HelixProperty data = accessor.getProperty(propertyKey);
        if (data == null) {
            throw new HelixException(String.format("Specified %s %s is not found!", dataType, dataSpec));
        }
        properties = Collections.singletonList(data);
    }
    return properties;
}