Examples with HelixDataAccessor org.apache.helix.HelixDataAccessor used on opensource projects

Example 41 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestTaskRebalancerFailover method test.

@Test
public void test() throws Exception {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue queue = new JobQueue.Builder(queueName).build();
    _driver.createQueue(queue);
    // Enqueue jobs
    Set<String> master = Sets.newHashSet("MASTER");
    JobConfig.Builder job = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(master);
    String job1Name = "masterJob";
    LOG.info("Enqueuing job: " + job1Name);
    _driver.enqueueJob(queueName, job1Name, job);
    // check all tasks completed on MASTER
    String namespacedJob1 = String.format("%s_%s", queueName, job1Name);
    _driver.pollForJobState(queueName, namespacedJob1, TaskState.COMPLETED);
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    ExternalView ev = accessor.getProperty(keyBuilder.externalView(WorkflowGenerator.DEFAULT_TGT_DB));
    JobContext ctx = _driver.getJobContext(namespacedJob1);
    Set<String> failOverPartitions = Sets.newHashSet();
    for (int p = 0; p < _numParitions; p++) {
        String instanceName = ctx.getAssignedParticipant(p);
        Assert.assertNotNull(instanceName);
        String partitionName = ctx.getTargetForPartition(p);
        Assert.assertNotNull(partitionName);
        String state = ev.getStateMap(partitionName).get(instanceName);
        Assert.assertNotNull(state);
        Assert.assertEquals(state, "MASTER");
        if (instanceName.equals("localhost_12918")) {
            failOverPartitions.add(partitionName);
        }
    }
    // enqueue another master job and fail localhost_12918
    String job2Name = "masterJob2";
    String namespacedJob2 = String.format("%s_%s", queueName, job2Name);
    LOG.info("Enqueuing job: " + job2Name);
    _driver.enqueueJob(queueName, job2Name, job);
    _driver.pollForJobState(queueName, namespacedJob2, TaskState.IN_PROGRESS);
    _participants[0].syncStop();
    _driver.pollForJobState(queueName, namespacedJob2, TaskState.COMPLETED);
    // tasks previously assigned to localhost_12918 should be re-scheduled on new master
    ctx = _driver.getJobContext(namespacedJob2);
    ev = accessor.getProperty(keyBuilder.externalView(WorkflowGenerator.DEFAULT_TGT_DB));
    for (int p = 0; p < _numParitions; p++) {
        String partitionName = ctx.getTargetForPartition(p);
        Assert.assertNotNull(partitionName);
        if (failOverPartitions.contains(partitionName)) {
            String instanceName = ctx.getAssignedParticipant(p);
            Assert.assertNotNull(instanceName);
            Assert.assertNotSame(instanceName, "localhost_12918");
            String state = ev.getStateMap(partitionName).get(instanceName);
            Assert.assertNotNull(state);
            Assert.assertEquals(state, "MASTER");
        }
    }
    // Flush queue and check cleanup
    _driver.flushQueue(queueName);
    Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(namespacedJob1)));
    Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(namespacedJob1)));
    Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(namespacedJob2)));
    Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(namespacedJob2)));
    WorkflowConfig workflowCfg = _driver.getWorkflowConfig(queueName);
    JobDag dag = workflowCfg.getJobDag();
    Assert.assertFalse(dag.getAllNodes().contains(namespacedJob1));
    Assert.assertFalse(dag.getAllNodes().contains(namespacedJob2));
    Assert.assertFalse(dag.getChildrenToParents().containsKey(namespacedJob1));
    Assert.assertFalse(dag.getChildrenToParents().containsKey(namespacedJob2));
    Assert.assertFalse(dag.getParentsToChildren().containsKey(namespacedJob1));
    Assert.assertFalse(dag.getParentsToChildren().containsKey(namespacedJob2));
}

Example 42 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestTaskRebalancerStopResume method stopAndDeleteQueue.

@Test
public void stopAndDeleteQueue() throws Exception {
    final String queueName = TestHelper.getTestMethodName();
    // Create a queue
    System.out.println("START " + queueName + " at " + new Date(System.currentTimeMillis()));
    WorkflowConfig wfCfg = new WorkflowConfig.Builder(queueName).setExpiry(2, TimeUnit.MINUTES).build();
    JobQueue qCfg = new JobQueue.Builder(queueName).fromMap(wfCfg.getResourceConfigMap()).build();
    _driver.createQueue(qCfg);
    // Enqueue 2 jobs
    Set<String> master = Sets.newHashSet("MASTER");
    JobConfig.Builder job1 = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(master);
    String job1Name = "masterJob";
    LOG.info("Enqueuing job1: " + job1Name);
    _driver.enqueueJob(queueName, job1Name, job1);
    Set<String> slave = Sets.newHashSet("SLAVE");
    JobConfig.Builder job2 = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(slave);
    String job2Name = "slaveJob";
    LOG.info("Enqueuing job2: " + job2Name);
    _driver.enqueueJob(queueName, job2Name, job2);
    String namespacedJob1 = String.format("%s_%s", queueName, job1Name);
    _driver.pollForJobState(queueName, namespacedJob1, TaskState.COMPLETED);
    String namespacedJob2 = String.format("%s_%s", queueName, job2Name);
    _driver.pollForJobState(queueName, namespacedJob2, TaskState.COMPLETED);
    // Stop queue
    _driver.stop(queueName);
    boolean result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
    Assert.assertTrue(result);
    // Delete queue
    _driver.delete(queueName);
    // Wait until all status are cleaned up
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            HelixDataAccessor accessor = _manager.getHelixDataAccessor();
            PropertyKey.Builder keyBuilder = accessor.keyBuilder();
            // check paths for resource-config, ideal-state, external-view, property-store
            List<String> paths = Lists.newArrayList(keyBuilder.resourceConfigs().getPath(), keyBuilder.idealStates().getPath(), keyBuilder.externalViews().getPath(), PropertyPathBuilder.propertyStore(CLUSTER_NAME) + TaskConstants.REBALANCER_CONTEXT_ROOT);
            for (String path : paths) {
                List<String> childNames = accessor.getBaseDataAccessor().getChildNames(path, 0);
                for (String childName : childNames) {
                    if (childName.startsWith(queueName)) {
                        return false;
                    }
                }
            }
            return true;
        }
    }, 30 * 1000);
    Assert.assertTrue(result);
    System.out.println("END " + queueName + " at " + new Date(System.currentTimeMillis()));
}

Example 43 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class TestTaskRebalancerStopResume method verifyJobDeleted.

private void verifyJobDeleted(String queueName, String jobName) throws Exception {
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(jobName)), jobName + "'s idealstate has not been deleted!");
    Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(jobName)), jobName + "'s resourceConfig has not been deleted!");
    TaskTestUtil.pollForEmptyJobState(_driver, queueName, jobName);
}

Example 44 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class WorkflowRebalancer method scheduleSingleJob.

/**
 * Posts new job to cluster
 */
private void scheduleSingleJob(String jobResource, JobConfig jobConfig) {
    HelixAdmin admin = _manager.getClusterManagmentTool();
    IdealState jobIS = admin.getResourceIdealState(_manager.getClusterName(), jobResource);
    if (jobIS != null) {
        LOG.info("Job " + jobResource + " idealstate already exists!");
        return;
    }
    // Set up job resource based on partitions from target resource
    TaskUtil.createUserContent(_manager.getHelixPropertyStore(), jobResource, new ZNRecord(TaskUtil.USER_CONTENT_NODE));
    int numIndependentTasks = jobConfig.getTaskConfigMap().size();
    int numPartitions = numIndependentTasks;
    if (numPartitions == 0) {
        IdealState targetIs = admin.getResourceIdealState(_manager.getClusterName(), jobConfig.getTargetResource());
        if (targetIs == null) {
            LOG.warn("Target resource does not exist for job " + jobResource);
        // do not need to fail here, the job will be marked as failure immediately when job starts running.
        } else {
            numPartitions = targetIs.getPartitionSet().size();
        }
    }
    admin.addResource(_manager.getClusterName(), jobResource, numPartitions, TaskConstants.STATE_MODEL_NAME);
    HelixDataAccessor accessor = _manager.getHelixDataAccessor();
    // Set the job configuration
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    HelixProperty resourceConfig = new HelixProperty(jobResource);
    resourceConfig.getRecord().getSimpleFields().putAll(jobConfig.getResourceConfigMap());
    Map<String, TaskConfig> taskConfigMap = jobConfig.getTaskConfigMap();
    if (taskConfigMap != null) {
        for (TaskConfig taskConfig : taskConfigMap.values()) {
            resourceConfig.getRecord().setMapField(taskConfig.getId(), taskConfig.getConfigMap());
        }
    }
    accessor.setProperty(keyBuilder.resourceConfig(jobResource), resourceConfig);
    // Push out new ideal state based on number of target partitions
    IdealStateBuilder builder = new CustomModeISBuilder(jobResource);
    builder.setRebalancerMode(IdealState.RebalanceMode.TASK);
    builder.setNumReplica(1);
    builder.setNumPartitions(numPartitions);
    builder.setStateModel(TaskConstants.STATE_MODEL_NAME);
    if (jobConfig.getInstanceGroupTag() != null) {
        builder.setNodeGroup(jobConfig.getInstanceGroupTag());
    }
    if (jobConfig.isDisableExternalView()) {
        builder.disableExternalView();
    }
    jobIS = builder.build();
    for (int i = 0; i < numPartitions; i++) {
        jobIS.getRecord().setListField(jobResource + "_" + i, new ArrayList<String>());
        jobIS.getRecord().setMapField(jobResource + "_" + i, new HashMap<String, String>());
    }
    jobIS.setRebalancerClassName(JobRebalancer.class.getName());
    admin.setResourceIdealState(_manager.getClusterName(), jobResource, jobIS);
}

Example 45 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class WorkflowRebalancer method cloneWorkflow.

/**
 * Create a new workflow based on an existing one
 *
 * @param manager          connection to Helix
 * @param origWorkflowName the name of the existing workflow
 * @param newWorkflowName  the name of the new workflow
 * @param newStartTime     a provided start time that deviates from the desired start time
 * @return the cloned workflow, or null if there was a problem cloning the existing one
 */
public static Workflow cloneWorkflow(HelixManager manager, String origWorkflowName, String newWorkflowName, Date newStartTime) {
    // Read all resources, including the workflow and jobs of interest
    HelixDataAccessor accessor = manager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Map<String, HelixProperty> resourceConfigMap = accessor.getChildValuesMap(keyBuilder.resourceConfigs());
    if (!resourceConfigMap.containsKey(origWorkflowName)) {
        LOG.error("No such workflow named " + origWorkflowName);
        return null;
    }
    if (resourceConfigMap.containsKey(newWorkflowName)) {
        LOG.error("Workflow with name " + newWorkflowName + " already exists!");
        return null;
    }
    // Create a new workflow with a new name
    Map<String, String> workflowConfigsMap = resourceConfigMap.get(origWorkflowName).getRecord().getSimpleFields();
    WorkflowConfig.Builder workflowConfigBlder = WorkflowConfig.Builder.fromMap(workflowConfigsMap);
    // Set the schedule, if applicable
    if (newStartTime != null) {
        ScheduleConfig scheduleConfig = ScheduleConfig.oneTimeDelayedStart(newStartTime);
        workflowConfigBlder.setScheduleConfig(scheduleConfig);
    }
    workflowConfigBlder.setTerminable(true);
    WorkflowConfig workflowConfig = workflowConfigBlder.build();
    JobDag jobDag = workflowConfig.getJobDag();
    Map<String, Set<String>> parentsToChildren = jobDag.getParentsToChildren();
    Workflow.Builder workflowBuilder = new Workflow.Builder(newWorkflowName);
    workflowBuilder.setWorkflowConfig(workflowConfig);
    // Add each job back as long as the original exists
    Set<String> namespacedJobs = jobDag.getAllNodes();
    for (String namespacedJob : namespacedJobs) {
        if (resourceConfigMap.containsKey(namespacedJob)) {
            // Copy over job-level and task-level configs
            String job = TaskUtil.getDenamespacedJobName(origWorkflowName, namespacedJob);
            HelixProperty jobConfig = resourceConfigMap.get(namespacedJob);
            Map<String, String> jobSimpleFields = jobConfig.getRecord().getSimpleFields();
            JobConfig.Builder jobCfgBuilder = JobConfig.Builder.fromMap(jobSimpleFields);
            // overwrite workflow name
            jobCfgBuilder.setWorkflow(newWorkflowName);
            Map<String, Map<String, String>> rawTaskConfigMap = jobConfig.getRecord().getMapFields();
            List<TaskConfig> taskConfigs = Lists.newLinkedList();
            for (Map<String, String> rawTaskConfig : rawTaskConfigMap.values()) {
                TaskConfig taskConfig = TaskConfig.Builder.from(rawTaskConfig);
                taskConfigs.add(taskConfig);
            }
            jobCfgBuilder.addTaskConfigs(taskConfigs);
            workflowBuilder.addJob(job, jobCfgBuilder);
            // Add dag dependencies
            Set<String> children = parentsToChildren.get(namespacedJob);
            if (children != null) {
                for (String namespacedChild : children) {
                    String child = TaskUtil.getDenamespacedJobName(origWorkflowName, namespacedChild);
                    workflowBuilder.addParentChildDependency(job, child);
                }
            }
        }
    }
    return workflowBuilder.build();
}