Examples with WorkflowContext org.apache.helix.task.WorkflowContext used on opensource projects

Example 11 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TestRecurringJobQueue method testCreateStoppedQueue.

@Test
public void testCreateStoppedQueue() throws InterruptedException {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue.Builder queueBuild = TaskTestUtil.buildRecurrentJobQueue(queueName, 0, 600000, TargetState.STOP);
    createAndEnqueueJob(queueBuild, 2);
    _driver.createQueue(queueBuild.build());
    WorkflowConfig workflowConfig = _driver.getWorkflowConfig(queueName);
    Assert.assertEquals(workflowConfig.getTargetState(), TargetState.STOP);
    _driver.resume(queueName);
    // TaskTestUtil.pollForWorkflowState(_driver, queueName, );
    WorkflowContext wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    // ensure current schedule is started
    String scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    _driver.pollForWorkflowState(scheduledQueue, TaskState.COMPLETED);
}

Example 12 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TestRetrieveWorkflows method testGetAllWorkflows.

@Test
public void testGetAllWorkflows() throws Exception {
    List<Workflow> workflowList = new ArrayList<Workflow>();
    for (int i = 0; i < 2; i++) {
        Workflow workflow = WorkflowGenerator.generateDefaultRepeatedJobWorkflowBuilder(TestHelper.getTestMethodName() + i).build();
        _driver.start(workflow);
        workflowList.add(workflow);
    }
    for (Workflow workflow : workflowList) {
        _driver.pollForWorkflowState(workflow.getName(), TaskState.COMPLETED);
    }
    Map<String, WorkflowConfig> workflowConfigMap = _driver.getWorkflows();
    Assert.assertEquals(workflowConfigMap.size(), workflowList.size());
    for (Map.Entry<String, WorkflowConfig> workflow : workflowConfigMap.entrySet()) {
        WorkflowConfig workflowConfig = workflow.getValue();
        WorkflowContext workflowContext = _driver.getWorkflowContext(workflow.getKey());
        Assert.assertNotNull(workflowContext);
        for (String job : workflowConfig.getJobDag().getAllNodes()) {
            JobConfig jobConfig = _driver.getJobConfig(job);
            JobContext jobContext = _driver.getJobContext(job);
            Assert.assertNotNull(jobConfig);
            Assert.assertNotNull(jobContext);
        }
    }
}

Example 13 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TestJobQueueCleanUp method testJobQueueAutoCleanUp.

@Test
public void testJobQueueAutoCleanUp() throws InterruptedException {
    int capacity = 10;
    String queueName = TestHelper.getTestMethodName();
    JobQueue.Builder builder = TaskTestUtil.buildJobQueue(queueName, capacity);
    WorkflowConfig.Builder cfgBuilder = new WorkflowConfig.Builder(builder.getWorkflowConfig());
    cfgBuilder.setJobPurgeInterval(1000);
    builder.setWorkflowConfig(cfgBuilder.build());
    JobConfig.Builder jobBuilder = new JobConfig.Builder().setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setCommand(MockTask.TASK_COMMAND).setMaxAttemptsPerTask(2).setJobCommandConfigMap(ImmutableMap.of(MockTask.SUCCESS_COUNT_BEFORE_FAIL, String.valueOf(capacity / 2))).setExpiry(200L);
    Set<String> deletedJobs = new HashSet<String>();
    Set<String> remainJobs = new HashSet<String>();
    for (int i = 0; i < capacity; i++) {
        builder.enqueueJob("JOB" + i, jobBuilder);
        if (i < capacity / 2) {
            deletedJobs.add("JOB" + i);
        } else {
            remainJobs.add(TaskUtil.getNamespacedJobName(queueName, "JOB" + i));
        }
    }
    _driver.start(builder.build());
    _driver.pollForJobState(queueName, TaskUtil.getNamespacedJobName(queueName, "JOB" + (capacity - 1)), TaskState.FAILED);
    Thread.sleep(2000);
    WorkflowConfig config = _driver.getWorkflowConfig(queueName);
    Assert.assertEquals(config.getJobDag().getAllNodes(), remainJobs);
    WorkflowContext context = _driver.getWorkflowContext(queueName);
    Assert.assertEquals(context.getJobStates().keySet(), remainJobs);
    Assert.assertTrue(remainJobs.containsAll(context.getJobStartTimes().keySet()));
    for (String job : deletedJobs) {
        JobConfig cfg = _driver.getJobConfig(job);
        JobContext ctx = _driver.getJobContext(job);
        Assert.assertNull(cfg);
        Assert.assertNull(ctx);
    }
}

Example 14 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project incubator-gobblin by apache.

the class GobblinHelixJobLauncherTest method testJobCleanup.

public void testJobCleanup() throws Exception {
    final ConcurrentHashMap<String, Boolean> runningMap = new ConcurrentHashMap<>();
    final Properties properties = generateJobProperties(this.baseConfig, "3", "_1504201348473");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher = new GobblinHelixJobLauncher(properties, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap);
    final Properties properties2 = generateJobProperties(this.baseConfig, "33", "_1504201348474");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher2 = new GobblinHelixJobLauncher(properties2, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap);
    gobblinHelixJobLauncher.launchJob(null);
    gobblinHelixJobLauncher2.launchJob(null);
    final TaskDriver taskDriver = new TaskDriver(this.helixManager);
    final String jobName = properties.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    final String jobIdKey = properties.getProperty(ConfigurationKeys.JOB_ID_KEY);
    final String jobContextName = jobName + "_" + jobIdKey;
    final String jobName2 = properties2.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    org.apache.helix.task.JobContext jobContext = taskDriver.getJobContext(jobContextName);
    // job context should be present until close
    Assert.assertNotNull(jobContext);
    gobblinHelixJobLauncher.close();
    // job queue deleted asynchronously after close
    waitForQueueCleanup(taskDriver, jobName);
    jobContext = taskDriver.getJobContext(jobContextName);
    // job context should have been deleted
    Assert.assertNull(jobContext);
    // job queue should have been deleted
    WorkflowConfig workflowConfig = taskDriver.getWorkflowConfig(jobName);
    Assert.assertNull(workflowConfig);
    WorkflowContext workflowContext = taskDriver.getWorkflowContext(jobName);
    Assert.assertNull(workflowContext);
    // second job queue with shared prefix should not be deleted when the first job queue is cleaned up
    workflowConfig = taskDriver.getWorkflowConfig(jobName2);
    Assert.assertNotNull(workflowConfig);
    gobblinHelixJobLauncher2.close();
    // job queue deleted asynchronously after close
    waitForQueueCleanup(taskDriver, jobName2);
    workflowConfig = taskDriver.getWorkflowConfig(jobName2);
    Assert.assertNull(workflowConfig);
    // check that workunit and taskstate directory for the job are cleaned up
    final File workunitsDir = new File(this.appWorkDir + File.separator + GobblinClusterConfigurationKeys.INPUT_WORK_UNIT_DIR_NAME + File.separator + jobIdKey);
    final File taskstatesDir = new File(this.appWorkDir + File.separator + GobblinClusterConfigurationKeys.OUTPUT_TASK_STATE_DIR_NAME + File.separator + jobIdKey);
    Assert.assertFalse(workunitsDir.exists());
    Assert.assertFalse(taskstatesDir.exists());
    // check that job.state file is cleaned up
    final File jobStateFile = new File(GobblinClusterUtils.getJobStateFilePath(true, this.appWorkDir, jobIdKey).toString());
    Assert.assertFalse(jobStateFile.exists());
}

Example 15 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project incubator-gobblin by apache.

the class GobblinHelixJobLauncher method waitForJobCompletion.

private void waitForJobCompletion() throws InterruptedException {
    LOGGER.info("Waiting for job to complete...");
    boolean timeoutEnabled = Boolean.parseBoolean(this.jobProps.getProperty(ConfigurationKeys.HELIX_JOB_TIMEOUT_ENABLED_KEY, ConfigurationKeys.DEFAULT_HELIX_JOB_TIMEOUT_ENABLED));
    long timeoutInSeconds = Long.parseLong(this.jobProps.getProperty(ConfigurationKeys.HELIX_JOB_TIMEOUT_SECONDS, ConfigurationKeys.DEFAULT_HELIX_JOB_TIMEOUT_SECONDS));
    long endTime = System.currentTimeMillis() + timeoutInSeconds * 1000;
    while (!timeoutEnabled || System.currentTimeMillis() <= endTime) {
        WorkflowContext workflowContext = TaskDriver.getWorkflowContext(this.helixManager, this.helixQueueName);
        if (workflowContext != null) {
            org.apache.helix.task.TaskState helixJobState = workflowContext.getJobState(this.jobResourceName);
            if (helixJobState == org.apache.helix.task.TaskState.COMPLETED || helixJobState == org.apache.helix.task.TaskState.FAILED || helixJobState == org.apache.helix.task.TaskState.STOPPED) {
                return;
            }
        }
        Thread.sleep(1000);
    }
    helixTaskDriverWaitToStop(this.helixQueueName, 10L);
    try {
        cancelJob(this.jobListener);
    } catch (JobException e) {
        throw new RuntimeException("Unable to cancel job " + jobContext.getJobName() + ": ", e);
    }
    this.helixTaskDriver.resume(this.helixQueueName);
    LOGGER.info("stopped the queue, deleted the job");
}