Examples with JobInfo alluxio.job.wire.JobInfo used on opensource projects

Example 11 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class CompositeIntegrationTest method nestedTest.

@Test
public void nestedTest() throws Exception {
    CompositeConfig config = new CompositeConfig(Lists.newArrayList(new CompositeConfig(Lists.newArrayList(new SleepJobConfig(2), new SleepJobConfig(3)), false), new SleepJobConfig(1)), true);
    long jobId = mJobMaster.run(config);
    waitForJobToFinish(jobId);
    JobInfo status = mJobMaster.getStatus(jobId);
    assertEquals(Status.COMPLETED, status.getStatus());
    List<JobInfo> children = status.getChildren();
    assertEquals(2, children.size());
    JobInfo child0 = children.get(0);
    assertEquals(child0, mJobMaster.getStatus(children.get(0).getId()));
    assertEquals(Status.COMPLETED, child0.getStatus());
    assertEquals(2, child0.getChildren().size());
    JobInfo child1 = children.get(1);
    assertEquals(child1, mJobMaster.getStatus(children.get(1).getId()));
    assertEquals(Status.COMPLETED, child1.getStatus());
    assertEquals(1, child1.getChildren().size());
}

Example 12 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class CompositeIntegrationTest method waitParallel.

@Test
public void waitParallel() throws Exception {
    CompositeConfig config = new CompositeConfig(Lists.newArrayList(new SleepJobConfig(1000000), new SleepJobConfig(1)), false);
    long jobId = mJobMaster.run(config);
    JobInfo status = mJobMaster.getStatus(jobId);
    assertEquals(Status.RUNNING, status.getStatus());
    assertEquals(2, status.getChildren().size());
}

Example 13 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobMasterIntegrationTest method cancel.

@Test
public void cancel() throws Exception {
    SleepJobConfig childJob1 = new SleepJobConfig(50000);
    SleepJobConfig childJob2 = new SleepJobConfig(45000);
    SleepJobConfig childJob3 = new SleepJobConfig(40000);
    CompositeConfig jobConfig = new CompositeConfig(Lists.newArrayList(childJob1, childJob2, childJob3), false);
    long jobId = mJobMaster.run(jobConfig);
    JobInfo status = mJobMaster.getStatus(jobId);
    List<JobInfo> children = status.getChildren();
    assertEquals(3, children.size());
    long child0 = children.get(0).getId();
    long child1 = children.get(1).getId();
    long child2 = children.get(2).getId();
    mJobMaster.cancel(jobId);
    JobTestUtils.waitForJobStatus(mJobMaster, jobId, Status.CANCELED);
    JobTestUtils.waitForJobStatus(mJobMaster, child0, Status.CANCELED);
    JobTestUtils.waitForJobStatus(mJobMaster, child1, Status.CANCELED);
    JobTestUtils.waitForJobStatus(mJobMaster, child2, Status.CANCELED);
}

Example 14 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobMasterIntegrationTest method multipleTasksPerWorker.

@Test
public void multipleTasksPerWorker() throws Exception {
    long jobId = mJobMaster.run(new SleepJobConfig(1, 2));
    JobInfo jobStatus = mJobMaster.getStatus(jobId);
    assertEquals(2, jobStatus.getChildren().size());
    JobTestUtils.waitForJobStatus(mJobMaster, jobId, Status.COMPLETED);
    jobStatus = mJobMaster.getStatus(jobId);
    assertEquals(2, jobStatus.getChildren().size());
}

Example 15 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobGrpcClientUtils method run.

/**
 * Runs the specified job and waits for it to finish. If the job fails, it is retried the given
 * number of times. If the job does not complete in the given number of attempts, an exception
 * is thrown.
 *
 * @param config configuration for the job to run
 * @param attempts number of times to try running the job before giving up
 * @param alluxioConf Alluxio configuration
 * @return the job id of the job
 */
public static long run(JobConfig config, int attempts, AlluxioConfiguration alluxioConf) throws InterruptedException {
    CountingRetry retryPolicy = new CountingRetry(attempts);
    String errorMessage = "";
    while (retryPolicy.attempt()) {
        long jobId;
        try (JobMasterClient client = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(ClientContext.create(alluxioConf)).build())) {
            jobId = client.run(config);
        } catch (Exception e) {
            // job could not be started, retry
            LOG.warn("Exception encountered when starting a job.", e);
            continue;
        }
        JobInfo jobInfo = waitFor(jobId, alluxioConf);
        if (jobInfo == null) {
            // job status could not be fetched, give up
            break;
        }
        if (jobInfo.getStatus() == Status.COMPLETED || jobInfo.getStatus() == Status.CANCELED) {
            return jobInfo.getId();
        }
        errorMessage = jobInfo.getErrorMessage();
        LOG.warn("Job {} failed to complete with attempt {}. error: {}", jobId, retryPolicy.getAttemptCount(), errorMessage);
    }
    throw new RuntimeException("Failed to successfully complete the job: " + errorMessage);
}