Examples with PlanInfo alluxio.job.plan.meta.PlanInfo used on opensource projects

Example 1 with PlanInfo

use of alluxio.job.plan.meta.PlanInfo in project alluxio by Alluxio.

the class WorkflowTrackerTest method testBasic.

@Test
public void testBasic() throws Exception {
    ArrayList<JobConfig> jobs = Lists.newArrayList();
    TestPlanConfig child1 = new TestPlanConfig("1");
    TestPlanConfig child2 = new TestPlanConfig("2");
    jobs.add(child1);
    jobs.add(child2);
    CompositeConfig config = new CompositeConfig(jobs, true);
    mWorkflowTracker.run(config, 0);
    verify(mMockJobMaster).run(child1, 100);
    WorkflowInfo info = mWorkflowTracker.getStatus(0, true);
    assertEquals(Status.RUNNING, info.getStatus());
    verify(mMockJobMaster, never()).run(child2, 101);
    PlanInfo plan100 = new PlanInfo(100, child1, null);
    plan100.setStatus(Status.COMPLETED);
    mWorkflowTracker.onPlanStatusChange(plan100);
    verify(mMockJobMaster).run(child2, 101);
    assertEquals(Status.RUNNING, mWorkflowTracker.getStatus(0, true).getStatus());
    PlanInfo plan101 = new PlanInfo(101, child2, null);
    plan101.setStatus(Status.COMPLETED);
    mWorkflowTracker.onPlanStatusChange(plan101);
    assertEquals(Status.COMPLETED, mWorkflowTracker.getStatus(0, true).getStatus());
}

Example 2 with PlanInfo

use of alluxio.job.plan.meta.PlanInfo in project alluxio by Alluxio.

the class PlanTracker method removeFinished.

/**
 * Removes all finished jobs outside of the retention time from the queue.
 *
 * @return true if at least one job was removed, or if not at maximum capacity yet, false if at
 *         capacity and no job was removed
 */
private synchronized boolean removeFinished() {
    boolean removedJob = false;
    boolean isFull = mCoordinators.size() >= mCapacity;
    if (!isFull) {
        return true;
    }
    // Try to clear the queue
    if (mFinished.isEmpty()) {
        // The job master is at full capacity and no job has finished.
        return false;
    }
    ArrayList<Long> removedJobIds = Lists.newArrayList();
    int removeCount = 0;
    while (!mFinished.isEmpty() && removeCount < mMaxJobPurgeCount) {
        PlanInfo oldestJob = mFinished.peek();
        if (oldestJob == null) {
            // no items to remove
            break;
        }
        long timeSinceCompletion = CommonUtils.getCurrentMs() - oldestJob.getLastStatusChangeMs();
        // guaranteed that the job at the top of is the oldest.
        if (timeSinceCompletion < mRetentionMs) {
            break;
        }
        // first peek()
        if (mFinished.poll() == null) {
            // This should not happen because peek() returned an element
            // there should be no other concurrent operations that remove from mFinished
            LOG.warn("Polling the queue resulted in a null element");
            break;
        }
        long oldestJobId = oldestJob.getId();
        removedJobIds.add(oldestJobId);
        // Don't remove from the plan coordinator yet because WorkflowTracker may need these job info
        if (mCoordinators.get(oldestJobId) == null) {
            LOG.warn("Did not find a coordinator with id {}", oldestJobId);
        } else {
            removedJob = true;
            removeCount++;
        }
    }
    mWorkflowTracker.cleanup(removedJobIds);
    // Remove from the plan coordinator
    for (long removedJobId : removedJobIds) {
        mCoordinators.remove(removedJobId);
    }
    return removedJob;
}

Example 3 with PlanInfo

use of alluxio.job.plan.meta.PlanInfo in project alluxio by Alluxio.

the class PlanInfoTest method compare.

@Test
public void compare() {
    JobConfig jobConfig = new TestPlanConfig("unused");
    PlanInfo a = new PlanInfo(0L, jobConfig, null);
    CommonUtils.sleepMs(1);
    PlanInfo b = new PlanInfo(0L, jobConfig, null);
    Assert.assertEquals(-1, a.compareTo(b));
    b.setStatus(Status.RUNNING);
    CommonUtils.sleepMs(1);
    a.setStatus(Status.RUNNING);
    Assert.assertEquals(1, a.compareTo(b));
    a.setStatus(Status.COMPLETED);
    CommonUtils.sleepMs(1);
    b.setStatus(Status.COMPLETED);
    Assert.assertEquals(-1, a.compareTo(b));
}

Example 4 with PlanInfo

use of alluxio.job.plan.meta.PlanInfo in project alluxio by Alluxio.

the class PlanInfoTest method callback.

@Test
public void callback() {
    final String result = "I was here!";
    JobConfig jobConfig = new TestPlanConfig("unused");
    PlanInfo a = new PlanInfo(0L, jobConfig, jobInfo -> jobInfo.setResult(result));
    a.setStatus(Status.COMPLETED);
    Assert.assertEquals(result, a.getResult());
}