Examples with JobInfo alluxio.job.wire.JobInfo used on opensource projects

Example 1 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobMasterTest method runNestedNonExistingJobConfig.

@Test
public void runNestedNonExistingJobConfig() throws Exception {
    JobConfig innerJobConfig = new CompositeConfig(Lists.newArrayList(new DummyPlanConfig()), true);
    CompositeConfig jobConfig = new CompositeConfig(Lists.newArrayList(innerJobConfig), true);
    long jobId = mJobMaster.run(jobConfig);
    JobInfo status = mJobMaster.getStatus(jobId);
    Assert.assertEquals(Status.FAILED, status.getStatus());
    List<JobInfo> children = status.getChildren();
    Assert.assertEquals(1, children.size());
    JobInfo child = children.get(0);
    Assert.assertEquals(Status.FAILED, child.getStatus());
    Assert.assertEquals(0, child.getChildren().size());
}

Example 2 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class WorkflowTracker method getStatus.

/**
 * Gets information of the given job id.
 *
 * @param jobId the id of the job
 * @param verbose whether the output should be verbose
 * @return null if the job id isn't know by the workflow tracker. WorkflowInfo otherwise
 */
public WorkflowInfo getStatus(long jobId, boolean verbose) {
    WorkflowExecution workflowExecution = mWorkflows.get(jobId);
    if (workflowExecution == null) {
        return null;
    }
    ArrayList<Long> children = Lists.newArrayList(mChildren.get(jobId).iterator());
    Collections.sort(children);
    List<JobInfo> jobInfos = Lists.newArrayList();
    if (verbose) {
        for (long child : children) {
            try {
                jobInfos.add(mJobMaster.getStatus(child));
            } catch (JobDoesNotExistException e) {
                LOG.info(String.format("No job info on child job id %s. Skipping", child));
            }
        }
    }
    WorkflowInfo workflowInfo = new WorkflowInfo(jobId, workflowExecution.getName(), workflowExecution.getStatus(), workflowExecution.getLastUpdated(), workflowExecution.getErrorType(), workflowExecution.getErrorMessage(), jobInfos);
    return workflowInfo;
}

Example 3 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobServiceMetricsCommand method printJobInfos.

private void printJobInfos(List<JobInfo> jobInfos) {
    for (JobInfo jobInfo : jobInfos) {
        mPrintStream.print(String.format("Timestamp: %-30s", CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), mDateFormatPattern)));
        mPrintStream.print(String.format("Id: %-20s", jobInfo.getId()));
        mPrintStream.print(String.format("Name: %-20s", jobInfo.getName()));
        mPrintStream.println(String.format("Status: %s", jobInfo.getStatus()));
    }
    mPrintStream.println();
}

Example 4 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobServiceMetricsCommandTest method testBasic.

@Test
public void testBasic() throws IOException, ParseException {
    JobWorkerHealth jobWorkerHealth = new JobWorkerHealth(1, Lists.newArrayList(1.2, 0.9, 0.7), 10, 2, 2, "testHost");
    Mockito.when(mJobMasterClient.getAllWorkerHealth()).thenReturn(Lists.newArrayList(jobWorkerHealth));
    List<JobInfo> jobInfos = new ArrayList<>();
    jobInfos.add(createJobInfo(1, "Test1", Status.RUNNING, "2019-10-17 12:00:00"));
    jobInfos.add(createJobInfo(2, "Test2", Status.FAILED, "2019-10-17 12:30:15"));
    Mockito.when(mJobMasterClient.getJobServiceSummary()).thenReturn(new JobServiceSummary(jobInfos));
    new JobServiceMetricsCommand(mJobMasterClient, mPrintStream, "MM-dd-yyyy HH:mm:ss:SSS").run();
    String output = new String(mOutputStream.toByteArray(), StandardCharsets.UTF_8);
    String[] lineByLine = output.split("\n");
    // Worker Health Section
    assertEquals("Worker: testHost    Task Pool Size: 10     Unfinished Tasks: 2" + "      Active Tasks: 2      Load Avg: 1.2, 0.9, 0.7", lineByLine[0]);
    assertEquals("", lineByLine[1]);
    // Group By Status
    lineByLine = ArrayUtils.subarray(lineByLine, 2, lineByLine.length);
    assertEquals("Status: CREATED   Count: 0", lineByLine[0]);
    assertEquals("Status: CANCELED  Count: 0", lineByLine[1]);
    assertEquals("Status: FAILED    Count: 1", lineByLine[2]);
    assertEquals("Status: RUNNING   Count: 1", lineByLine[3]);
    assertEquals("Status: COMPLETED Count: 0", lineByLine[4]);
    assertEquals("", lineByLine[5]);
    // Top 10
    lineByLine = ArrayUtils.subarray(lineByLine, 6, lineByLine.length);
    assertEquals("10 Most Recently Modified Jobs:", lineByLine[0]);
    assertEquals("Timestamp: 01-17-2019 12:30:15:000       Id: 2                   Name: Test2" + "               Status: FAILED", lineByLine[1]);
    assertEquals("Timestamp: 01-17-2019 12:00:00:000       Id: 1                   Name: Test1" + "               Status: RUNNING", lineByLine[2]);
    assertEquals("", lineByLine[3]);
    assertEquals("10 Most Recently Failed Jobs:", lineByLine[4]);
    assertEquals("Timestamp: 01-17-2019 12:30:15:000       Id: 2                   Name: Test2" + "               Status: FAILED", lineByLine[5]);
    assertEquals("", lineByLine[6]);
    assertEquals("10 Longest Running Jobs:", lineByLine[7]);
    assertEquals("Timestamp: 01-17-2019 12:00:00:000       Id: 1                   Name: Test1" + "               Status: RUNNING", lineByLine[8]);
}

Example 5 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class TableMasterJournalIntegrationTest method journalTransformDb.

@Test
public void journalTransformDb() throws Exception {
    LocalAlluxioCluster mCluster = sClusterResource.get();
    TableMaster tableMaster = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(TableMaster.class);
    LocalAlluxioJobCluster jobCluster = new LocalAlluxioJobCluster();
    jobCluster.start();
    JobMaster jobMaster = jobCluster.getMaster().getJobMaster();
    genTable(1, 2, true);
    tableMaster.attachDatabase(TestUdbFactory.TYPE, "connect", DB_NAME, DB_NAME, Collections.emptyMap(), false);
    List<String> tables = tableMaster.getAllTables(DB_NAME);
    assertFalse(tables.isEmpty());
    // all partitions are not transformed, so baselayout is the same as layout
    String tableName = tables.get(0);
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() == partition.getLayout()));
    long jobid = tableMaster.transformTable(DB_NAME, tableName, null);
    assertNotEquals(0, jobid);
    JobTestUtils.waitForJobStatus(jobMaster, jobid, ImmutableSet.of(Status.COMPLETED, Status.CANCELED, Status.FAILED));
    final JobInfo status = jobMaster.getStatus(jobid);
    assertEquals("", status.getErrorMessage());
    assertEquals(Status.COMPLETED, status.getStatus());
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TABLE_TRANSFORMATION_MONITOR);
    // all partitions are transformed, so baselayout should be different as layout
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() != partition.getLayout()));
    restartMaster();
    genTable(1, 4, true);
    TableMaster tableMasterRestart = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(TableMaster.class);
    Table table = tableMaster.getTable(DB_NAME, tableName);
    // all partitions remain transformed
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() != partition.getLayout()));
    tableMasterRestart.syncDatabase(DB_NAME);
    // The first two partitions should remain transformed, the new partitions are not transformed
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> (partition.getSpec().endsWith("0") || partition.getSpec().endsWith("1")) == (partition.getBaseLayout() != partition.getLayout())));
}