Examples with JobInfo alluxio.job.wire.JobInfo used on opensource projects

Example 16 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class Benchmark method runSingleTask.

protected String runSingleTask(String[] args) throws Exception {
    // prepare the benchmark.
    prepare();
    if (!mBaseParameters.mProfileAgent.isEmpty()) {
        mBaseParameters.mJavaOpts.add("-javaagent:" + mBaseParameters.mProfileAgent + "=" + BaseParameters.AGENT_OUTPUT_PATH);
    }
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    String className = this.getClass().getCanonicalName();
    if (mBaseParameters.mCluster) {
        // run on job service
        long jobId = JobGrpcClientUtils.run(generateJobConfig(args), 0, conf);
        JobInfo jobInfo = JobGrpcClientUtils.getJobStatus(jobId, conf, true);
        return jobInfo.getResult().toString();
    }
    // run locally
    if (mBaseParameters.mInProcess) {
        // run in process
        T result = runLocal();
        if (mBaseParameters.mDistributed) {
            return result.toJson();
        }
        // aggregate the results
        final String s = result.aggregator().aggregate(Collections.singletonList(result)).toJson();
        return s;
    } else {
        // Spawn a new process
        List<String> command = new ArrayList<>();
        command.add(conf.get(PropertyKey.HOME) + "/bin/alluxio");
        command.add("runClass");
        command.add(className);
        command.addAll(Arrays.asList(args));
        command.add(BaseParameters.IN_PROCESS_FLAG);
        command.addAll(mBaseParameters.mJavaOpts.stream().map(String::trim).collect(Collectors.toList()));
        LOG.info("running command: " + String.join(" ", command));
        return ShellUtils.execCommand(command.toArray(new String[0]));
    }
}

Example 17 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class StressJobServiceBench method runNoop.

private void runNoop() throws IOException, InterruptedException, TimeoutException {
    long jobId = mJobMasterClient.run(new NoopPlanConfig());
    // TODO(jianjian): refactor JobTestUtils
    ImmutableSet<Status> statuses = ImmutableSet.of(Status.COMPLETED, Status.CANCELED, Status.FAILED);
    final AtomicReference<JobInfo> singleton = new AtomicReference<>();
    CommonUtils.waitFor(String.format("job %d to be one of status %s", jobId, Arrays.toString(statuses.toArray())), () -> {
        JobInfo info;
        try {
            info = mJobMasterClient.getJobStatus(jobId);
            if (statuses.contains(info.getStatus())) {
                singleton.set(info);
            }
            return statuses.contains(info.getStatus());
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }, WaitForOptions.defaults().setTimeoutMs(30 * Constants.SECOND_MS));
    JobInfo jobInfo = singleton.get();
    if (jobInfo.getStatus().equals(Status.FAILED)) {
        throw new IOException(jobInfo.getErrorMessage());
    }
}

Example 18 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobAttempt method check.

/**
 * Returns the status of the job attempt.
 * @return True if finished successfully or cancelled, False if FAILED and should be retried,
 *              null if the status should be checked again later
 */
public Status check() {
    if (mJobId == null) {
        return Status.FAILED;
    }
    JobInfo jobInfo;
    try {
        jobInfo = mClient.getJobStatus(mJobId);
    } catch (IOException e) {
        LOG.warn("Failed to get status for job (jobId={})", mJobId, e);
        return Status.FAILED;
    }
    // This make an assumption that this job tree only goes 1 level deep
    boolean finished = true;
    for (JobInfo child : jobInfo.getChildren()) {
        if (!child.getStatus().isFinished()) {
            finished = false;
            break;
        }
    }
    if (finished) {
        if (jobInfo.getStatus().equals(Status.FAILED)) {
            logFailedAttempt(jobInfo);
        } else if (jobInfo.getStatus().equals(Status.COMPLETED)) {
            logCompleted();
        }
        return jobInfo.getStatus();
    }
    return Status.RUNNING;
}

Example 19 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobMaster method listDetailed.

/**
 * @return list of all job infos
 */
public List<JobInfo> listDetailed() {
    try (JobMasterAuditContext auditContext = createAuditContext("listDetailed")) {
        List<JobInfo> jobInfos = new ArrayList<>();
        for (PlanCoordinator coordinator : mPlanTracker.coordinators()) {
            jobInfos.add(coordinator.getPlanInfoWire(false));
        }
        jobInfos.addAll(mWorkflowTracker.getAllInfo());
        jobInfos.sort(Comparator.comparingLong(JobInfo::getId));
        auditContext.setSucceeded(true);
        return jobInfos;
    }
}

Example 20 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobServiceMetricsCommand method run.

/**
 * Runs a job services report metrics command.
 *
 * @return 0 on success, 1 otherwise
 */
public int run() throws IOException {
    List<JobWorkerHealth> allWorkerHealth = mJobMasterClient.getAllWorkerHealth();
    for (JobWorkerHealth workerHealth : allWorkerHealth) {
        mPrintStream.print(String.format("Worker: %-10s  ", workerHealth.getHostname()));
        mPrintStream.print(String.format("Task Pool Size: %-7s", workerHealth.getTaskPoolSize()));
        mPrintStream.print(String.format("Unfinished Tasks: %-7s", workerHealth.getUnfinishedTasks()));
        mPrintStream.print(String.format("Active Tasks: %-7s", workerHealth.getNumActiveTasks()));
        mPrintStream.println(String.format("Load Avg: %s", StringUtils.join(workerHealth.getLoadAverage(), ", ")));
    }
    mPrintStream.println();
    JobServiceSummary jobServiceSummary = mJobMasterClient.getJobServiceSummary();
    Collection<StatusSummary> jobStatusSummaries = jobServiceSummary.getSummaryPerStatus();
    for (StatusSummary statusSummary : jobStatusSummaries) {
        mPrintStream.print(String.format("Status: %-10s", statusSummary.getStatus()));
        mPrintStream.println(String.format("Count: %s", statusSummary.getCount()));
    }
    mPrintStream.println();
    mPrintStream.println(String.format("%s Most Recently Modified Jobs:", JobServiceSummary.RECENT_LENGTH));
    List<JobInfo> lastActivities = jobServiceSummary.getRecentActivities();
    printJobInfos(lastActivities);
    mPrintStream.println(String.format("%s Most Recently Failed Jobs:", JobServiceSummary.RECENT_LENGTH));
    List<JobInfo> lastFailures = jobServiceSummary.getRecentFailures();
    printJobInfos(lastFailures);
    mPrintStream.println(String.format("%s Longest Running Jobs:", JobServiceSummary.RECENT_LENGTH));
    List<JobInfo> longestRunning = jobServiceSummary.getLongestRunning();
    printJobInfos(longestRunning);
    return 0;
}