Examples with HadoopJobStatus org.apache.ignite.internal.processors.hadoop.HadoopJobStatus used on opensource projects

Example 1 with HadoopJobStatus

use of org.apache.ignite.internal.processors.hadoop.HadoopJobStatus in project ignite by apache.

the class HadoopJobTrackerSelfTest method checkStatus.

/**
     * Checks job execution status.
     *
     * @param jobId Job ID.
     * @param complete Completion status.
     * @throws Exception If failed.
     */
private void checkStatus(HadoopJobId jobId, boolean complete) throws Exception {
    for (int i = 0; i < gridCount(); i++) {
        IgniteKernal kernal = (IgniteKernal) grid(i);
        Hadoop hadoop = kernal.hadoop();
        HadoopJobStatus stat = hadoop.status(jobId);
        assert stat != null;
        IgniteInternalFuture<?> fut = hadoop.finishFuture(jobId);
        if (!complete)
            assertFalse(fut.isDone());
        else {
            info("Waiting for status future completion on node [idx=" + i + ", nodeId=" + kernal.getLocalNodeId() + ']');
            fut.get();
        }
    }
}

Example 2 with HadoopJobStatus

use of org.apache.ignite.internal.processors.hadoop.HadoopJobStatus in project ignite by apache.

the class HadoopClientProtocol method submitJob.

/** {@inheritDoc} */
@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException, InterruptedException {
    try {
        conf.setLong(HadoopCommonUtils.JOB_SUBMISSION_START_TS_PROPERTY, U.currentTimeMillis());
        HadoopJobStatus status = execute(HadoopProtocolSubmitJobTask.class, jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf));
        if (status == null)
            throw new IOException("Failed to submit job (null status obtained): " + jobId);
        return processStatus(status);
    } catch (GridClientException | IgniteCheckedException e) {
        throw new IOException("Failed to submit job.", e);
    }
}

Example 3 with HadoopJobStatus

use of org.apache.ignite.internal.processors.hadoop.HadoopJobStatus in project ignite by apache.

the class HadoopClientProtocol method getJobStatus.

/** {@inheritDoc} */
@Override
public JobStatus getJobStatus(JobID jobId) throws IOException, InterruptedException {
    try {
        Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1);
        HadoopJobStatus status;
        if (delay >= 0)
            status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId(), delay);
        else
            status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId());
        if (status == null)
            throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
        return processStatus(status);
    } catch (GridClientException e) {
        throw new IOException("Failed to get job status: " + jobId, e);
    }
}

Example 4 with HadoopJobStatus

use of org.apache.ignite.internal.processors.hadoop.HadoopJobStatus in project ignite by apache.

the class HadoopUtils method status.

/**
     * Convert Ignite job status to Hadoop job status.
     *
     * @param status Ignite job status.
     * @return Hadoop job status.
     */
public static JobStatus status(HadoopJobStatus status, Configuration conf) {
    JobID jobId = new JobID(status.jobId().globalId().toString(), status.jobId().localId());
    float setupProgress = 0;
    float mapProgress = 0;
    float reduceProgress = 0;
    float cleanupProgress = 0;
    JobStatus.State state = JobStatus.State.RUNNING;
    switch(status.jobPhase()) {
        case PHASE_SETUP:
            setupProgress = 0.42f;
            break;
        case PHASE_MAP:
            setupProgress = 1;
            mapProgress = 1f - status.pendingMapperCnt() / (float) status.totalMapperCnt();
            break;
        case PHASE_REDUCE:
            setupProgress = 1;
            mapProgress = 1;
            if (status.totalReducerCnt() > 0)
                reduceProgress = 1f - status.pendingReducerCnt() / (float) status.totalReducerCnt();
            else
                reduceProgress = 1f;
            break;
        case PHASE_CANCELLING:
        case PHASE_COMPLETE:
            if (!status.isFailed()) {
                setupProgress = 1;
                mapProgress = 1;
                reduceProgress = 1;
                cleanupProgress = 1;
                state = JobStatus.State.SUCCEEDED;
            } else
                state = JobStatus.State.FAILED;
            break;
        default:
            assert false;
    }
    return new JobStatus(jobId, setupProgress, mapProgress, reduceProgress, cleanupProgress, state, JobPriority.NORMAL, status.user(), status.jobName(), jobFile(conf, status.user(), jobId).toString(), "N/A");
}

Example 5 with HadoopJobStatus

use of org.apache.ignite.internal.processors.hadoop.HadoopJobStatus in project ignite by apache.

the class HadoopProtocolSubmitJobTask method run.

/** {@inheritDoc} */
@Override
public HadoopJobStatus run(ComputeJobContext jobCtx, Hadoop hadoop, HadoopProtocolTaskArguments args) throws IgniteCheckedException {
    UUID nodeId = UUID.fromString(args.<String>get(0));
    Integer id = args.get(1);
    HadoopDefaultJobInfo info = args.get(2);
    assert nodeId != null;
    assert id != null;
    assert info != null;
    HadoopJobId jobId = new HadoopJobId(nodeId, id);
    hadoop.submit(jobId, info);
    HadoopJobStatus res = hadoop.status(jobId);
    if (// Submission failed.
    res == null)
        res = new HadoopJobStatus(jobId, info.jobName(), info.user(), 0, 0, 0, 0, PHASE_CANCELLING, true, 1);
    return res;
}