Examples with JobException org.apache.metron.job.JobException used on opensource projects

Example 6 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapJob method get.

/**
 * Synchronous call blocks until completion.
 */
@Override
public Pageable<Path> get() throws JobException, InterruptedException {
    if (PcapOptions.PRINT_JOB_STATUS.getOrDefault(configuration, Boolean.class, false) && mrJob != null) {
        try {
            mrJob.monitorAndPrintJob();
        } catch (IOException e) {
            throw new JobException("Could not monitor job status", e);
        }
    }
    for (; ; ) {
        JobStatus status = getStatus();
        if (status.getState() == State.SUCCEEDED || status.getState() == State.KILLED || status.getState() == State.FAILED) {
            return getFinalResults();
        } else {
            LOG.info("Percent complete: {}, description: {}", status.getPercentComplete(), status.getDescription());
        }
        Thread.sleep(completeCheckInterval);
    }
}

Example 7 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapJob method query.

/**
 * Run query asynchronously.
 */
public Statusable<Path> query(Optional<String> jobName, Path basePath, Path baseInterimResultPath, long beginNS, long endNS, int numReducers, T fields, Configuration conf, FileSystem fs, PcapFilterConfigurator<T> filterImpl) throws IOException, ClassNotFoundException, InterruptedException {
    String outputDirName = outputDirFormatter.format(beginNS, endNS, filterImpl.queryToString(fields));
    if (LOG.isDebugEnabled()) {
        DateFormat format = SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.LONG, SimpleDateFormat.LONG);
        String from = format.format(new Date(Long.divideUnsigned(beginNS, 1000000)));
        String to = format.format(new Date(Long.divideUnsigned(endNS, 1000000)));
        LOG.debug("Executing query {} on timerange from {} to {}", () -> filterImpl.queryToString(fields), () -> from, () -> to);
    }
    Path interimResultPath = new Path(baseInterimResultPath, outputDirName);
    PcapOptions.INTERIM_RESULT_PATH.put(configuration, interimResultPath);
    mrJob = createJob(jobName, basePath, interimResultPath, beginNS, endNS, numReducers, fields, conf, fs, filterImpl);
    if (mrJob == null) {
        LOG.info("No files to process with specified date range.");
        try {
            setFinalResults(input -> new PcapPages(), configuration);
            jobStatus.withState(State.SUCCEEDED).withDescription("No results in specified date range.").withPercentComplete(100.0);
        } catch (JobException e) {
            // This should not cause an error as we simply set results to an empty result set.
            jobStatus.withState(State.FAILED).withDescription("Unable to finalize empty job.").withFailureException(e);
        }
        return this;
    }
    synchronized (this) {
        // this block synchronized for proper variable visibility across threads once the status timer
        // is started. mrJob and jobStatus need to be synchronized so that their references and internal
        // state are made available to the timer thread. The references to these variables above need
        // not be synchronized because the job will exit when only 1 thread will have had to use them.
        mrJob.submit();
        jobStatus.withState(State.SUBMITTED).withDescription("Job submitted").withJobId(mrJob.getJobID().toString());
    }
    startJobStatusTimerThread(statusInterval);
    return this;
}

Example 8 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapJob method updateStatus.

/**
 * Update job status info. Will finalize job when underlying MR job completes.
 *
 * @return true if should continue updating status, false otherwise.
 */
private boolean updateStatus() {
    JobStatus tempStatus = null;
    // fraction of total job progress calculation we're allocating to the MR job vs finalization
    final float mrJobFraction = 0.75f;
    synchronized (this) {
        tempStatus = new JobStatus(jobStatus);
    }
    boolean keepUpdating = true;
    try {
        boolean mrJobComplete = false;
        org.apache.hadoop.mapreduce.JobStatus.State mrJobState = null;
        String mrJobFailureInfo = null;
        float mapProg = 0.0f;
        float reduceProg = 0.0f;
        synchronized (this) {
            mrJobComplete = mrJob.isComplete();
            org.apache.hadoop.mapreduce.JobStatus mrJobStatus = mrJob.getStatus();
            mrJobState = mrJobStatus.getState();
            mrJobFailureInfo = mrJobStatus.getFailureInfo();
            mapProg = mrJob.mapProgress();
            reduceProg = mrJob.reduceProgress();
        }
        if (mrJobComplete) {
            switch(mrJobState) {
                case SUCCEEDED:
                    tempStatus.withPercentComplete(100.0 * mrJobFraction).withState(State.FINALIZING).withDescription("Finalizing job.");
                    try {
                        synchronized (this) {
                            // want to update the description while the job is finalizing
                            jobStatus = new JobStatus(tempStatus);
                        }
                        setFinalResults(finalizer, configuration);
                        tempStatus.withPercentComplete(100.0).withState(State.SUCCEEDED).withDescription("Job completed.");
                    } catch (JobException je) {
                        tempStatus.withPercentComplete(100.0).withState(State.FAILED).withDescription("Job finalize failed.").withFailureException(je);
                    }
                    break;
                case FAILED:
                    tempStatus.withPercentComplete(100.0).withState(State.FAILED).withDescription(mrJobFailureInfo);
                    break;
                case KILLED:
                    tempStatus.withPercentComplete(100.0).withState(State.KILLED).withDescription(mrJobFailureInfo);
                    break;
            }
            keepUpdating = false;
        } else {
            float mrJobProgress = ((mapProg / 2) + (reduceProg / 2)) * 100;
            float totalProgress = mrJobProgress * mrJobFraction;
            String description = String.format("map: %s%%, reduce: %s%%", mapProg * 100, reduceProg * 100);
            tempStatus.withPercentComplete(totalProgress).withState(State.RUNNING).withDescription(description);
        }
    } catch (InterruptedException | IOException e) {
        tempStatus.withPercentComplete(100.0).withState(State.FAILED).withFailureException(e);
        keepUpdating = false;
    }
    synchronized (this) {
        jobStatus = new JobStatus(tempStatus);
    }
    return keepUpdating;
}