Examples with JobStatus org.apache.metron.job.JobStatus used on opensource projects

Example 16 with JobStatus

use of org.apache.metron.job.JobStatus in project metron by apache.

the class PcapControllerIntegrationTest method testGetStatus.

@Test
public void testGetStatus() throws Exception {
    MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING));
    this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.jobId").value("jobId")).andExpect(jsonPath("$.jobStatus").value("RUNNING"));
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.SUCCEEDED));
    Pageable<Path> pageable = new PcapPages(Arrays.asList(new Path("path1"), new Path("path1")));
    mockPcapJob.setIsDone(true);
    mockPcapJob.setPageable(pageable);
    this.mockMvc.perform(get(pcapUrl + "/jobId").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.jobStatus").value("SUCCEEDED")).andExpect(jsonPath("$.pageTotal").value(2));
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.FINALIZING));
    this.mockMvc.perform(get(pcapUrl + "/jobId").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.jobStatus").value("FINALIZING"));
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.FAILED));
    this.mockMvc.perform(get(pcapUrl + "/jobId").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.jobStatus").value("FAILED"));
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.KILLED));
    this.mockMvc.perform(get(pcapUrl + "/jobId").with(httpBasic(user, password))).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.jobStatus").value("KILLED"));
    this.mockMvc.perform(get(pcapUrl + "/someJobId").with(httpBasic(user, password))).andExpect(status().isNotFound());
}

Example 17 with JobStatus

use of org.apache.metron.job.JobStatus in project metron by apache.

the class PcapControllerIntegrationTest method testFixedRequestDefaults.

@Test
public void testFixedRequestDefaults() throws Exception {
    MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
    mockPcapJob.setStatus(new JobStatus().withState(JobStatus.State.RUNNING));
    long beforeJobTime = System.currentTimeMillis();
    this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedWithDefaultsJson)).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.jobStatus").value("RUNNING"));
    assertEquals("/apps/metron/pcap/input", mockPcapJob.getBasePath());
    assertEquals("/apps/metron/pcap/interim", mockPcapJob.getBaseInterrimResultPath());
    assertEquals("/apps/metron/pcap/output", mockPcapJob.getFinalOutputPath());
    assertEquals(0, mockPcapJob.getStartTimeNs());
    assertTrue(beforeJobTime < mockPcapJob.getEndTimeNs() / 1000000);
    assertTrue(System.currentTimeMillis() > mockPcapJob.getEndTimeNs() / 1000000);
    assertEquals(10, mockPcapJob.getNumReducers());
    assertTrue(mockPcapJob.getFilterImpl() instanceof FixedPcapFilter.Configurator);
    Map<String, String> actualFixedFields = mockPcapJob.getFixedFields();
    assertEquals("192.168.1.2", actualFixedFields.get(Constants.Fields.SRC_ADDR.getName()));
    assertEquals("2000", actualFixedFields.get(Constants.Fields.SRC_PORT.getName()));
    assertEquals("192.168.1.1", actualFixedFields.get(Constants.Fields.DST_ADDR.getName()));
    assertEquals("1000", actualFixedFields.get(Constants.Fields.DST_PORT.getName()));
    assertEquals("true", actualFixedFields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName()));
    assertEquals("TCP", actualFixedFields.get(Constants.Fields.PROTOCOL.getName()));
    assertEquals("filter", actualFixedFields.get(PcapHelper.PacketFields.PACKET_FILTER.getName()));
}

Example 18 with JobStatus

use of org.apache.metron.job.JobStatus in project metron by apache.

the class PcapJob method get.

/**
 * Synchronous call blocks until completion.
 */
@Override
public Pageable<Path> get() throws JobException, InterruptedException {
    if (PcapOptions.PRINT_JOB_STATUS.getOrDefault(configuration, Boolean.class, false) && mrJob != null) {
        try {
            mrJob.monitorAndPrintJob();
        } catch (IOException e) {
            throw new JobException("Could not monitor job status", e);
        }
    }
    for (; ; ) {
        JobStatus status = getStatus();
        if (status.getState() == State.SUCCEEDED || status.getState() == State.KILLED || status.getState() == State.FAILED) {
            return getFinalResults();
        } else {
            LOG.info("Percent complete: {}, description: {}", status.getPercentComplete(), status.getDescription());
        }
        Thread.sleep(completeCheckInterval);
    }
}

Example 19 with JobStatus

use of org.apache.metron.job.JobStatus in project metron by apache.

the class PcapJob method updateStatus.

/**
 * Update job status info. Will finalize job when underlying MR job completes.
 *
 * @return true if should continue updating status, false otherwise.
 */
private boolean updateStatus() {
    JobStatus tempStatus = null;
    // fraction of total job progress calculation we're allocating to the MR job vs finalization
    final float mrJobFraction = 0.75f;
    synchronized (this) {
        tempStatus = new JobStatus(jobStatus);
    }
    boolean keepUpdating = true;
    try {
        boolean mrJobComplete = false;
        org.apache.hadoop.mapreduce.JobStatus.State mrJobState = null;
        String mrJobFailureInfo = null;
        float mapProg = 0.0f;
        float reduceProg = 0.0f;
        synchronized (this) {
            mrJobComplete = mrJob.isComplete();
            org.apache.hadoop.mapreduce.JobStatus mrJobStatus = mrJob.getStatus();
            mrJobState = mrJobStatus.getState();
            mrJobFailureInfo = mrJobStatus.getFailureInfo();
            mapProg = mrJob.mapProgress();
            reduceProg = mrJob.reduceProgress();
        }
        if (mrJobComplete) {
            switch(mrJobState) {
                case SUCCEEDED:
                    tempStatus.withPercentComplete(100.0 * mrJobFraction).withState(State.FINALIZING).withDescription("Finalizing job.");
                    try {
                        synchronized (this) {
                            // want to update the description while the job is finalizing
                            jobStatus = new JobStatus(tempStatus);
                        }
                        setFinalResults(finalizer, configuration);
                        tempStatus.withPercentComplete(100.0).withState(State.SUCCEEDED).withDescription("Job completed.");
                    } catch (JobException je) {
                        tempStatus.withPercentComplete(100.0).withState(State.FAILED).withDescription("Job finalize failed.").withFailureException(je);
                    }
                    break;
                case FAILED:
                    tempStatus.withPercentComplete(100.0).withState(State.FAILED).withDescription(mrJobFailureInfo);
                    break;
                case KILLED:
                    tempStatus.withPercentComplete(100.0).withState(State.KILLED).withDescription(mrJobFailureInfo);
                    break;
            }
            keepUpdating = false;
        } else {
            float mrJobProgress = ((mapProg / 2) + (reduceProg / 2)) * 100;
            float totalProgress = mrJobProgress * mrJobFraction;
            String description = String.format("map: %s%%, reduce: %s%%", mapProg * 100, reduceProg * 100);
            tempStatus.withPercentComplete(totalProgress).withState(State.RUNNING).withDescription(description);
        }
    } catch (InterruptedException | IOException e) {
        tempStatus.withPercentComplete(100.0).withState(State.FAILED).withFailureException(e);
        keepUpdating = false;
    }
    synchronized (this) {
        jobStatus = new JobStatus(tempStatus);
    }
    return keepUpdating;
}

Example 20 with JobStatus

use of org.apache.metron.job.JobStatus in project metron by apache.

the class PcapJobTest method job_fails_synchronously.

@Test
public void job_fails_synchronously() throws Exception {
    when(mrJob.isComplete()).thenReturn(true);
    when(mrStatus.getState()).thenReturn(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
    when(mrJob.getStatus()).thenReturn(mrStatus);
    Statusable<Path> statusable = testJob.submit(finalizer, config);
    timer.updateJobStatus();
    Pageable<Path> results = statusable.get();
    JobStatus status = statusable.getStatus();
    assertThat(status.getState(), equalTo(State.FAILED));
    assertThat(status.getPercentComplete(), equalTo(100.0));
    assertThat(results.getSize(), equalTo(0));
}