Examples with JobManager org.apache.metron.job.manager.JobManager used on opensource projects

Example 1 with JobManager

use of org.apache.metron.job.manager.JobManager in project metron by apache.

the class PcapServiceImplTest method getConfigurationShouldProperlyReturnQueryFilterConfiguration.

@Test
public void getConfigurationShouldProperlyReturnQueryFilterConfiguration() throws Exception {
    QueryPcapRequest queryPcapRequest = new QueryPcapRequest();
    queryPcapRequest.setBasePath("basePath");
    queryPcapRequest.setBaseInterimResultPath("baseOutputPath");
    queryPcapRequest.setFinalOutputPath("finalOutputPath");
    queryPcapRequest.setStartTimeMs(1L);
    queryPcapRequest.setEndTimeMs(2L);
    queryPcapRequest.setNumReducers(2);
    queryPcapRequest.setQuery("query");
    MockPcapJob mockPcapJob = new MockPcapJob();
    mockPcapJobSupplier.setMockPcapJob(mockPcapJob);
    JobManager jobManager = new InMemoryJobManager<>();
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId"));
    pcapService.submit("user", queryPcapRequest);
    Map<String, Object> configuration = pcapService.getConfiguration("user", "jobId");
    assertEquals("basePath", configuration.get(PcapOptions.BASE_PATH.getKey()));
    assertEquals("finalOutputPath", configuration.get(PcapOptions.FINAL_OUTPUT_PATH.getKey()));
    assertEquals(1L, configuration.get(PcapOptions.START_TIME_MS.getKey()));
    assertEquals(2L, configuration.get(PcapOptions.END_TIME_MS.getKey()));
    assertEquals(2, configuration.get(PcapOptions.NUM_REDUCERS.getKey()));
    assertEquals("query", configuration.get(QueryPcapOptions.QUERY.getKey()));
}

Example 2 with JobManager

use of org.apache.metron.job.manager.JobManager in project metron by apache.

the class PcapServiceImplTest method getStatusShouldThrowRestException.

@Test
public void getStatusShouldThrowRestException() throws Exception {
    JobManager jobManager = mock(JobManager.class);
    when(jobManager.getJob("user", "jobId")).thenThrow(new JobException("some job exception"));
    PcapServiceImpl pcapService = new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), jobManager, pcapToPdmlScriptWrapper);
    RestException e = assertThrows(RestException.class, () -> pcapService.getJobStatus("user", "jobId"));
    assertEquals("some job exception", e.getMessage());
}

Example 3 with JobManager

use of org.apache.metron.job.manager.JobManager in project metron by apache.

the class PcapServiceImplTest method submitShouldProperlySubmitFixedPcapRequest.

@Test
public void submitShouldProperlySubmitFixedPcapRequest() throws Exception {
    when(environment.containsProperty(MetronRestConstants.PCAP_YARN_QUEUE_SPRING_PROPERTY)).thenReturn(true);
    when(environment.getProperty(MetronRestConstants.PCAP_YARN_QUEUE_SPRING_PROPERTY)).thenReturn("pcap");
    FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
    fixedPcapRequest.setBasePath("basePath");
    fixedPcapRequest.setBaseInterimResultPath("baseOutputPath");
    fixedPcapRequest.setFinalOutputPath("finalOutputPath");
    fixedPcapRequest.setStartTimeMs(1L);
    fixedPcapRequest.setEndTimeMs(2L);
    fixedPcapRequest.setNumReducers(2);
    fixedPcapRequest.setIpSrcAddr("ip_src_addr");
    fixedPcapRequest.setIpDstAddr("ip_dst_addr");
    fixedPcapRequest.setIpSrcPort(1000);
    fixedPcapRequest.setIpDstPort(2000);
    fixedPcapRequest.setProtocol("tcp");
    fixedPcapRequest.setPacketFilter("filter");
    fixedPcapRequest.setIncludeReverse(true);
    MockPcapJob mockPcapJob = new MockPcapJob();
    mockPcapJobSupplier.setMockPcapJob(mockPcapJob);
    JobManager jobManager = new InMemoryJobManager<>();
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withDescription("description").withPercentComplete(0L).withState(JobStatus.State.RUNNING));
    Map<String, String> expectedFields = new HashMap<String, String>() {

        {
            put(Constants.Fields.SRC_ADDR.getName(), "ip_src_addr");
            put(Constants.Fields.DST_ADDR.getName(), "ip_dst_addr");
            put(Constants.Fields.SRC_PORT.getName(), "1000");
            put(Constants.Fields.DST_PORT.getName(), "2000");
            put(Constants.Fields.PROTOCOL.getName(), "tcp");
            put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
            put(PcapHelper.PacketFields.PACKET_FILTER.getName(), "filter");
        }
    };
    PcapStatus expectedPcapStatus = new PcapStatus();
    expectedPcapStatus.setJobId("jobId");
    expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
    expectedPcapStatus.setDescription("description");
    assertEquals(expectedPcapStatus, pcapService.submit("user", fixedPcapRequest));
    assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus()));
    assertEquals("basePath", mockPcapJob.getBasePath());
    assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath());
    assertEquals("finalOutputPath", mockPcapJob.getFinalOutputPath());
    assertEquals(1000000, mockPcapJob.getStartTimeNs());
    assertEquals(2000000, mockPcapJob.getEndTimeNs());
    assertEquals(2, mockPcapJob.getNumReducers());
    assertEquals(100, mockPcapJob.getRecPerFile());
    assertEquals("pcap", mockPcapJob.getYarnQueue());
    assertEquals("2C", mockPcapJob.getFinalizerThreadpoolSize());
    assertTrue(mockPcapJob.getFilterImpl() instanceof FixedPcapFilter.Configurator);
    Map<String, String> actualFixedFields = mockPcapJob.getFixedFields();
    assertEquals("ip_src_addr", actualFixedFields.get(Constants.Fields.SRC_ADDR.getName()));
    assertEquals("1000", actualFixedFields.get(Constants.Fields.SRC_PORT.getName()));
    assertEquals("ip_dst_addr", actualFixedFields.get(Constants.Fields.DST_ADDR.getName()));
    assertEquals("2000", actualFixedFields.get(Constants.Fields.DST_PORT.getName()));
    assertEquals("true", actualFixedFields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName()));
    assertEquals("tcp", actualFixedFields.get(Constants.Fields.PROTOCOL.getName()));
    assertEquals("filter", actualFixedFields.get(PcapHelper.PacketFields.PACKET_FILTER.getName()));
}

Example 4 with JobManager

use of org.apache.metron.job.manager.JobManager in project metron by apache.

the class PcapServiceImplTest method getPathShouldReturnNullOnInvalidPageSize.

@Test
public void getPathShouldReturnNullOnInvalidPageSize() throws Exception {
    MockPcapJob mockPcapJob = mock(MockPcapJob.class);
    JobManager jobManager = mock(JobManager.class);
    Pageable pageable = mock(Pageable.class);
    PcapServiceImpl pcapService = new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), jobManager, pcapToPdmlScriptWrapper);
    when(pageable.getSize()).thenReturn(2);
    when(mockPcapJob.isDone()).thenReturn(true);
    when(mockPcapJob.get()).thenReturn(pageable);
    when(jobManager.getJob("user", "jobId")).thenReturn(mockPcapJob);
    assertNull(pcapService.getPath("user", "jobId", 0));
    assertNull(pcapService.getPath("user", "jobId", 3));
}

Example 5 with JobManager

use of org.apache.metron.job.manager.JobManager in project metron by apache.

the class PcapServiceImplTest method getStatusForStateShouldProperlyReturnJobs.

@Test
public void getStatusForStateShouldProperlyReturnJobs() throws Exception {
    MockPcapJob mockPcapJob = mock(MockPcapJob.class);
    JobManager jobManager = mock(JobManager.class);
    Statusable<Path> runningJob = mock(Statusable.class);
    JobStatus runningStatus = mock(JobStatus.class);
    when(runningStatus.getJobId()).thenReturn("runningJob");
    when(runningStatus.getState()).thenReturn(JobStatus.State.RUNNING);
    when(runningJob.getStatus()).thenReturn(runningStatus);
    Statusable<Path> failedJob = mock(Statusable.class);
    when(failedJob.getStatus()).thenThrow(new JobException("job exception"));
    Statusable<Path> succeededJob = mock(Statusable.class);
    JobStatus succeededStatus = mock(JobStatus.class);
    when(succeededStatus.getJobId()).thenReturn("succeededJob");
    when(succeededStatus.getState()).thenReturn(JobStatus.State.SUCCEEDED);
    when(succeededJob.isDone()).thenReturn(true);
    when(succeededJob.getStatus()).thenReturn(succeededStatus);
    Pageable<Path> succeededPageable = mock(Pageable.class);
    when(succeededPageable.getSize()).thenReturn(5);
    when(succeededJob.get()).thenReturn(succeededPageable);
    when(jobManager.getJobs("user")).thenReturn(Arrays.asList(runningJob, failedJob, succeededJob));
    PcapServiceImpl pcapService = new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper);
    PcapStatus expectedRunningPcapStatus = new PcapStatus();
    expectedRunningPcapStatus.setJobId("runningJob");
    expectedRunningPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
    assertEquals(expectedRunningPcapStatus, pcapService.getJobStatus("user", JobStatus.State.RUNNING).get(0));
    PcapStatus expectedFailedPcapStatus = new PcapStatus();
    expectedFailedPcapStatus.setJobStatus(JobStatus.State.FAILED.name());
    expectedFailedPcapStatus.setDescription("job exception");
    assertEquals(expectedFailedPcapStatus, pcapService.getJobStatus("user", JobStatus.State.FAILED).get(0));
    PcapStatus expectedSucceededPcapStatus = new PcapStatus();
    expectedSucceededPcapStatus.setJobId("succeededJob");
    expectedSucceededPcapStatus.setJobStatus(JobStatus.State.SUCCEEDED.name());
    expectedSucceededPcapStatus.setPageTotal(5);
    assertEquals(expectedSucceededPcapStatus, pcapService.getJobStatus("user", JobStatus.State.SUCCEEDED).get(0));
}