Examples with InMemoryJobManager org.apache.metron.job.manager.InMemoryJobManager used on opensource projects

Example 6 with InMemoryJobManager

use of org.apache.metron.job.manager.InMemoryJobManager in project metron by apache.

the class PcapServiceImplTest method submitShouldProperlySubmitFixedPcapRequest.

@Test
public void submitShouldProperlySubmitFixedPcapRequest() throws Exception {
    when(environment.containsProperty(MetronRestConstants.PCAP_YARN_QUEUE_SPRING_PROPERTY)).thenReturn(true);
    when(environment.getProperty(MetronRestConstants.PCAP_YARN_QUEUE_SPRING_PROPERTY)).thenReturn("pcap");
    FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
    fixedPcapRequest.setBasePath("basePath");
    fixedPcapRequest.setBaseInterimResultPath("baseOutputPath");
    fixedPcapRequest.setFinalOutputPath("finalOutputPath");
    fixedPcapRequest.setStartTimeMs(1L);
    fixedPcapRequest.setEndTimeMs(2L);
    fixedPcapRequest.setNumReducers(2);
    fixedPcapRequest.setIpSrcAddr("ip_src_addr");
    fixedPcapRequest.setIpDstAddr("ip_dst_addr");
    fixedPcapRequest.setIpSrcPort(1000);
    fixedPcapRequest.setIpDstPort(2000);
    fixedPcapRequest.setProtocol("tcp");
    fixedPcapRequest.setPacketFilter("filter");
    fixedPcapRequest.setIncludeReverse(true);
    MockPcapJob mockPcapJob = new MockPcapJob();
    mockPcapJobSupplier.setMockPcapJob(mockPcapJob);
    JobManager jobManager = new InMemoryJobManager<>();
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withDescription("description").withPercentComplete(0L).withState(JobStatus.State.RUNNING));
    Map<String, String> expectedFields = new HashMap<String, String>() {

        {
            put(Constants.Fields.SRC_ADDR.getName(), "ip_src_addr");
            put(Constants.Fields.DST_ADDR.getName(), "ip_dst_addr");
            put(Constants.Fields.SRC_PORT.getName(), "1000");
            put(Constants.Fields.DST_PORT.getName(), "2000");
            put(Constants.Fields.PROTOCOL.getName(), "tcp");
            put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
            put(PcapHelper.PacketFields.PACKET_FILTER.getName(), "filter");
        }
    };
    PcapStatus expectedPcapStatus = new PcapStatus();
    expectedPcapStatus.setJobId("jobId");
    expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
    expectedPcapStatus.setDescription("description");
    assertEquals(expectedPcapStatus, pcapService.submit("user", fixedPcapRequest));
    assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus()));
    assertEquals("basePath", mockPcapJob.getBasePath());
    assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath());
    assertEquals("finalOutputPath", mockPcapJob.getFinalOutputPath());
    assertEquals(1000000, mockPcapJob.getStartTimeNs());
    assertEquals(2000000, mockPcapJob.getEndTimeNs());
    assertEquals(2, mockPcapJob.getNumReducers());
    assertEquals(100, mockPcapJob.getRecPerFile());
    assertEquals("pcap", mockPcapJob.getYarnQueue());
    assertEquals("2C", mockPcapJob.getFinalizerThreadpoolSize());
    assertTrue(mockPcapJob.getFilterImpl() instanceof FixedPcapFilter.Configurator);
    Map<String, String> actualFixedFields = mockPcapJob.getFixedFields();
    assertEquals("ip_src_addr", actualFixedFields.get(Constants.Fields.SRC_ADDR.getName()));
    assertEquals("1000", actualFixedFields.get(Constants.Fields.SRC_PORT.getName()));
    assertEquals("ip_dst_addr", actualFixedFields.get(Constants.Fields.DST_ADDR.getName()));
    assertEquals("2000", actualFixedFields.get(Constants.Fields.DST_PORT.getName()));
    assertEquals("true", actualFixedFields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName()));
    assertEquals("tcp", actualFixedFields.get(Constants.Fields.PROTOCOL.getName()));
    assertEquals("filter", actualFixedFields.get(PcapHelper.PacketFields.PACKET_FILTER.getName()));
}

Example 7 with InMemoryJobManager

use of org.apache.metron.job.manager.InMemoryJobManager in project metron by apache.

the class PcapServiceImplTest method getRawShouldReturnNullOnNonexistentPath.

@Test
public void getRawShouldReturnNullOnNonexistentPath() throws Exception {
    Path path = new Path("/some/path");
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    when(fileSystem.exists(path)).thenReturn(false);
    doReturn(path).when(pcapService).getPath("user", "jobId", 1);
    assertNull(pcapService.getRawPcap("user", "jobId", 1));
}

Example 8 with InMemoryJobManager

use of org.apache.metron.job.manager.InMemoryJobManager in project metron by apache.

the class PcapServiceImplTest method submitShouldProperlySubmitQueryPcapRequest.

@Test
public void submitShouldProperlySubmitQueryPcapRequest() throws Exception {
    QueryPcapRequest queryPcapRequest = new QueryPcapRequest();
    queryPcapRequest.setBasePath("basePath");
    queryPcapRequest.setBaseInterimResultPath("baseOutputPath");
    queryPcapRequest.setFinalOutputPath("finalOutputPath");
    queryPcapRequest.setStartTimeMs(1L);
    queryPcapRequest.setEndTimeMs(2L);
    queryPcapRequest.setNumReducers(2);
    queryPcapRequest.setQuery("query");
    MockPcapJob mockPcapJob = new MockPcapJob();
    mockPcapJobSupplier.setMockPcapJob(mockPcapJob);
    JobManager jobManager = new InMemoryJobManager<>();
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withDescription("description").withPercentComplete(0L).withState(JobStatus.State.RUNNING));
    String expectedFields = "query";
    PcapStatus expectedPcapStatus = new PcapStatus();
    expectedPcapStatus.setJobId("jobId");
    expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
    expectedPcapStatus.setDescription("description");
    assertEquals(expectedPcapStatus, pcapService.submit("user", queryPcapRequest));
    assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus()));
    assertEquals("basePath", mockPcapJob.getBasePath());
    assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath());
    assertEquals("finalOutputPath", mockPcapJob.getFinalOutputPath());
    assertEquals(1000000, mockPcapJob.getStartTimeNs());
    assertEquals(2000000, mockPcapJob.getEndTimeNs());
    assertEquals(2, mockPcapJob.getNumReducers());
    assertEquals(100, mockPcapJob.getRecPerFile());
    assertTrue(mockPcapJob.getFilterImpl() instanceof QueryPcapFilter.Configurator);
    Map<String, String> actualFixedFields = mockPcapJob.getFixedFields();
    assertEquals("query", mockPcapJob.getQuery());
}

Example 9 with InMemoryJobManager

use of org.apache.metron.job.manager.InMemoryJobManager in project metron by apache.

the class PcapServiceImplTest method getPdmlShouldGetPdml.

@Test
public void getPdmlShouldGetPdml() throws Exception {
    Path path = new Path("./target");
    PcapToPdmlScriptWrapper pcapToPdmlScriptWrapper = spy(new PcapToPdmlScriptWrapper());
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    when(fileSystem.exists(path)).thenReturn(true);
    doReturn(path).when(pcapService).getPath("user", "jobId", 1);
    doReturn(new ByteArrayInputStream(pdmlXml.getBytes(StandardCharsets.UTF_8))).when(pcapToPdmlScriptWrapper).getRawInputStream(fileSystem, path);
    ProcessBuilder pb = mock(ProcessBuilder.class);
    Process p = mock(Process.class);
    OutputStream outputStream = new ByteArrayOutputStream();
    when(p.getOutputStream()).thenReturn(outputStream);
    when(p.isAlive()).thenReturn(true);
    when(p.getInputStream()).thenReturn(new ByteArrayInputStream(pdmlXml.getBytes(StandardCharsets.UTF_8)));
    doReturn(pb).when(pcapToPdmlScriptWrapper).getProcessBuilder(any());
    when(pb.start()).thenReturn(p);
    assertEquals(JSONUtils.INSTANCE.load(expectedPdml, Pdml.class), pcapService.getPdml("user", "jobId", 1));
}

Example 10 with InMemoryJobManager

use of org.apache.metron.job.manager.InMemoryJobManager in project metron by apache.

the class PcapServiceImplTest method getConfigurationShouldProperlyReturnFixedFilterConfiguration.

@Test
public void getConfigurationShouldProperlyReturnFixedFilterConfiguration() throws Exception {
    FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
    fixedPcapRequest.setBasePath("basePath");
    fixedPcapRequest.setBaseInterimResultPath("baseOutputPath");
    fixedPcapRequest.setFinalOutputPath("finalOutputPath");
    fixedPcapRequest.setStartTimeMs(1L);
    fixedPcapRequest.setEndTimeMs(2L);
    fixedPcapRequest.setNumReducers(2);
    fixedPcapRequest.setIpSrcAddr("ip_src_addr");
    fixedPcapRequest.setIpDstAddr("ip_dst_addr");
    fixedPcapRequest.setIpSrcPort(1000);
    fixedPcapRequest.setIpDstPort(2000);
    fixedPcapRequest.setProtocol("tcp");
    fixedPcapRequest.setPacketFilter("filter");
    fixedPcapRequest.setIncludeReverse(true);
    MockPcapJob mockPcapJob = new MockPcapJob();
    mockPcapJobSupplier.setMockPcapJob(mockPcapJob);
    JobManager jobManager = new InMemoryJobManager<>();
    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper));
    FileSystem fileSystem = mock(FileSystem.class);
    doReturn(fileSystem).when(pcapService).getFileSystem();
    mockPcapJob.setStatus(new JobStatus().withJobId("jobId"));
    pcapService.submit("user", fixedPcapRequest);
    Map<String, Object> configuration = pcapService.getConfiguration("user", "jobId");
    assertEquals("basePath", configuration.get(PcapOptions.BASE_PATH.getKey()));
    assertEquals("finalOutputPath", configuration.get(PcapOptions.FINAL_OUTPUT_PATH.getKey()));
    assertEquals(1L, configuration.get(PcapOptions.START_TIME_MS.getKey()));
    assertEquals(2L, configuration.get(PcapOptions.END_TIME_MS.getKey()));
    assertEquals(2, configuration.get(PcapOptions.NUM_REDUCERS.getKey()));
    assertEquals("ip_src_addr", configuration.get(FixedPcapOptions.IP_SRC_ADDR.getKey()));
    assertEquals("ip_dst_addr", configuration.get(FixedPcapOptions.IP_DST_ADDR.getKey()));
    assertEquals(1000, configuration.get(FixedPcapOptions.IP_SRC_PORT.getKey()));
    assertEquals(2000, configuration.get(FixedPcapOptions.IP_DST_PORT.getKey()));
    assertEquals("tcp", configuration.get(FixedPcapOptions.PROTOCOL.getKey()));
    assertEquals("filter", configuration.get(FixedPcapOptions.PACKET_FILTER.getKey()));
    assertEquals(true, configuration.get(FixedPcapOptions.INCLUDE_REVERSE.getKey()));
}