Examples with SequenceFileIterable org.apache.metron.common.hadoop.SequenceFileIterable used on opensource projects

Example 1 with SequenceFileIterable

use of org.apache.metron.common.hadoop.SequenceFileIterable in project metron by apache.

the class PcapReceiverImplRestEasy method getPcapsByIdentifiers.

/**
 * Enable filtering PCAP results by fixed properties and start/end packet TS
 *
 * @param srcIp filter value
 * @param dstIp filter value
 * @param protocol filter value
 * @param srcPort filter value
 * @param dstPort filter value
 * @param startTime filter value
 * @param endTime filter value
 * @param numReducers Specify the number of reducers to use when executing the mapreduce job
 * @param includeReverseTraffic Indicates if filter should check swapped src/dest addresses and IPs
 * @param servlet_response
 * @return REST response
 * @throws IOException
 */
@GET
@Path("/pcapGetter/getPcapsByIdentifiers")
public Response getPcapsByIdentifiers(@QueryParam("srcIp") String srcIp, @QueryParam("dstIp") String dstIp, @QueryParam("protocol") String protocol, @QueryParam("srcPort") String srcPort, @QueryParam("dstPort") String dstPort, @DefaultValue("-1") @QueryParam("startTime") long startTime, @DefaultValue("-1") @QueryParam("endTime") long endTime, @DefaultValue("10") @QueryParam("numReducers") int numReducers, @DefaultValue("false") @QueryParam("includeReverseTraffic") boolean includeReverseTraffic, @DefaultValue("") @QueryParam("packetFilter") String packetFilter, @Context HttpServletResponse servlet_response) throws IOException {
    if (!isValidPort(srcPort)) {
        return Response.serverError().status(Response.Status.NO_CONTENT).entity("'srcPort' must not be null, empty or a non-integer").build();
    }
    if (!isValidPort(dstPort)) {
        return Response.serverError().status(Response.Status.NO_CONTENT).entity("'dstPort' must not be null, empty or a non-integer").build();
    }
    final boolean includeReverseTrafficF = includeReverseTraffic;
    PcapsResponse response = new PcapsResponse();
    SequenceFileIterable results = null;
    try {
        if (startTime < 0) {
            startTime = 0L;
        }
        if (endTime < 0) {
            endTime = System.currentTimeMillis();
        }
        // convert to nanoseconds since the epoch
        startTime = TimestampConverters.MILLISECONDS.toNanoseconds(startTime);
        endTime = TimestampConverters.MILLISECONDS.toNanoseconds(endTime);
        Map<String, String> query = new HashMap<String, String>() {

            {
                if (srcIp != null) {
                    put(Constants.Fields.SRC_ADDR.getName(), srcIp);
                }
                if (dstIp != null) {
                    put(Constants.Fields.DST_ADDR.getName(), dstIp);
                }
                if (srcPort != null) {
                    put(Constants.Fields.SRC_PORT.getName(), srcPort);
                }
                if (dstPort != null) {
                    put(Constants.Fields.DST_PORT.getName(), dstPort);
                }
                if (protocol != null) {
                    put(Constants.Fields.PROTOCOL.getName(), protocol);
                }
                put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "" + includeReverseTrafficF);
                if (!org.apache.commons.lang3.StringUtils.isEmpty(packetFilter)) {
                    put(PcapHelper.PacketFields.PACKET_FILTER.getName(), packetFilter);
                }
            }
        };
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Query received: {}", Joiner.on(",").join(query.entrySet()));
        }
        results = getQueryUtil().query(new org.apache.hadoop.fs.Path(ConfigurationUtil.getPcapOutputPath()), new org.apache.hadoop.fs.Path(ConfigurationUtil.getTempQueryOutputPath()), startTime, endTime, numReducers, query, CONFIGURATION.get(), FileSystem.get(CONFIGURATION.get()), new FixedPcapFilter.Configurator());
        response.setPcaps(results != null ? Lists.newArrayList(results) : null);
    } catch (Exception e) {
        LOGGER.error("Exception occurred while fetching Pcaps by identifiers :", e);
        throw new WebApplicationException("Unable to fetch Pcaps via MR job", e);
    } finally {
        if (null != results) {
            results.cleanup();
        }
    }
    // and headers
    return Response.ok(response.getPcaps(), MediaType.APPLICATION_OCTET_STREAM).status(200).build();
}

Example 2 with SequenceFileIterable

use of org.apache.metron.common.hadoop.SequenceFileIterable in project metron by apache.

the class PcapReceiverImplRestEasy method getPcapsByIdentifiers.

/**
 * Enable filtering PCAP results by query filter string and start/end packet TS
 *
 * @param query Filter results based on this query
 * @param startTime Only return packets originating after this start time
 * @param endTime Only return packets originating before this end time
 * @param numReducers Number of reducers to use
 * @param servlet_response
 * @return REST response
 * @throws IOException
 */
@GET
@Path("/pcapGetter/getPcapsByQuery")
public Response getPcapsByIdentifiers(@QueryParam("query") String query, @DefaultValue("-1") @QueryParam("startTime") long startTime, @DefaultValue("-1") @QueryParam("endTime") long endTime, @DefaultValue("10") @QueryParam("numReducers") int numReducers, @Context HttpServletResponse servlet_response) throws IOException {
    PcapsResponse response = new PcapsResponse();
    SequenceFileIterable results = null;
    try {
        if (startTime < 0) {
            startTime = 0L;
        }
        if (endTime < 0) {
            endTime = System.currentTimeMillis();
        }
        if (query == null) {
            return Response.serverError().status(Response.Status.NO_CONTENT).entity("Query is null").build();
        }
        // convert to nanoseconds since the epoch
        startTime = TimestampConverters.MILLISECONDS.toNanoseconds(startTime);
        endTime = TimestampConverters.MILLISECONDS.toNanoseconds(endTime);
        LOGGER.debug("Query received: {}", query);
        results = getQueryUtil().query(new org.apache.hadoop.fs.Path(ConfigurationUtil.getPcapOutputPath()), new org.apache.hadoop.fs.Path(ConfigurationUtil.getTempQueryOutputPath()), startTime, endTime, numReducers, query, CONFIGURATION.get(), FileSystem.get(CONFIGURATION.get()), new QueryPcapFilter.Configurator());
        response.setPcaps(results != null ? Lists.newArrayList(results) : null);
    } catch (Exception e) {
        LOGGER.error("Exception occurred while fetching Pcaps by identifiers :", e);
        throw new WebApplicationException("Unable to fetch Pcaps via MR job", e);
    } finally {
        if (null != results) {
            results.cleanup();
        }
    }
    // and headers
    return Response.ok(response.getPcaps(), MediaType.APPLICATION_OCTET_STREAM).status(200).build();
}

Example 3 with SequenceFileIterable

use of org.apache.metron.common.hadoop.SequenceFileIterable in project metron by apache.

the class PcapCliTest method runs_query_pcap_filter_job_with_full_argument_list.

@Test
public void runs_query_pcap_filter_job_with_full_argument_list() throws Exception {
    String[] args = { "query", "-start_time", "500", "-end_time", "1000", "-num_reducers", "10", "-base_path", "/base/path", "-base_output_path", "/base/output/path", "-query", "some query string", "-records_per_file", "1000" };
    List<byte[]> pcaps = Arrays.asList(new byte[][] { asBytes("abc"), asBytes("def"), asBytes("ghi") });
    Iterator iterator = pcaps.iterator();
    SequenceFileIterable iterable = mock(SequenceFileIterable.class);
    when(iterable.iterator()).thenReturn(iterator);
    Path base_path = new Path("/base/path");
    Path base_output_path = new Path("/base/output/path");
    String query = "some query string";
    when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable);
    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
    assertThat("Expect no errors on run", cli.run(args), equalTo(0));
    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
}

Example 4 with SequenceFileIterable

use of org.apache.metron.common.hadoop.SequenceFileIterable in project metron by apache.

the class PcapCliTest method runs_fixed_pcap_filter_job_with_default_argument_list.

@Test
public void runs_fixed_pcap_filter_job_with_default_argument_list() throws Exception {
    String[] args = { "fixed", "-start_time", "500", "-ip_src_addr", "192.168.1.1", "-ip_dst_addr", "192.168.1.2", "-ip_src_port", "8081", "-ip_dst_port", "8082", "-protocol", "6", "-packet_filter", "`casey`" };
    List<byte[]> pcaps = Arrays.asList(new byte[][] { asBytes("abc"), asBytes("def"), asBytes("ghi") });
    Iterator iterator = pcaps.iterator();
    SequenceFileIterable iterable = mock(SequenceFileIterable.class);
    when(iterable.iterator()).thenReturn(iterator);
    Path base_path = new Path(CliParser.BASE_PATH_DEFAULT);
    Path base_output_path = new Path(CliParser.BASE_OUTPUT_PATH_DEFAULT);
    HashMap<String, String> query = new HashMap<String, String>() {

        {
            put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
            put(Constants.Fields.DST_ADDR.getName(), "192.168.1.2");
            put(Constants.Fields.SRC_PORT.getName(), "8081");
            put(Constants.Fields.DST_PORT.getName(), "8082");
            put(Constants.Fields.PROTOCOL.getName(), "6");
            put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false");
            put(PcapHelper.PacketFields.PACKET_FILTER.getName(), "`casey`");
        }
    };
    when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
    assertThat("Expect no errors on run", cli.run(args), equalTo(0));
    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
}

Example 5 with SequenceFileIterable

use of org.apache.metron.common.hadoop.SequenceFileIterable in project metron by apache.

the class PcapJob method query.

public <T> SequenceFileIterable query(Path basePath, Path baseOutputPath, long beginNS, long endNS, int numReducers, T fields, Configuration conf, FileSystem fs, PcapFilterConfigurator<T> filterImpl) throws IOException, ClassNotFoundException, InterruptedException {
    String fileName = Joiner.on("_").join(beginNS, endNS, filterImpl.queryToString(fields), UUID.randomUUID().toString());
    if (LOG.isDebugEnabled()) {
        DateFormat format = SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.LONG, SimpleDateFormat.LONG);
        String from = format.format(new Date(Long.divideUnsigned(beginNS, 1000000)));
        String to = format.format(new Date(Long.divideUnsigned(endNS, 1000000)));
        LOG.debug("Executing query {} on timerange from {} to {}", filterImpl.queryToString(fields), from, to);
    }
    Path outputPath = new Path(baseOutputPath, fileName);
    Job job = createJob(basePath, outputPath, beginNS, endNS, numReducers, fields, conf, fs, filterImpl);
    if (job == null) {
        LOG.info("No files to process with specified date range.");
        return new SequenceFileIterable(new ArrayList<>(), conf);
    }
    boolean completed = job.waitForCompletion(true);
    if (completed) {
        return readResults(outputPath, conf, fs);
    } else {
        throw new RuntimeException("Unable to complete query due to errors.  Please check logs for full errors.");
    }
}