Examples with ExtractorHandler org.apache.metron.dataloads.extractor.ExtractorHandler used on opensource projects

Example 6 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class SimpleFlatFileSummarizerTest method testLineByLine.

public void testLineByLine(final int numThreads) throws IOException, InvalidWriterOutput {
    ExtractorHandler handler = ExtractorHandler.load(stellarExtractorConfigLineByLine);
    LocalSummarizer summarizer = new MockSummarizer(ImmutableMap.of("input.csv", generateData()));
    final AtomicReference<Object> finalObj = new AtomicReference<>(null);
    EnumMap<SummarizeOptions, Optional<Object>> options = new EnumMap<SummarizeOptions, Optional<Object>>(SummarizeOptions.class) {

        {
            put(SummarizeOptions.INPUT, Optional.of("input.csv"));
            put(SummarizeOptions.BATCH_SIZE, Optional.of(5));
            put(SummarizeOptions.QUIET, Optional.of(true));
            put(SummarizeOptions.OUTPUT_MODE, Optional.of(new PeekingWriter(finalObj)));
            put(SummarizeOptions.OUTPUT, Optional.of("out"));
            put(SummarizeOptions.NUM_THREADS, Optional.of(numThreads));
        }
    };
    summarizer.importData(options, handler, new Configuration());
    String expr = "MAP_GET(DOMAIN_REMOVE_TLD(domain), s) > 0";
    for (String domain : domains) {
        Boolean b = (Boolean) StellarProcessorUtils.run(expr, ImmutableMap.of("s", finalObj.get(), "domain", domain));
        Assert.assertTrue("Can't find " + domain, b);
    }
}

Example 7 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class StixExtractorTest method testStixAddresses.

public void testStixAddresses(final String stixDoc) throws Exception {
    Thread t1 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV4);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(3, Iterables.size(results));
            Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator);
            Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator);
            Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator);
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    Thread t2 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfig);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(3, Iterables.size(results));
            Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator);
            Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator);
            Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator);
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    Thread t3 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV6);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(0, Iterables.size(results));
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    t1.run();
    t2.run();
    t3.run();
    t1.join();
    t2.join();
    t3.join();
}

Example 8 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class SimpleFlatFileSummarizerTest method testWholeFile.

public void testWholeFile(final int numThreads) throws IOException, InvalidWriterOutput {
    ExtractorHandler handler = ExtractorHandler.load(stellarExtractorConfigWholeFile);
    LocalSummarizer summarizer = new MockSummarizer(new HashMap<String, String>() {

        {
            for (String domain : domains) {
                put(domain, "1," + domain);
            }
        }
    });
    final AtomicReference<Object> finalObj = new AtomicReference<>(null);
    EnumMap<SummarizeOptions, Optional<Object>> options = new EnumMap<SummarizeOptions, Optional<Object>>(SummarizeOptions.class) {

        {
            put(SummarizeOptions.INPUT, Optional.of("."));
            put(SummarizeOptions.BATCH_SIZE, Optional.of(5));
            put(SummarizeOptions.QUIET, Optional.of(true));
            put(SummarizeOptions.OUTPUT_MODE, Optional.of(new PeekingWriter(finalObj)));
            put(SummarizeOptions.OUTPUT, Optional.of("out"));
            put(SummarizeOptions.NUM_THREADS, Optional.of(numThreads));
        }
    };
    summarizer.importData(options, handler, new Configuration());
    String expr = "MAP_GET(DOMAIN_REMOVE_TLD(domain), s) > 0";
    for (String domain : domains) {
        Boolean b = (Boolean) StellarProcessorUtils.run(expr, ImmutableMap.of("s", finalObj.get(), "domain", domain));
        Assert.assertTrue("Can't find " + domain, b);
    }
}