Examples with LookupKV org.apache.metron.enrichment.lookup.LookupKV used on opensource projects

Example 26 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class TransformFilterExtractorDecoratorTest method filters_indicators.

@Test
public void filters_indicators() throws Exception {
    EnrichmentKey lookupKey = new EnrichmentKey("testenrichment", "");
    EnrichmentValue lookupValue = new EnrichmentValue(new HashMap<String, Object>() {

        {
            put("foo", "val1");
            put("bar", "");
            put("baz", "val3");
        }
    });
    LookupKV lkv = new LookupKV<>(lookupKey, lookupValue);
    List<LookupKV> extractedLkvs = new ArrayList<>();
    extractedLkvs.add(lkv);
    Mockito.when(extractor.extract("val1,,val3")).thenReturn(extractedLkvs);
    Iterable<LookupKV> extracted = decorator.extract("val1,,val3");
    Assert.assertThat(extracted, CoreMatchers.equalTo(new ArrayList<>()));
}

Example 27 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class StixExtractorTest method testStixAddresses.

public void testStixAddresses(final String stixDoc) throws Exception {
    Thread t1 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV4);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(3, Iterables.size(results));
            Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator);
            Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator);
            Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator);
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    Thread t2 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfig);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(3, Iterables.size(results));
            Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator);
            Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator);
            Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator);
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    Thread t3 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV6);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(0, Iterables.size(results));
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    t1.run();
    t2.run();
    t3.run();
    t1.join();
    t2.join();
    t3.join();
}

Example 28 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class BulkLoadMapperTest method testMapper.

@Test
public void testMapper() throws IOException, InterruptedException {
    final Map<ImmutableBytesWritable, Put> puts = new HashMap<>();
    BulkLoadMapper mapper = new BulkLoadMapper() {

        @Override
        protected void write(ImmutableBytesWritable key, Put value, Context context) throws IOException, InterruptedException {
            puts.put(key, value);
        }
    };
    mapper.initialize(new Configuration() {

        {
            set(BulkLoadMapper.COLUMN_FAMILY_KEY, "cf");
            set(BulkLoadMapper.CONFIG_KEY, extractorConfig);
            set(BulkLoadMapper.LAST_SEEN_KEY, "0");
            set(BulkLoadMapper.CONVERTER_KEY, EnrichmentConverter.class.getName());
        }
    });
    {
        mapper.map(null, new Text("#google.com,1,foo"), null);
        Assert.assertTrue(puts.size() == 0);
    }
    {
        mapper.map(null, new Text("google.com,1,foo"), null);
        Assert.assertTrue(puts.size() == 1);
        EnrichmentKey expectedKey = new EnrichmentKey() {

            {
                indicator = "google.com";
                type = "threat";
            }
        };
        EnrichmentConverter converter = new EnrichmentConverter();
        Put put = puts.get(new ImmutableBytesWritable(expectedKey.toBytes()));
        Assert.assertNotNull(puts);
        LookupKV<EnrichmentKey, EnrichmentValue> results = converter.fromPut(put, "cf");
        Assert.assertEquals(results.getKey().indicator, "google.com");
        Assert.assertEquals(results.getValue().getMetadata().size(), 2);
        Assert.assertEquals(results.getValue().getMetadata().get("meta"), "foo");
        Assert.assertEquals(results.getValue().getMetadata().get("host"), "google.com");
    }
}

Example 29 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testLocalWholeFile.

@Test
public void testLocalWholeFile() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + wholeFileExtractorConfigFile.getPath(), "-i " + file1.getPath() + "," + file2.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(2, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}

Example 30 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method stellar_transforms_and_filters_indicators_and_value_metadata.

@Test
public void stellar_transforms_and_filters_indicators_and_value_metadata() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + stellarExtractorConfigFile.getPath(), "-i " + multilineFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertThat(results.get(0).getKey().getIndicator(), startsWith("GOOGLE"));
    Assert.assertThat(results.get(0).getKey().type, equalTo("enrichment"));
    Assert.assertThat(results.get(0).getValue().getMetadata().size(), equalTo(3));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("meta").toString(), startsWith("foo"));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("empty").toString(), startsWith("valfromglobalconfig"));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("host").toString(), startsWith("GOOGLE"));
}