Examples with EnrichmentKey org.apache.metron.enrichment.converter.EnrichmentKey used on opensource projects

Example 21 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class HBaseEnrichmentConverterTest method testKeySerialization.

@Test
public void testKeySerialization() {
    byte[] serialized = key.toBytes();
    EnrichmentKey deserialized = new EnrichmentKey();
    deserialized.fromBytes(serialized);
    Assert.assertEquals(key, deserialized);
}

Example 22 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class HBaseEnrichmentConverterTest method testPut.

@Test
public void testPut() throws IOException {
    HbaseConverter<EnrichmentKey, EnrichmentValue> converter = new EnrichmentConverter();
    Put put = converter.toPut("cf", key, value);
    LookupKV<EnrichmentKey, EnrichmentValue> converted = converter.fromPut(put, "cf");
    Assert.assertEquals(results, converted);
}

Example 23 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class BulkLoadMapperTest method testMapper.

@Test
public void testMapper() throws IOException, InterruptedException {
    final Map<ImmutableBytesWritable, Put> puts = new HashMap<>();
    BulkLoadMapper mapper = new BulkLoadMapper() {

        @Override
        protected void write(ImmutableBytesWritable key, Put value, Context context) throws IOException, InterruptedException {
            puts.put(key, value);
        }
    };
    mapper.initialize(new Configuration() {

        {
            set(BulkLoadMapper.COLUMN_FAMILY_KEY, "cf");
            set(BulkLoadMapper.CONFIG_KEY, extractorConfig);
            set(BulkLoadMapper.LAST_SEEN_KEY, "0");
            set(BulkLoadMapper.CONVERTER_KEY, EnrichmentConverter.class.getName());
        }
    });
    {
        mapper.map(null, new Text("#google.com,1,foo"), null);
        Assert.assertTrue(puts.size() == 0);
    }
    {
        mapper.map(null, new Text("google.com,1,foo"), null);
        Assert.assertTrue(puts.size() == 1);
        EnrichmentKey expectedKey = new EnrichmentKey() {

            {
                indicator = "google.com";
                type = "threat";
            }
        };
        EnrichmentConverter converter = new EnrichmentConverter();
        Put put = puts.get(new ImmutableBytesWritable(expectedKey.toBytes()));
        Assert.assertNotNull(puts);
        LookupKV<EnrichmentKey, EnrichmentValue> results = converter.fromPut(put, "cf");
        Assert.assertEquals(results.getKey().indicator, "google.com");
        Assert.assertEquals(results.getValue().getMetadata().size(), 2);
        Assert.assertEquals(results.getValue().getMetadata().get("meta"), "foo");
        Assert.assertEquals(results.getValue().getMetadata().get("host"), "google.com");
    }
}

Example 24 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class LeastRecentlyUsedPrunerIntegrationTest method test.

@Test
public void test() throws Exception {
    long ts = System.currentTimeMillis();
    BloomAccessTracker bat = new BloomAccessTracker("tracker1", 100, 0.03);
    PersistentAccessTracker pat = new PersistentAccessTracker(tableName, "0", atTable, atCF, bat, 0L);
    EnrichmentLookup lookup = new EnrichmentLookup(testTable, cf, pat);
    List<LookupKey> goodKeysHalf = getKeys(0, 5);
    List<LookupKey> goodKeysOtherHalf = getKeys(5, 10);
    Iterable<LookupKey> goodKeys = Iterables.concat(goodKeysHalf, goodKeysOtherHalf);
    List<LookupKey> badKey = getKeys(10, 11);
    EnrichmentConverter converter = new EnrichmentConverter();
    for (LookupKey k : goodKeysHalf) {
        testTable.put(converter.toPut(cf, (EnrichmentKey) k, new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    pat.persist(true);
    for (LookupKey k : goodKeysOtherHalf) {
        testTable.put(converter.toPut(cf, (EnrichmentKey) k, new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    testUtil.flush();
    Assert.assertFalse(lookup.getAccessTracker().hasSeen(goodKeysHalf.get(0)));
    for (LookupKey k : goodKeysOtherHalf) {
        Assert.assertTrue(lookup.getAccessTracker().hasSeen(k));
    }
    pat.persist(true);
    {
        testTable.put(converter.toPut(cf, (EnrichmentKey) badKey.get(0), new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
    }
    testUtil.flush();
    Assert.assertFalse(lookup.getAccessTracker().hasSeen(badKey.get(0)));
    Job job = LeastRecentlyUsedPruner.createJob(config, tableName, cf, atTableName, atCF, ts);
    Assert.assertTrue(job.waitForCompletion(true));
    for (LookupKey k : goodKeys) {
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    for (LookupKey k : badKey) {
        Assert.assertFalse(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
}

Example 25 with EnrichmentKey

use of org.apache.metron.enrichment.converter.EnrichmentKey in project metron by apache.

the class TaxiiIntegrationTest method getIndicators.

private static Set<String> getIndicators(String indicatorType, Iterable<Put> puts, String cf) throws IOException {
    EnrichmentConverter converter = new EnrichmentConverter();
    Set<String> ret = new HashSet<>();
    for (Put p : puts) {
        LookupKV<EnrichmentKey, EnrichmentValue> kv = converter.fromPut(p, cf);
        if (kv.getKey().type.equals(indicatorType)) {
            ret.add(kv.getKey().indicator);
        }
    }
    return ret;
}