Examples with BloomAccessTracker org.apache.metron.enrichment.lookup.accesstracker.BloomAccessTracker used on opensource projects

Example 1 with BloomAccessTracker

use of org.apache.metron.enrichment.lookup.accesstracker.BloomAccessTracker in project metron by apache.

the class ThreatIntelAdapter method initializeAdapter.

@Override
public boolean initializeAdapter(Map<String, Object> configuration) {
    PersistentAccessTracker accessTracker;
    String hbaseTable = config.getHBaseTable();
    int expectedInsertions = config.getExpectedInsertions();
    double falsePositives = config.getFalsePositiveRate();
    String trackerHBaseTable = config.getTrackerHBaseTable();
    String trackerHBaseCF = config.getTrackerHBaseCF();
    long millisecondsBetweenPersist = config.getMillisecondsBetweenPersists();
    BloomAccessTracker bat = new BloomAccessTracker(hbaseTable, expectedInsertions, falsePositives);
    Configuration hbaseConfig = HBaseConfiguration.create();
    try {
        accessTracker = new PersistentAccessTracker(hbaseTable, UUID.randomUUID().toString(), config.getProvider().getTable(hbaseConfig, trackerHBaseTable), trackerHBaseCF, bat, millisecondsBetweenPersist);
        lookup = new EnrichmentLookup(config.getProvider().getTable(hbaseConfig, hbaseTable), config.getHBaseCF(), accessTracker);
    } catch (IOException e) {
        LOG.error("Unable to initialize ThreatIntelAdapter", e);
        return false;
    }
    return true;
}

Example 2 with BloomAccessTracker

use of org.apache.metron.enrichment.lookup.accesstracker.BloomAccessTracker in project metron by apache.

the class ThreatIntelAdapterTest method setup.

@Before
public void setup() throws Exception {
    final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(atTableName, cf);
    final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf);
    EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
        }
    });
    BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03);
    PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L);
    lookup = new EnrichmentLookup(threatIntelTable, cf, pat);
    JSONParser jsonParser = new JSONParser();
    expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
}

Example 3 with BloomAccessTracker

use of org.apache.metron.enrichment.lookup.accesstracker.BloomAccessTracker in project metron by apache.

the class SimpleHBaseAdapterTest method setup.

@Before
public void setup() throws Exception {
    final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(atTableName, cf);
    final MockHTable hbaseTable = (MockHTable) MockHBaseTableProvider.addToCache(hbaseTableName, cf);
    EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3"), new EnrichmentValue(PLAYFUL_ENRICHMENT)));
        }
    });
    EnrichmentHelper.INSTANCE.load(hbaseTable, cf1, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(CF1_CLASSIFICATION_TYPE, "10.0.2.4"), new EnrichmentValue(CF1_ENRICHMENT)));
        }
    });
    BloomAccessTracker bat = new BloomAccessTracker(hbaseTableName, 100, 0.03);
    PersistentAccessTracker pat = new PersistentAccessTracker(hbaseTableName, "0", trackerTable, cf, bat, 0L);
    lookup = new EnrichmentLookup(hbaseTable, cf, pat);
    JSONParser jsonParser = new JSONParser();
    expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
}

Example 4 with BloomAccessTracker

use of org.apache.metron.enrichment.lookup.accesstracker.BloomAccessTracker in project metron by apache.

the class LeastRecentlyUsedPrunerIntegrationTest method test.

@Test
public void test() throws Exception {
    long ts = System.currentTimeMillis();
    BloomAccessTracker bat = new BloomAccessTracker("tracker1", 100, 0.03);
    PersistentAccessTracker pat = new PersistentAccessTracker(tableName, "0", atTable, atCF, bat, 0L);
    EnrichmentLookup lookup = new EnrichmentLookup(testTable, cf, pat);
    List<LookupKey> goodKeysHalf = getKeys(0, 5);
    List<LookupKey> goodKeysOtherHalf = getKeys(5, 10);
    Iterable<LookupKey> goodKeys = Iterables.concat(goodKeysHalf, goodKeysOtherHalf);
    List<LookupKey> badKey = getKeys(10, 11);
    EnrichmentConverter converter = new EnrichmentConverter();
    for (LookupKey k : goodKeysHalf) {
        testTable.put(converter.toPut(cf, (EnrichmentKey) k, new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    pat.persist(true);
    for (LookupKey k : goodKeysOtherHalf) {
        testTable.put(converter.toPut(cf, (EnrichmentKey) k, new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    testUtil.flush();
    Assert.assertFalse(lookup.getAccessTracker().hasSeen(goodKeysHalf.get(0)));
    for (LookupKey k : goodKeysOtherHalf) {
        Assert.assertTrue(lookup.getAccessTracker().hasSeen(k));
    }
    pat.persist(true);
    {
        testTable.put(converter.toPut(cf, (EnrichmentKey) badKey.get(0), new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
    }
    testUtil.flush();
    Assert.assertFalse(lookup.getAccessTracker().hasSeen(badKey.get(0)));
    Job job = LeastRecentlyUsedPruner.createJob(config, tableName, cf, atTableName, atCF, ts);
    Assert.assertTrue(job.waitForCompletion(true));
    for (LookupKey k : goodKeys) {
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    for (LookupKey k : badKey) {
        Assert.assertFalse(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
}