Examples with EnrichmentConverter org.apache.metron.enrichment.converter.EnrichmentConverter used on opensource projects

Example 11 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class LeastRecentlyUsedPrunerIntegrationTest method test.

@Test
public void test() throws Exception {
    long ts = System.currentTimeMillis();
    BloomAccessTracker bat = new BloomAccessTracker("tracker1", 100, 0.03);
    PersistentAccessTracker pat = new PersistentAccessTracker(tableName, "0", atTable, atCF, bat, 0L);
    EnrichmentLookup lookup = new EnrichmentLookup(testTable, cf, pat);
    List<LookupKey> goodKeysHalf = getKeys(0, 5);
    List<LookupKey> goodKeysOtherHalf = getKeys(5, 10);
    Iterable<LookupKey> goodKeys = Iterables.concat(goodKeysHalf, goodKeysOtherHalf);
    List<LookupKey> badKey = getKeys(10, 11);
    EnrichmentConverter converter = new EnrichmentConverter();
    for (LookupKey k : goodKeysHalf) {
        testTable.put(converter.toPut(cf, (EnrichmentKey) k, new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    pat.persist(true);
    for (LookupKey k : goodKeysOtherHalf) {
        testTable.put(converter.toPut(cf, (EnrichmentKey) k, new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    testUtil.flush();
    Assert.assertFalse(lookup.getAccessTracker().hasSeen(goodKeysHalf.get(0)));
    for (LookupKey k : goodKeysOtherHalf) {
        Assert.assertTrue(lookup.getAccessTracker().hasSeen(k));
    }
    pat.persist(true);
    {
        testTable.put(converter.toPut(cf, (EnrichmentKey) badKey.get(0), new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("k", "dummy");
            }
        })));
    }
    testUtil.flush();
    Assert.assertFalse(lookup.getAccessTracker().hasSeen(badKey.get(0)));
    Job job = LeastRecentlyUsedPruner.createJob(config, tableName, cf, atTableName, atCF, ts);
    Assert.assertTrue(job.waitForCompletion(true));
    for (LookupKey k : goodKeys) {
        Assert.assertTrue(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
    for (LookupKey k : badKey) {
        Assert.assertFalse(lookup.exists((EnrichmentKey) k, new EnrichmentLookup.HBaseContext(testTable, cf), true));
    }
}

Example 12 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testLocalWholeFile.

@Test
public void testLocalWholeFile() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + wholeFileExtractorConfigFile.getPath(), "-i " + file1.getPath() + "," + file2.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(2, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}

Example 13 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method stellar_transforms_and_filters_indicators_and_value_metadata.

@Test
public void stellar_transforms_and_filters_indicators_and_value_metadata() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + stellarExtractorConfigFile.getPath(), "-i " + multilineFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertThat(results.get(0).getKey().getIndicator(), startsWith("GOOGLE"));
    Assert.assertThat(results.get(0).getKey().type, equalTo("enrichment"));
    Assert.assertThat(results.get(0).getValue().getMetadata().size(), equalTo(3));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("meta").toString(), startsWith("foo"));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("empty").toString(), startsWith("valfromglobalconfig"));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("host").toString(), startsWith("GOOGLE"));
}

Example 14 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testLocalLineByLine_zip.

@Test
public void testLocalLineByLine_zip() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + lineByLineExtractorConfigFile.getPath(), "-i " + multilineZipFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}

Example 15 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testMRLineByLine.

@Test
public void testMRLineByLine() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + lineByLineExtractorConfigFile.getPath(), "-i " + multilineFile.getName(), "-m MR", "-p 2", "-b 128", "-q" };
    FileSystem fs = FileSystem.get(config);
    HBaseUtil.INSTANCE.writeFile(new String(Files.readAllBytes(multilineFile.toPath())), new Path(multilineFile.getName()), fs);
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}