Examples with LookupKV org.apache.metron.enrichment.lookup.LookupKV used on opensource projects

Example 31 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testLocalLineByLine_zip.

@Test
public void testLocalLineByLine_zip() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + lineByLineExtractorConfigFile.getPath(), "-i " + multilineZipFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}

Example 32 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testMRLineByLine.

@Test
public void testMRLineByLine() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + lineByLineExtractorConfigFile.getPath(), "-i " + multilineFile.getName(), "-m MR", "-p 2", "-b 128", "-q" };
    FileSystem fs = FileSystem.get(config);
    HBaseUtil.INSTANCE.writeFile(new String(Files.readAllBytes(multilineFile.toPath())), new Path(multilineFile.getName()), fs);
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}

Example 33 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testLocalLineByLine_gz.

@Test
public void testLocalLineByLine_gz() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + lineByLineExtractorConfigFile.getPath(), "-i " + multilineGzFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}