Examples with EnrichmentConverter org.apache.metron.enrichment.converter.EnrichmentConverter used on opensource projects

Example 1 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class LocalImporter method createState.

@Override
protected ThreadLocal<HBaseExtractorState> createState(EnumMap<LoadOptions, Optional<Object>> config, Configuration hadoopConfig, final ExtractorHandler handler) {
    ThreadLocal<HBaseExtractorState> state = new ThreadLocal<HBaseExtractorState>() {

        @Override
        protected HBaseExtractorState initialValue() {
            try {
                String cf = (String) config.get(LoadOptions.HBASE_CF).get();
                HTableInterface table = provider.retrieve().getTable(hadoopConfig, (String) config.get(LoadOptions.HBASE_TABLE).get());
                return new HBaseExtractorState(table, cf, handler.getExtractor(), new EnrichmentConverter(), hadoopConfig);
            } catch (IOException e1) {
                throw new IllegalStateException("Unable to get table: " + e1);
            }
        }
    };
    return state;
}

Example 2 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method testLocalLineByLine.

@Test
public void testLocalLineByLine() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + lineByLineExtractorConfigFile.getPath(), "-i " + multilineFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertTrue(results.get(0).getKey().indicator.startsWith("google"));
    Assert.assertEquals(results.get(0).getKey().type, "enrichment");
    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("meta").toString().startsWith("foo"));
    Assert.assertTrue(results.get(0).getValue().getMetadata().get("host").toString().startsWith("google"));
}

Example 3 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleEnrichmentFlatFileLoaderIntegrationTest method custom_extractor_transforms_and_filters_indicators_and_value_metadata.

@Test
public void custom_extractor_transforms_and_filters_indicators_and_value_metadata() throws Exception {
    String[] argv = { "-c cf", "-t enrichment", "-e " + customLineByLineExtractorConfigFile.getPath(), "-i " + multilineFile.getPath(), "-p 2", "-b 128", "-q" };
    SimpleEnrichmentFlatFileLoader.main(config, argv);
    EnrichmentConverter converter = new EnrichmentConverter();
    ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
    List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
    for (Result r : scanner) {
        results.add(converter.fromResult(r, cf));
        testTable.delete(new Delete(r.getRow()));
    }
    Assert.assertEquals(NUM_LINES, results.size());
    Assert.assertThat(results.get(0).getKey().getIndicator(), startsWith("GOOGLE"));
    Assert.assertThat(results.get(0).getKey().type, equalTo("enrichment"));
    Assert.assertThat(results.get(0).getValue().getMetadata().size(), equalTo(2));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("meta").toString(), startsWith("foo"));
    Assert.assertThat(results.get(0).getValue().getMetadata().get("host").toString(), startsWith("GOOGLE"));
}

Example 4 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleHBaseEnrichmentWriterTest method getValues.

public static List<LookupKV<EnrichmentKey, EnrichmentValue>> getValues() throws IOException {
    MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(TABLE_NAME);
    Assert.assertNotNull(table);
    List<LookupKV<EnrichmentKey, EnrichmentValue>> ret = new ArrayList<>();
    EnrichmentConverter converter = new EnrichmentConverter();
    for (Result r : table.getScanner(Bytes.toBytes(TABLE_CF))) {
        ret.add(converter.fromResult(r, TABLE_CF));
    }
    return ret;
}

Example 5 with EnrichmentConverter

use of org.apache.metron.enrichment.converter.EnrichmentConverter in project metron by apache.

the class SimpleHbaseEnrichmentWriterIntegrationTest method test.

@Test
public void test() throws UnableToStartException, IOException {
    final String sensorType = "dummy";
    final List<byte[]> inputMessages = new ArrayList<byte[]>() {

        {
            add(Bytes.toBytes("col11,col12,col13"));
            add(Bytes.toBytes("col21,col22,col23"));
            add(Bytes.toBytes("col31,col32,col33"));
        }
    };
    MockHBaseTableProvider.addToCache(sensorType, "cf");
    final Properties topologyProperties = new Properties();
    final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
    final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {

        {
            add(new KafkaComponent.Topic(sensorType, 1));
        }
    });
    topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH).withParserSensorConfig(sensorType, JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class));
    ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder().withSensorType(sensorType).withTopologyProperties(topologyProperties).withBrokerUrl(kafkaComponent.getBrokerList()).build();
    // UnitTestHelper.verboseLogging();
    ComponentRunner runner = new ComponentRunner.Builder().withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("org/apache/storm", parserTopologyComponent).withMillisecondsBetweenAttempts(5000).withCustomShutdownOrder(new String[] { "org/apache/storm", "config", "kafka", "zk" }).withNumRetries(10).build();
    try {
        runner.start();
        kafkaComponent.writeMessages(sensorType, inputMessages);
        ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> result = runner.process(new Processor<List<LookupKV<EnrichmentKey, EnrichmentValue>>>() {

            List<LookupKV<EnrichmentKey, EnrichmentValue>> messages = null;

            @Override
            public ReadinessState process(ComponentRunner runner) {
                MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(sensorType);
                if (table != null && table.size() == inputMessages.size()) {
                    EnrichmentConverter converter = new EnrichmentConverter();
                    messages = new ArrayList<>();
                    try {
                        for (Result r : table.getScanner(Bytes.toBytes("cf"))) {
                            messages.add(converter.fromResult(r, "cf"));
                        }
                    } catch (IOException e) {
                    }
                    return ReadinessState.READY;
                }
                return ReadinessState.NOT_READY;
            }

            @Override
            public ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> getResult() {
                ProcessorResult.Builder<List<LookupKV<EnrichmentKey, EnrichmentValue>>> builder = new ProcessorResult.Builder();
                return builder.withResult(messages).build();
            }
        });
        Set<String> validIndicators = new HashSet<>(ImmutableList.of("col12", "col22", "col32"));
        Map<String, Map<String, String>> validMetadata = new HashMap<String, Map<String, String>>() {

            {
                put("col12", new HashMap<String, String>() {

                    {
                        put("col1", "col11");
                        put("col3", "col13");
                    }
                });
                put("col22", new HashMap<String, String>() {

                    {
                        put("col1", "col21");
                        put("col3", "col23");
                    }
                });
                put("col32", new HashMap<String, String>() {

                    {
                        put("col1", "col31");
                        put("col3", "col33");
                    }
                });
            }
        };
        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : result.getResult()) {
            Assert.assertTrue(validIndicators.contains(kv.getKey().indicator));
            Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy");
            Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp"));
            Assert.assertNotNull(kv.getValue().getMetadata().get("original_string"));
            Map<String, String> metadata = validMetadata.get(kv.getKey().indicator);
            for (Map.Entry<String, String> x : metadata.entrySet()) {
                Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
            }
            Assert.assertEquals(metadata.size() + 4, kv.getValue().getMetadata().size());
        }
    } finally {
        if (runner != null) {
            runner.stop();
        }
    }
}