Examples with LookupKV org.apache.metron.enrichment.lookup.LookupKV used on opensource projects

Example 21 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleHBaseAdapterTest method setup.

@Before
public void setup() throws Exception {
    final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(atTableName, cf);
    final MockHTable hbaseTable = (MockHTable) MockHBaseTableProvider.addToCache(hbaseTableName, cf);
    EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3"), new EnrichmentValue(PLAYFUL_ENRICHMENT)));
        }
    });
    EnrichmentHelper.INSTANCE.load(hbaseTable, cf1, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(CF1_CLASSIFICATION_TYPE, "10.0.2.4"), new EnrichmentValue(CF1_ENRICHMENT)));
        }
    });
    BloomAccessTracker bat = new BloomAccessTracker(hbaseTableName, 100, 0.03);
    PersistentAccessTracker pat = new PersistentAccessTracker(hbaseTableName, "0", trackerTable, cf, bat, 0L);
    lookup = new EnrichmentLookup(hbaseTable, cf, pat);
    JSONParser jsonParser = new JSONParser();
    expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
}

Example 22 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class EnrichmentIntegrationTest method test.

@Test
public void test() throws Exception {
    final String cf = "cf";
    final String trackerHBaseTableName = "tracker";
    final String threatIntelTableName = "threat_intel";
    final String enrichmentsTableName = "enrichments";
    final Properties topologyProperties = new Properties() {

        {
            setProperty("enrichment_workers", "1");
            setProperty("enrichment_acker_executors", "0");
            setProperty("enrichment_topology_worker_childopts", "");
            setProperty("topology_auto_credentials", "[]");
            setProperty("enrichment_topology_max_spout_pending", "");
            setProperty("enrichment_kafka_start", "UNCOMMITTED_EARLIEST");
            setProperty("kafka_security_protocol", "PLAINTEXT");
            setProperty("enrichment_input_topic", Constants.ENRICHMENT_TOPIC);
            setProperty("enrichment_output_topic", Constants.INDEXING_TOPIC);
            setProperty("enrichment_error_topic", ERROR_TOPIC);
            setProperty("threatintel_error_topic", ERROR_TOPIC);
            setProperty("enrichment_join_cache_size", "1000");
            setProperty("threatintel_join_cache_size", "1000");
            setProperty("enrichment_hbase_provider_impl", "" + MockHBaseTableProvider.class.getName());
            setProperty("enrichment_hbase_table", enrichmentsTableName);
            setProperty("enrichment_hbase_cf", cf);
            setProperty("enrichment_host_known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
            setProperty("threatintel_hbase_table", threatIntelTableName);
            setProperty("threatintel_hbase_cf", cf);
            setProperty("enrichment_kafka_spout_parallelism", "1");
            setProperty("enrichment_split_parallelism", "1");
            setProperty("enrichment_stellar_parallelism", "1");
            setProperty("enrichment_join_parallelism", "1");
            setProperty("threat_intel_split_parallelism", "1");
            setProperty("threat_intel_stellar_parallelism", "1");
            setProperty("threat_intel_join_parallelism", "1");
            setProperty("kafka_writer_parallelism", "1");
        }
    };
    final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
    final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {

        {
            add(new KafkaComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
            add(new KafkaComponent.Topic(Constants.INDEXING_TOPIC, 1));
            add(new KafkaComponent.Topic(ERROR_TOPIC, 1));
        }
    });
    String globalConfigStr = null;
    {
        File globalConfig = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), "global.json");
        Map<String, Object> config = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
        config.put(SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF, MockHBaseTableProvider.class.getName());
        config.put(SimpleHBaseEnrichmentFunctions.ACCESS_TRACKER_TYPE_CONF, "PERSISTENT_BLOOM");
        config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_TABLE, trackerHBaseTableName);
        config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_CF, cf);
        config.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
        globalConfigStr = JSONUtils.INSTANCE.toJSON(config, true);
    }
    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfig(globalConfigStr).withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
    // create MockHBaseTables
    final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(trackerHBaseTableName, cf);
    final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf);
    EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
        }
    });
    final MockHTable enrichmentTable = (MockHTable) MockHBaseTableProvider.addToCache(enrichmentsTableName, cf);
    EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3"), new EnrichmentValue(PLAYFUL_ENRICHMENT)));
        }
    });
    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder().withTopologyLocation(new File(fluxPath())).withTopologyName("test").withTemplateLocation(new File(templatePath)).withTopologyProperties(topologyProperties).build();
    // UnitTestHelper.verboseLogging();
    ComponentRunner runner = new ComponentRunner.Builder().withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("storm", fluxComponent).withMillisecondsBetweenAttempts(15000).withCustomShutdownOrder(new String[] { "storm", "config", "kafka", "zk" }).withNumRetries(10).build();
    try {
        runner.start();
        fluxComponent.submitTopology();
        kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
        ProcessorResult<Map<String, List<Map<String, Object>>>> result = runner.process(getProcessor());
        Map<String, List<Map<String, Object>>> outputMessages = result.getResult();
        List<Map<String, Object>> docs = outputMessages.get(Constants.INDEXING_TOPIC);
        Assert.assertEquals(inputMessages.size(), docs.size());
        validateAll(docs);
        List<Map<String, Object>> errors = outputMessages.get(ERROR_TOPIC);
        Assert.assertEquals(inputMessages.size(), errors.size());
        validateErrors(errors);
    } finally {
        runner.stop();
    }
}

Example 23 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class SimpleHBaseEnrichmentFunctionsTest method setup.

@Before
public void setup() throws Exception {
    final MockHTable hbaseTable = (MockHTable) MockHBaseTableProvider.addToCache(hbaseTableName, cf);
    EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            for (int i = 0; i < 5; ++i) {
                add(new LookupKV<>(new EnrichmentKey(ENRICHMENT_TYPE, "indicator" + i), new EnrichmentValue(ImmutableMap.of("key" + i, "value" + i))));
            }
        }
    });
    context = new Context.Builder().with(Context.Capabilities.GLOBAL_CONFIG, () -> ImmutableMap.of(SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF, MockHBaseTableProvider.class.getName())).build();
}

Example 24 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class CSVExtractor method extract.

@Override
public Iterable<LookupKV> extract(String line) throws IOException {
    if (ignore(line)) {
        return Collections.emptyList();
    }
    String[] tokens = parser.parseLine(line);
    LookupKey key = converter.toKey(getType(tokens), tokens[indicatorColumn]);
    Map<String, Object> values = new HashMap<>();
    for (Map.Entry<String, Integer> kv : columnMap.entrySet()) {
        values.put(kv.getKey(), tokens[kv.getValue()]);
    }
    return Arrays.asList(new LookupKV(key, converter.toValue(values)));
}

Example 25 with LookupKV

use of org.apache.metron.enrichment.lookup.LookupKV in project metron by apache.

the class HostnameHandler method extract.

@Override
public Iterable<LookupKV> extract(final Hostname type, Map<String, Object> config) throws IOException {
    StringObjectPropertyType value = type.getHostnameValue();
    String typeStr = getType();
    if (config != null) {
        Object o = config.get(TYPE_CONFIG);
        if (o != null) {
            typeStr = o.toString();
        }
    }
    List<LookupKV> ret = new ArrayList<>();
    for (String token : StixExtractor.split(value)) {
        final String indicatorType = typeStr;
        LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token), new EnrichmentValue(new HashMap<String, Object>() {

            {
                put("source-type", "STIX");
                put("indicator-type", indicatorType);
                put("source", type.toXMLString());
            }
        }));
        ret.add(results);
    }
    return ret;
}