Examples with ConfigHandler org.apache.metron.common.configuration.enrichment.handler.ConfigHandler used on opensource projects

Example 11 with ConfigHandler

use of org.apache.metron.common.configuration.enrichment.handler.ConfigHandler in project metron by apache.

the class StellarEnrichmentConfigTest method testSplitter_listWithTemporaryVariables.

@Test
public void testSplitter_listWithTemporaryVariables() throws IOException {
    JSONObject message = new JSONObject(ImmutableMap.of("domain_without_subdomains", "yahoo.com"));
    EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(conf, EnrichmentConfig.class);
    Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
    ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
    List<JSONObject> splits = Configs.STELLAR.splitByFields(message, null, x -> null, handler);
    Assert.assertEquals(1, splits.size());
    Map<String, Object> split = (Map<String, Object>) (splits.get(0)).get("");
    Assert.assertEquals("yahoo.com", split.get("domain_without_subdomains"));
    Assert.assertTrue(split.containsKey("dga_result"));
    Assert.assertTrue(split.containsKey("dga_model_endpoint"));
    Assert.assertTrue(split.containsKey("dga_result_map"));
}

Example 12 with ConfigHandler

use of org.apache.metron.common.configuration.enrichment.handler.ConfigHandler in project metron by apache.

the class StellarEnrichmentConfigTest method testSplitter_mixed.

@Test
public void testSplitter_mixed() throws IOException {
    JSONObject message = getMessage();
    for (String c : Iterables.concat(MIXED_CONFIGS, ImmutableList.of(tempVarStellarConfig_list))) {
        EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(c, EnrichmentConfig.class);
        Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
        ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
        List<JSONObject> splits = Configs.STELLAR.splitByFields(message, null, x -> null, handler);
        Assert.assertEquals(3, splits.size());
        {
            Map<String, Object> split = (Map<String, Object>) splits.get(0).get("group1");
            Assert.assertEquals(2, split.size());
            Assert.assertEquals("stellar_test", split.get("source.type"));
            Assert.assertNull(split.get("stmt1"));
        }
        {
            Map<String, Object> split = (Map<String, Object>) splits.get(1).get("group2");
            Assert.assertEquals(1, split.size());
            Assert.assertEquals("foo", split.get("string"));
        }
        {
            Map<String, Object> split = (Map<String, Object>) splits.get(2).get("");
            Assert.assertEquals(1, split.size());
            Assert.assertEquals("stellar_test", split.get("source.type"));
        }
    }
}

Example 13 with ConfigHandler

use of org.apache.metron.common.configuration.enrichment.handler.ConfigHandler in project metron by apache.

the class StellarEnrichmentConfigTest method testGetSubgroups_mixed.

@Test
public void testGetSubgroups_mixed() throws IOException {
    for (String c : MIXED_CONFIGS) {
        EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(c, EnrichmentConfig.class);
        Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
        ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
        List<String> subgroups = Configs.STELLAR.getSubgroups(handler);
        Assert.assertEquals("group1", subgroups.get(0));
        Assert.assertEquals("group2", subgroups.get(1));
        Assert.assertEquals("", subgroups.get(2));
        Assert.assertEquals(3, subgroups.size());
    }
}

Example 14 with ConfigHandler

use of org.apache.metron.common.configuration.enrichment.handler.ConfigHandler in project metron by apache.

the class EnrichmentJoinBolt method getStreamIds.

@Override
public Set<String> getStreamIds(JSONObject message) {
    Set<String> streamIds = new HashSet<>();
    String sourceType = MessageUtils.getSensorType(message);
    if (sourceType == null) {
        String errorMessage = "Unable to find source type for message: " + message;
        throw new IllegalStateException(errorMessage);
    }
    Map<String, Object> fieldMap = getFieldMap(sourceType);
    Map<String, ConfigHandler> handlerMap = getFieldToHandlerMap(sourceType);
    if (fieldMap != null) {
        for (String enrichmentType : fieldMap.keySet()) {
            ConfigHandler handler = handlerMap.get(enrichmentType);
            List<String> subgroups = handler.getType().getSubgroups(handler.getType().toConfig(handler.getConfig()));
            for (String subgroup : subgroups) {
                streamIds.add(Joiner.on(":").join(enrichmentType, subgroup));
            }
        }
    }
    streamIds.add("message:");
    return streamIds;
}

Example 15 with ConfigHandler

use of org.apache.metron.common.configuration.enrichment.handler.ConfigHandler in project metron by apache.

the class EnrichmentSplitterBolt method splitMessage.

@SuppressWarnings("unchecked")
@Override
public Map<String, List<JSONObject>> splitMessage(JSONObject message) {
    Map<String, List<JSONObject>> streamMessageMap = new HashMap<>();
    String sensorType = MessageUtils.getSensorType(message);
    Map<String, Object> enrichmentFieldMap = getFieldMap(sensorType);
    Map<String, ConfigHandler> fieldToHandler = getFieldToHandlerMap(sensorType);
    Set<String> enrichmentTypes = new HashSet<>(enrichmentFieldMap.keySet());
    enrichmentTypes.addAll(fieldToHandler.keySet());
    for (String enrichmentType : enrichmentTypes) {
        Object fields = enrichmentFieldMap.get(enrichmentType);
        ConfigHandler retriever = fieldToHandler.get(enrichmentType);
        List<JSONObject> enrichmentObject = retriever.getType().splitByFields(message, fields, field -> getKeyName(enrichmentType, field), retriever);
        for (JSONObject eo : enrichmentObject) {
            eo.put(Constants.SENSOR_TYPE, sensorType);
        }
        streamMessageMap.put(enrichmentType, enrichmentObject);
    }
    message.put(getClass().getSimpleName().toLowerCase() + ".splitter.end.ts", "" + System.currentTimeMillis());
    return streamMessageMap;
}