Examples with SensorEnrichmentConfig org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig used on opensource projects

Example 6 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class SensorEnrichmentConfigTest method test.

@Test
public void test() throws IOException {
    EqualsVerifier.forClass(SensorEnrichmentConfig.class).suppress(Warning.NONFINAL_FIELDS).usingGetClass().verify();
    Map<String, byte[]> testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
    byte[] sensorConfigBytes = testSensorConfigMap.get("yaf");
    SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(sensorConfigBytes);
    Assert.assertNotNull(sensorEnrichmentConfig);
    Assert.assertTrue(sensorEnrichmentConfig.toString() != null && sensorEnrichmentConfig.toString().length() > 0);
}

Example 7 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class SensorEnrichmentConfigTest method testSerDe.

@Test
public void testSerDe() throws IOException {
    for (File enrichmentConfig : new File(new File(TestConstants.ENRICHMENTS_CONFIGS_PATH), "enrichments").listFiles()) {
        SensorEnrichmentConfig config = null;
        try (BufferedReader br = new BufferedReader(new FileReader(enrichmentConfig))) {
            String parserStr = IOUtils.toString(br);
            config = SensorEnrichmentConfig.fromBytes(parserStr.getBytes());
        }
        SensorEnrichmentConfig config2 = SensorEnrichmentConfig.fromBytes(config.toJSON().getBytes());
        Assert.assertEquals(config2, config);
    }
}

Example 8 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class ZKConfigurationsCacheIntegrationTest method validateBaseWrite.

@Test
public void validateBaseWrite() throws Exception {
    File globalConfigFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/global.json");
    Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfigFile, JSONUtils.MAP_SUPPLIER);
    // indexing
    {
        File inFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/indexing/test.json");
        Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(inFile, JSONUtils.MAP_SUPPLIER);
        IndexingConfigurations config = cache.get(IndexingConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorIndexingConfig("test")));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
        assertEventually(() -> Assert.assertNull(config.getSensorIndexingConfig("notthere", false)));
    }
    // enrichment
    {
        File inFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/enrichments/test.json");
        SensorEnrichmentConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, SensorEnrichmentConfig.class);
        EnrichmentConfigurations config = cache.get(EnrichmentConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test")));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
        assertEventually(() -> Assert.assertNull(config.getSensorEnrichmentConfig("notthere")));
    }
    // parsers
    {
        File inFile = new File(TestConstants.PARSER_CONFIGS_PATH + "/parsers/bro.json");
        SensorParserConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, SensorParserConfig.class);
        ParserConfigurations config = cache.get(ParserConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorParserConfig("bro")));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
        assertEventually(() -> Assert.assertNull(config.getSensorParserConfig("notthere")));
    }
    // profiler
    {
        File inFile = new File(profilerDir, "/readme-example-1/profiler.json");
        ProfilerConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, ProfilerConfig.class);
        ProfilerConfigurations config = cache.get(ProfilerConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getProfilerConfig()));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
    }
}

Example 9 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class EnrichmentConfigurations method updateSensorEnrichmentConfig.

public void updateSensorEnrichmentConfig(String sensorType, InputStream io) throws IOException {
    SensorEnrichmentConfig sensorEnrichmentConfig = JSONUtils.INSTANCE.load(io, SensorEnrichmentConfig.class);
    updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfig);
}

Example 10 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class GenericEnrichmentBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    perfLog.mark("execute");
    String key = tuple.getStringByField("key");
    JSONObject rawMessage = (JSONObject) tuple.getValueByField("message");
    String subGroup = "";
    JSONObject enrichedMessage = new JSONObject();
    enrichedMessage.put("adapter." + adapter.getClass().getSimpleName().toLowerCase() + ".begin.ts", "" + System.currentTimeMillis());
    try {
        if (rawMessage == null || rawMessage.isEmpty())
            throw new Exception("Could not parse binary stream to JSON");
        if (key == null)
            throw new Exception("Key is not valid");
        String sourceType = null;
        if (rawMessage.containsKey(Constants.SENSOR_TYPE)) {
            sourceType = rawMessage.get(Constants.SENSOR_TYPE).toString();
        } else {
            throw new RuntimeException("Source type is missing from enrichment fragment: " + rawMessage.toJSONString());
        }
        String prefix = null;
        for (Object o : rawMessage.keySet()) {
            String field = (String) o;
            Object value = rawMessage.get(field);
            if (field.equals(Constants.SENSOR_TYPE)) {
                enrichedMessage.put(Constants.SENSOR_TYPE, value);
            } else {
                JSONObject enrichedField = new JSONObject();
                if (value != null) {
                    SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType);
                    if (config == null) {
                        LOG.debug("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType);
                        MetronError metronError = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withMessage("Unable to find SensorEnrichmentConfig for sourceType: " + sourceType).addRawMessage(rawMessage);
                        ErrorUtils.handleError(collector, metronError);
                        continue;
                    }
                    config.getConfiguration().putIfAbsent(STELLAR_CONTEXT_CONF, stellarContext);
                    CacheKey cacheKey = new CacheKey(field, value, config);
                    try {
                        adapter.logAccess(cacheKey);
                        prefix = adapter.getOutputPrefix(cacheKey);
                        subGroup = adapter.getStreamSubGroup(enrichmentType, field);
                        perfLog.mark("enrich");
                        enrichedField = cache.get(cacheKey);
                        perfLog.log("enrich", "key={}, time to run enrichment type={}", key, enrichmentType);
                        if (enrichedField == null)
                            throw new Exception("[Metron] Could not enrich string: " + value);
                    } catch (Exception e) {
                        LOG.error(e.getMessage(), e);
                        MetronError metronError = new MetronError().withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withThrowable(e).withErrorFields(new HashSet() {

                            {
                                add(field);
                            }
                        }).addRawMessage(rawMessage);
                        ErrorUtils.handleError(collector, metronError);
                        continue;
                    }
                }
                enrichedMessage = EnrichmentUtils.adjustKeys(enrichedMessage, enrichedField, field, prefix);
            }
        }
        enrichedMessage.put("adapter." + adapter.getClass().getSimpleName().toLowerCase() + ".end.ts", "" + System.currentTimeMillis());
        if (!enrichedMessage.isEmpty()) {
            collector.emit(enrichmentType, new Values(key, enrichedMessage, subGroup));
        }
    } catch (Exception e) {
        handleError(key, rawMessage, subGroup, enrichedMessage, e);
    }
    perfLog.log("execute", "key={}, elapsed time to run execute", key);
}