Examples with SensorEnrichmentConfig org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig used on opensource projects

Example 16 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class ParallelEnricherTest method testGoodConfig.

@Test
public void testGoodConfig() throws Exception {
    SensorEnrichmentConfig config = JSONUtils.INSTANCE.load(goodConfig, SensorEnrichmentConfig.class);
    config.getConfiguration().putIfAbsent("stellarContext", stellarContext);
    JSONObject message = new JSONObject() {

        {
            put(Constants.SENSOR_TYPE, "test");
        }
    };
    ParallelEnricher.EnrichmentResult result = enricher.apply(message, EnrichmentStrategies.ENRICHMENT, config, null);
    JSONObject ret = result.getResult();
    Assert.assertEquals("Got the wrong result count: " + ret, 8, ret.size());
    Assert.assertEquals(1, ret.get("map.blah"));
    Assert.assertEquals("test", ret.get("source.type"));
    Assert.assertEquals(1, ret.get("one"));
    Assert.assertEquals(2, ret.get("foo"));
    Assert.assertEquals("TEST", ret.get("ALL_CAPS"));
    Assert.assertEquals(0, result.getEnrichmentErrors().size());
}

Example 17 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class SensorEnrichmentConfigServiceImplTest method findOneShouldProperlyReturnSensorEnrichmentConfig.

@Test
public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception {
    final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig();
    EnrichmentConfigurations configs = new EnrichmentConfigurations() {

        @Override
        public Map<String, Object> getConfigurations() {
            return ImmutableMap.of(EnrichmentConfigurations.getKey("bro"), sensorEnrichmentConfig);
        }
    };
    when(cache.get(eq(EnrichmentConfigurations.class))).thenReturn(configs);
    // We only have bro, so we should expect it to be returned
    assertEquals(getTestSensorEnrichmentConfig(), sensorEnrichmentConfigService.findOne("bro"));
    // and blah should be a miss.
    assertNull(sensorEnrichmentConfigService.findOne("blah"));
}

Example 18 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class ConfiguredEnrichmentBoltTest method test.

@Test
public void test() throws Exception {
    EnrichmentConfigurations sampleConfigurations = new EnrichmentConfigurations();
    UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.FATAL);
    try {
        StandAloneConfiguredEnrichmentBolt configuredBolt = new StandAloneConfiguredEnrichmentBolt(null);
        configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
        Assert.fail("A valid zookeeper url must be supplied");
    } catch (RuntimeException e) {
    }
    UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.ERROR);
    configsUpdated = new HashSet<>();
    sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
    Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
    for (String sensorType : sensorEnrichmentConfigs.keySet()) {
        sampleConfigurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType));
    }
    StandAloneConfiguredEnrichmentBolt configuredBolt = new StandAloneConfiguredEnrichmentBolt(zookeeperUrl);
    configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
    waitForConfigUpdate(enrichmentConfigurationTypes);
    Assert.assertEquals(sampleConfigurations, configuredBolt.getConfigurations());
    configsUpdated = new HashSet<>();
    Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig();
    sampleGlobalConfig.put("newGlobalField", "newGlobalValue");
    ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
    Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig());
    configsUpdated = new HashSet<>();
    sampleGlobalConfig.remove("newGlobalField");
    ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
    Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations());
    configsUpdated = new HashSet<>();
    String sensorType = "testSensorConfig";
    SensorEnrichmentConfig testSensorConfig = new SensorEnrichmentConfig();
    Map<String, Object> enrichmentFieldMap = new HashMap<>();
    enrichmentFieldMap.put("enrichmentTest", new ArrayList<String>() {

        {
            add("enrichmentField");
        }
    });
    testSensorConfig.getEnrichment().setFieldMap(enrichmentFieldMap);
    Map<String, Object> threatIntelFieldMap = new HashMap<>();
    threatIntelFieldMap.put("threatIntelTest", new ArrayList<String>() {

        {
            add("threatIntelField");
        }
    });
    testSensorConfig.getThreatIntel().setFieldMap(threatIntelFieldMap);
    sampleConfigurations.updateSensorEnrichmentConfig(sensorType, testSensorConfig);
    ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl);
    waitForConfigUpdate(sensorType);
    Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations());
    configuredBolt.cleanup();
}

Example 19 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class SensorEnrichmentUpdateConfigTest method testEnrichment.

@Test
public void testEnrichment() throws Exception {
    SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
    SensorEnrichmentUpdateConfig config = JSONUtils.INSTANCE.load(enrichmentConfigStr, SensorEnrichmentUpdateConfig.class);
    final Map<String, SensorEnrichmentConfig> outputScs = new HashMap<>();
    SensorEnrichmentUpdateConfig.SourceConfigHandler scHandler = new SensorEnrichmentUpdateConfig.SourceConfigHandler() {

        @Override
        public SensorEnrichmentConfig readConfig(String sensor) throws Exception {
            if (sensor.equals("bro")) {
                return JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
            } else {
                throw new IllegalStateException("Tried to retrieve an unexpected sensor: " + sensor);
            }
        }

        @Override
        public void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception {
            outputScs.put(sensor, config);
        }
    };
    SensorEnrichmentUpdateConfig.updateSensorConfigs(scHandler, config.getSensorToFieldList());
    Assert.assertNotNull(outputScs.get("bro"));
    Assert.assertNotSame(outputScs.get("bro"), broSc);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).size(), 2);
    Assert.assertTrue(outputScs.get("bro").toJSON(), ((List<String>) outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).contains("ip_src_addr"));
    Assert.assertTrue(outputScs.get("bro").toJSON(), ((List<String>) outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).contains("ip_dst_addr"));
    Assert.assertEquals(outputScs.get("bro").toJSON(), outputScs.get("bro").getEnrichment().getFieldToTypeMap().keySet().size(), 2);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_src_addr"))).size(), 1);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_src_addr"))).get(0), "playful");
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_dst_addr"))).size(), 1);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_dst_addr"))).get(0), "playful");
}

Example 20 with SensorEnrichmentConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig in project metron by apache.

the class SensorEnrichmentUpdateConfigTest method testThreatIntel.

@Test
public void testThreatIntel() throws Exception {
    SensorEnrichmentConfig broSc = (SensorEnrichmentConfig) ConfigurationType.ENRICHMENT.deserialize(sourceConfigStr);
    SensorEnrichmentUpdateConfig threatIntelConfig = JSONUtils.INSTANCE.load(threatIntelConfigStr, SensorEnrichmentUpdateConfig.class);
    final Map<String, SensorEnrichmentConfig> finalEnrichmentConfig = new HashMap<>();
    SensorEnrichmentUpdateConfig.SourceConfigHandler scHandler = new SensorEnrichmentUpdateConfig.SourceConfigHandler() {

        @Override
        public SensorEnrichmentConfig readConfig(String sensor) throws Exception {
            if (sensor.equals("bro")) {
                return JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
            } else {
                throw new IllegalStateException("Tried to retrieve an unexpected sensor: " + sensor);
            }
        }

        @Override
        public void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception {
            finalEnrichmentConfig.put(sensor, config);
        }
    };
    SensorEnrichmentUpdateConfig.updateSensorConfigs(scHandler, threatIntelConfig.getSensorToFieldList());
    Assert.assertNotNull(finalEnrichmentConfig.get("bro"));
    Assert.assertNotSame(finalEnrichmentConfig.get("bro"), broSc);
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).size(), 2);
    Assert.assertEquals(1, finalEnrichmentConfig.get("bro").getThreatIntel().getTriageConfig().getRiskLevelRules().size());
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).contains("ip_src_addr"));
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).contains("ip_dst_addr"));
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().keySet().size(), 2);
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).size(), 2);
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).contains("playful"));
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).contains("malicious_ip"));
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_dst_addr"))).size(), 2);
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_dst_addr"))).contains("playful"));
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_dst_addr"))).contains("malicious_ip"));
}