Examples with SensorEnrichmentUpdateConfig org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig used on opensource projects

Example 1 with SensorEnrichmentUpdateConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig in project metron by apache.

the class SimpleEnrichmentFlatFileLoader method main.

public static void main(Configuration hadoopConfig, String[] argv) throws Exception {
    CommandLine cli = LoadOptions.parse(new PosixParser(), argv);
    EnumMap<LoadOptions, Optional<Object>> config = LoadOptions.createConfig(cli);
    if (LoadOptions.LOG4J_PROPERTIES.has(cli)) {
        PropertyConfigurator.configure(LoadOptions.LOG4J_PROPERTIES.get(cli));
    }
    ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(LoadOptions.EXTRACTOR_CONFIG.get(cli).trim())));
    ImportStrategy strategy = (ImportStrategy) config.get(LoadOptions.IMPORT_MODE).get();
    strategy.getImporter().importData(config, handler, hadoopConfig);
    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
    if (LoadOptions.ENRICHMENT_CONFIG.has(cli)) {
        sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load(new File(LoadOptions.ENRICHMENT_CONFIG.get(cli)), SensorEnrichmentUpdateConfig.class);
    }
    if (sensorEnrichmentUpdateConfig != null) {
        sensorEnrichmentUpdateConfig.updateSensorConfigs();
    }
}

Example 2 with SensorEnrichmentUpdateConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig in project metron by apache.

the class TaxiiLoader method main.

public static void main(String... argv) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
    String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
    CommandLine cli = TaxiiOptions.parse(new PosixParser(), otherArgs);
    if (TaxiiOptions.LOG4J_PROPERTIES.has(cli)) {
        PropertyConfigurator.configure(TaxiiOptions.LOG4J_PROPERTIES.get(cli));
    }
    ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(TaxiiOptions.EXTRACTOR_CONFIG.get(cli))));
    Extractor e = handler.getExtractor();
    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
    if (TaxiiOptions.ENRICHMENT_CONFIG.has(cli)) {
        sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load(new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli)), SensorEnrichmentUpdateConfig.class);
        sensorEnrichmentUpdateConfig.updateSensorConfigs();
    }
    Timer timer = new Timer();
    if (isStixExtractor(e)) {
        Extractor extractor = e;
        TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli))));
        if (TaxiiOptions.BEGIN_TIME.has(cli)) {
            Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli));
            connectionConfig.withBeginTime(d);
        }
        long timeBetween = DEFAULT_TIME_BETWEEN_POLLS;
        if (TaxiiOptions.TIME_BETWEEN_POLLS.has(cli)) {
            timeBetween = Long.parseLong(TaxiiOptions.TIME_BETWEEN_POLLS.get(cli));
        }
        timer.scheduleAtFixedRate(new TaxiiHandler(connectionConfig, extractor, conf), 0, timeBetween);
    } else {
        throw new IllegalStateException("Extractor must be a STIX Extractor");
    }
}

Example 3 with SensorEnrichmentUpdateConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig in project metron by apache.

the class SensorEnrichmentUpdateConfigTest method testEnrichment.

@Test
public void testEnrichment() throws Exception {
    SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
    SensorEnrichmentUpdateConfig config = JSONUtils.INSTANCE.load(enrichmentConfigStr, SensorEnrichmentUpdateConfig.class);
    final Map<String, SensorEnrichmentConfig> outputScs = new HashMap<>();
    SensorEnrichmentUpdateConfig.SourceConfigHandler scHandler = new SensorEnrichmentUpdateConfig.SourceConfigHandler() {

        @Override
        public SensorEnrichmentConfig readConfig(String sensor) throws Exception {
            if (sensor.equals("bro")) {
                return JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
            } else {
                throw new IllegalStateException("Tried to retrieve an unexpected sensor: " + sensor);
            }
        }

        @Override
        public void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception {
            outputScs.put(sensor, config);
        }
    };
    SensorEnrichmentUpdateConfig.updateSensorConfigs(scHandler, config.getSensorToFieldList());
    Assert.assertNotNull(outputScs.get("bro"));
    Assert.assertNotSame(outputScs.get("bro"), broSc);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).size(), 2);
    Assert.assertTrue(outputScs.get("bro").toJSON(), ((List<String>) outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).contains("ip_src_addr"));
    Assert.assertTrue(outputScs.get("bro").toJSON(), ((List<String>) outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).contains("ip_dst_addr"));
    Assert.assertEquals(outputScs.get("bro").toJSON(), outputScs.get("bro").getEnrichment().getFieldToTypeMap().keySet().size(), 2);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_src_addr"))).size(), 1);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_src_addr"))).get(0), "playful");
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_dst_addr"))).size(), 1);
    Assert.assertEquals(outputScs.get("bro").toJSON(), ((List<String>) (outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_dst_addr"))).get(0), "playful");
}

Example 4 with SensorEnrichmentUpdateConfig

use of org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig in project metron by apache.

the class SensorEnrichmentUpdateConfigTest method testThreatIntel.

@Test
public void testThreatIntel() throws Exception {
    SensorEnrichmentConfig broSc = (SensorEnrichmentConfig) ConfigurationType.ENRICHMENT.deserialize(sourceConfigStr);
    SensorEnrichmentUpdateConfig threatIntelConfig = JSONUtils.INSTANCE.load(threatIntelConfigStr, SensorEnrichmentUpdateConfig.class);
    final Map<String, SensorEnrichmentConfig> finalEnrichmentConfig = new HashMap<>();
    SensorEnrichmentUpdateConfig.SourceConfigHandler scHandler = new SensorEnrichmentUpdateConfig.SourceConfigHandler() {

        @Override
        public SensorEnrichmentConfig readConfig(String sensor) throws Exception {
            if (sensor.equals("bro")) {
                return JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
            } else {
                throw new IllegalStateException("Tried to retrieve an unexpected sensor: " + sensor);
            }
        }

        @Override
        public void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception {
            finalEnrichmentConfig.put(sensor, config);
        }
    };
    SensorEnrichmentUpdateConfig.updateSensorConfigs(scHandler, threatIntelConfig.getSensorToFieldList());
    Assert.assertNotNull(finalEnrichmentConfig.get("bro"));
    Assert.assertNotSame(finalEnrichmentConfig.get("bro"), broSc);
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).size(), 2);
    Assert.assertEquals(1, finalEnrichmentConfig.get("bro").getThreatIntel().getTriageConfig().getRiskLevelRules().size());
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).contains("ip_src_addr"));
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).contains("ip_dst_addr"));
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().keySet().size(), 2);
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).size(), 2);
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).contains("playful"));
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).contains("malicious_ip"));
    Assert.assertEquals(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_dst_addr"))).size(), 2);
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_dst_addr"))).contains("playful"));
    Assert.assertTrue(finalEnrichmentConfig.get("bro").toJSON(), ((List<String>) (finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_dst_addr"))).contains("malicious_ip"));
}