Examples with ExtractorHandler org.apache.metron.dataloads.extractor.ExtractorHandler used on opensource projects

Example 1 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class SimpleEnrichmentFlatFileLoader method main.

public static void main(Configuration hadoopConfig, String[] argv) throws Exception {
    CommandLine cli = LoadOptions.parse(new PosixParser(), argv);
    EnumMap<LoadOptions, Optional<Object>> config = LoadOptions.createConfig(cli);
    if (LoadOptions.LOG4J_PROPERTIES.has(cli)) {
        PropertyConfigurator.configure(LoadOptions.LOG4J_PROPERTIES.get(cli));
    }
    ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(LoadOptions.EXTRACTOR_CONFIG.get(cli).trim())));
    ImportStrategy strategy = (ImportStrategy) config.get(LoadOptions.IMPORT_MODE).get();
    strategy.getImporter().importData(config, handler, hadoopConfig);
    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
    if (LoadOptions.ENRICHMENT_CONFIG.has(cli)) {
        sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load(new File(LoadOptions.ENRICHMENT_CONFIG.get(cli)), SensorEnrichmentUpdateConfig.class);
    }
    if (sensorEnrichmentUpdateConfig != null) {
        sensorEnrichmentUpdateConfig.updateSensorConfigs();
    }
}

Example 2 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class SimpleFlatFileSummarizer method main.

public static void main(Configuration hadoopConfig, String[] argv) throws Exception {
    CommandLine cli = SummarizeOptions.parse(new PosixParser(), argv);
    EnumMap<SummarizeOptions, Optional<Object>> config = SummarizeOptions.createConfig(cli);
    if (SummarizeOptions.LOG4J_PROPERTIES.has(cli)) {
        PropertyConfigurator.configure(SummarizeOptions.LOG4J_PROPERTIES.get(cli));
    }
    ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(SummarizeOptions.EXTRACTOR_CONFIG.get(cli).trim())));
    Summarizers strategy = (Summarizers) config.get(SummarizeOptions.IMPORT_MODE).get();
    strategy.getSummarizer().importData(config, handler, hadoopConfig);
}

Example 3 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class TaxiiLoader method main.

public static void main(String... argv) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
    String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
    CommandLine cli = TaxiiOptions.parse(new PosixParser(), otherArgs);
    if (TaxiiOptions.LOG4J_PROPERTIES.has(cli)) {
        PropertyConfigurator.configure(TaxiiOptions.LOG4J_PROPERTIES.get(cli));
    }
    ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(TaxiiOptions.EXTRACTOR_CONFIG.get(cli))));
    Extractor e = handler.getExtractor();
    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
    if (TaxiiOptions.ENRICHMENT_CONFIG.has(cli)) {
        sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load(new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli)), SensorEnrichmentUpdateConfig.class);
        sensorEnrichmentUpdateConfig.updateSensorConfigs();
    }
    Timer timer = new Timer();
    if (isStixExtractor(e)) {
        Extractor extractor = e;
        TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli))));
        if (TaxiiOptions.BEGIN_TIME.has(cli)) {
            Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli));
            connectionConfig.withBeginTime(d);
        }
        long timeBetween = DEFAULT_TIME_BETWEEN_POLLS;
        if (TaxiiOptions.TIME_BETWEEN_POLLS.has(cli)) {
            timeBetween = Long.parseLong(TaxiiOptions.TIME_BETWEEN_POLLS.get(cli));
        }
        timer.scheduleAtFixedRate(new TaxiiHandler(connectionConfig, extractor, conf), 0, timeBetween);
    } else {
        throw new IllegalStateException("Extractor must be a STIX Extractor");
    }
}

Example 4 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class CSVExtractorTest method testCSVExtractor.

@Test
public void testCSVExtractor() throws Exception {
    ExtractorHandler handler = ExtractorHandler.load(testCSVConfig);
    validate(handler);
}

Example 5 with ExtractorHandler

use of org.apache.metron.dataloads.extractor.ExtractorHandler in project metron by apache.

the class CSVExtractorTest method testInitialize.

@Test
public void testInitialize() throws Exception {
    CSVExtractor ex = new CSVExtractor();
    ExtractorHandler handler = ExtractorHandler.load(testCSVConfig);
    ex.initialize(handler.getConfig());
    Assert.assertEquals(0, (int) ex.getColumnMap().get("host"));
    Assert.assertEquals(2, (int) ex.getColumnMap().get("meta"));
    Assert.assertEquals(0, ex.getTypeColumnIndex());
    Assert.assertEquals(0, ex.getIndicatorColumn());
    Assert.assertEquals("threat", ex.getType());
    Assert.assertEquals(',', ex.getParser().getSeparator());
}