Examples with Extractor org.apache.metron.dataloads.extractor.Extractor used on opensource projects

Example 1 with Extractor

use of org.apache.metron.dataloads.extractor.Extractor in project metron by apache.

the class TaxiiLoader method main.

public static void main(String... argv) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
    String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
    CommandLine cli = TaxiiOptions.parse(new PosixParser(), otherArgs);
    if (TaxiiOptions.LOG4J_PROPERTIES.has(cli)) {
        PropertyConfigurator.configure(TaxiiOptions.LOG4J_PROPERTIES.get(cli));
    }
    ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(TaxiiOptions.EXTRACTOR_CONFIG.get(cli))));
    Extractor e = handler.getExtractor();
    SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
    if (TaxiiOptions.ENRICHMENT_CONFIG.has(cli)) {
        sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load(new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli)), SensorEnrichmentUpdateConfig.class);
        sensorEnrichmentUpdateConfig.updateSensorConfigs();
    }
    Timer timer = new Timer();
    if (isStixExtractor(e)) {
        Extractor extractor = e;
        TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli))));
        if (TaxiiOptions.BEGIN_TIME.has(cli)) {
            Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli));
            connectionConfig.withBeginTime(d);
        }
        long timeBetween = DEFAULT_TIME_BETWEEN_POLLS;
        if (TaxiiOptions.TIME_BETWEEN_POLLS.has(cli)) {
            timeBetween = Long.parseLong(TaxiiOptions.TIME_BETWEEN_POLLS.get(cli));
        }
        timer.scheduleAtFixedRate(new TaxiiHandler(connectionConfig, extractor, conf), 0, timeBetween);
    } else {
        throw new IllegalStateException("Extractor must be a STIX Extractor");
    }
}

Example 2 with Extractor

use of org.apache.metron.dataloads.extractor.Extractor in project metron by apache.

the class TaxiiIntegrationTest method testTaxii.

@Test
public void testTaxii() throws Exception {
    final MockHBaseTableProvider provider = new MockHBaseTableProvider();
    final Configuration config = HBaseConfiguration.create();
    Extractor extractor = new TransformFilterExtractorDecorator(new StixExtractor());
    TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), extractor, config) {

        @Override
        protected synchronized HTableInterface createHTable(String tableInfo) throws IOException {
            return provider.addToCache("threat_intel", "cf");
        }
    };
    // UnitTestHelper.verboseLogging();
    handler.run();
    Set<String> maliciousDomains;
    {
        MockHTable table = (MockHTable) provider.getTable(config, "threat_intel");
        maliciousDomains = getIndicators("domainname:FQDN", table.getPutLog(), "cf");
    }
    Assert.assertTrue(maliciousDomains.contains("www.office-112.com"));
    Assert.assertEquals(numStringsMatch(MockTaxiiService.pollMsg, "DomainNameObj:Value condition=\"Equals\""), maliciousDomains.size());
    Set<String> maliciousAddresses;
    {
        MockHTable table = (MockHTable) provider.getTable(config, "threat_intel");
        maliciousAddresses = getIndicators("address:IPV_4_ADDR", table.getPutLog(), "cf");
    }
    Assert.assertTrue(maliciousAddresses.contains("94.102.53.142"));
    Assert.assertEquals(numStringsMatch(MockTaxiiService.pollMsg, "AddressObj:Address_Value condition=\"Equal\""), maliciousAddresses.size());
    MockHBaseTableProvider.clear();
    // Ensure that the handler can be run multiple times without connection issues.
    handler.run();
}

Example 3 with Extractor

use of org.apache.metron.dataloads.extractor.Extractor in project metron by apache.

the class StixExtractorTest method testStixAddresses.

public void testStixAddresses(final String stixDoc) throws Exception {
    Thread t1 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV4);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(3, Iterables.size(results));
            Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator);
            Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator);
            Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator);
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    Thread t2 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfig);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(3, Iterables.size(results));
            Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator);
            Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator);
            Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator);
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    Thread t3 = new Thread(() -> {
        try {
            ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV6);
            Extractor extractor = handler.getExtractor();
            Iterable<LookupKV> results = extractor.extract(stixDoc);
            Assert.assertEquals(0, Iterables.size(results));
        } catch (Exception ex) {
            throw new RuntimeException(ex.getMessage(), ex);
        }
    });
    t1.run();
    t2.run();
    t3.run();
    t1.join();
    t2.join();
    t3.join();
}