Examples with KafkaComponent org.apache.metron.integration.components.KafkaComponent used on opensource projects

Example 6 with KafkaComponent

use of org.apache.metron.integration.components.KafkaComponent in project metron by apache.

the class SimpleHbaseEnrichmentWriterIntegrationTest method test.

@Test
public void test() throws UnableToStartException, IOException {
    final String sensorType = "dummy";
    final List<byte[]> inputMessages = new ArrayList<byte[]>() {

        {
            add(Bytes.toBytes("col11,col12,col13"));
            add(Bytes.toBytes("col21,col22,col23"));
            add(Bytes.toBytes("col31,col32,col33"));
        }
    };
    MockHBaseTableProvider.addToCache(sensorType, "cf");
    final Properties topologyProperties = new Properties();
    final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
    final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {

        {
            add(new KafkaComponent.Topic(sensorType, 1));
        }
    });
    topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH).withParserSensorConfig(sensorType, JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class));
    ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder().withSensorType(sensorType).withTopologyProperties(topologyProperties).withBrokerUrl(kafkaComponent.getBrokerList()).build();
    // UnitTestHelper.verboseLogging();
    ComponentRunner runner = new ComponentRunner.Builder().withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("org/apache/storm", parserTopologyComponent).withMillisecondsBetweenAttempts(5000).withCustomShutdownOrder(new String[] { "org/apache/storm", "config", "kafka", "zk" }).withNumRetries(10).build();
    try {
        runner.start();
        kafkaComponent.writeMessages(sensorType, inputMessages);
        ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> result = runner.process(new Processor<List<LookupKV<EnrichmentKey, EnrichmentValue>>>() {

            List<LookupKV<EnrichmentKey, EnrichmentValue>> messages = null;

            @Override
            public ReadinessState process(ComponentRunner runner) {
                MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(sensorType);
                if (table != null && table.size() == inputMessages.size()) {
                    EnrichmentConverter converter = new EnrichmentConverter();
                    messages = new ArrayList<>();
                    try {
                        for (Result r : table.getScanner(Bytes.toBytes("cf"))) {
                            messages.add(converter.fromResult(r, "cf"));
                        }
                    } catch (IOException e) {
                    }
                    return ReadinessState.READY;
                }
                return ReadinessState.NOT_READY;
            }

            @Override
            public ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> getResult() {
                ProcessorResult.Builder<List<LookupKV<EnrichmentKey, EnrichmentValue>>> builder = new ProcessorResult.Builder();
                return builder.withResult(messages).build();
            }
        });
        Set<String> validIndicators = new HashSet<>(ImmutableList.of("col12", "col22", "col32"));
        Map<String, Map<String, String>> validMetadata = new HashMap<String, Map<String, String>>() {

            {
                put("col12", new HashMap<String, String>() {

                    {
                        put("col1", "col11");
                        put("col3", "col13");
                    }
                });
                put("col22", new HashMap<String, String>() {

                    {
                        put("col1", "col21");
                        put("col3", "col23");
                    }
                });
                put("col32", new HashMap<String, String>() {

                    {
                        put("col1", "col31");
                        put("col3", "col33");
                    }
                });
            }
        };
        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : result.getResult()) {
            Assert.assertTrue(validIndicators.contains(kv.getKey().indicator));
            Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy");
            Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp"));
            Assert.assertNotNull(kv.getValue().getMetadata().get("original_string"));
            Map<String, String> metadata = validMetadata.get(kv.getKey().indicator);
            for (Map.Entry<String, String> x : metadata.entrySet()) {
                Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
            }
            Assert.assertEquals(metadata.size() + 4, kv.getValue().getMetadata().size());
        }
    } finally {
        if (runner != null) {
            runner.stop();
        }
    }
}

Example 7 with KafkaComponent

use of org.apache.metron.integration.components.KafkaComponent in project metron by apache.

the class EnrichmentIntegrationTest method test.

@Test
public void test() throws Exception {
    final String cf = "cf";
    final String trackerHBaseTableName = "tracker";
    final String threatIntelTableName = "threat_intel";
    final String enrichmentsTableName = "enrichments";
    final Properties topologyProperties = new Properties() {

        {
            setProperty("enrichment_workers", "1");
            setProperty("enrichment_acker_executors", "0");
            setProperty("enrichment_topology_worker_childopts", "");
            setProperty("topology_auto_credentials", "[]");
            setProperty("enrichment_topology_max_spout_pending", "");
            setProperty("enrichment_kafka_start", "UNCOMMITTED_EARLIEST");
            setProperty("kafka_security_protocol", "PLAINTEXT");
            setProperty("enrichment_input_topic", Constants.ENRICHMENT_TOPIC);
            setProperty("enrichment_output_topic", Constants.INDEXING_TOPIC);
            setProperty("enrichment_error_topic", ERROR_TOPIC);
            setProperty("threatintel_error_topic", ERROR_TOPIC);
            setProperty("enrichment_join_cache_size", "1000");
            setProperty("threatintel_join_cache_size", "1000");
            setProperty("enrichment_hbase_provider_impl", "" + MockHBaseTableProvider.class.getName());
            setProperty("enrichment_hbase_table", enrichmentsTableName);
            setProperty("enrichment_hbase_cf", cf);
            setProperty("enrichment_host_known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
            setProperty("threatintel_hbase_table", threatIntelTableName);
            setProperty("threatintel_hbase_cf", cf);
            setProperty("enrichment_kafka_spout_parallelism", "1");
            setProperty("enrichment_split_parallelism", "1");
            setProperty("enrichment_stellar_parallelism", "1");
            setProperty("enrichment_join_parallelism", "1");
            setProperty("threat_intel_split_parallelism", "1");
            setProperty("threat_intel_stellar_parallelism", "1");
            setProperty("threat_intel_join_parallelism", "1");
            setProperty("kafka_writer_parallelism", "1");
        }
    };
    final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
    final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {

        {
            add(new KafkaComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
            add(new KafkaComponent.Topic(Constants.INDEXING_TOPIC, 1));
            add(new KafkaComponent.Topic(ERROR_TOPIC, 1));
        }
    });
    String globalConfigStr = null;
    {
        File globalConfig = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), "global.json");
        Map<String, Object> config = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
        config.put(SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF, MockHBaseTableProvider.class.getName());
        config.put(SimpleHBaseEnrichmentFunctions.ACCESS_TRACKER_TYPE_CONF, "PERSISTENT_BLOOM");
        config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_TABLE, trackerHBaseTableName);
        config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_CF, cf);
        config.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
        globalConfigStr = JSONUtils.INSTANCE.toJSON(config, true);
    }
    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfig(globalConfigStr).withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
    // create MockHBaseTables
    final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(trackerHBaseTableName, cf);
    final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf);
    EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
        }
    });
    final MockHTable enrichmentTable = (MockHTable) MockHBaseTableProvider.addToCache(enrichmentsTableName, cf);
    EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {

        {
            add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3"), new EnrichmentValue(PLAYFUL_ENRICHMENT)));
        }
    });
    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder().withTopologyLocation(new File(fluxPath())).withTopologyName("test").withTemplateLocation(new File(templatePath)).withTopologyProperties(topologyProperties).build();
    // UnitTestHelper.verboseLogging();
    ComponentRunner runner = new ComponentRunner.Builder().withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("storm", fluxComponent).withMillisecondsBetweenAttempts(15000).withCustomShutdownOrder(new String[] { "storm", "config", "kafka", "zk" }).withNumRetries(10).build();
    try {
        runner.start();
        fluxComponent.submitTopology();
        kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
        ProcessorResult<Map<String, List<Map<String, Object>>>> result = runner.process(getProcessor());
        Map<String, List<Map<String, Object>>> outputMessages = result.getResult();
        List<Map<String, Object>> docs = outputMessages.get(Constants.INDEXING_TOPIC);
        Assert.assertEquals(inputMessages.size(), docs.size());
        validateAll(docs);
        List<Map<String, Object>> errors = outputMessages.get(ERROR_TOPIC);
        Assert.assertEquals(inputMessages.size(), errors.size());
        validateErrors(errors);
    } finally {
        runner.stop();
    }
}

Example 8 with KafkaComponent

use of org.apache.metron.integration.components.KafkaComponent in project metron by apache.

the class IndexingIntegrationTest method test.

@Test
public void test() throws Exception {
    preTest();
    final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
    final Properties topologyProperties = new Properties() {

        {
            setProperty("indexing_kafka_start", "UNCOMMITTED_EARLIEST");
            setProperty("kafka_security_protocol", "PLAINTEXT");
            setProperty("topology_auto_credentials", "[]");
            setProperty("indexing_workers", "1");
            setProperty("indexing_acker_executors", "0");
            setProperty("indexing_topology_worker_childopts", "");
            setProperty("indexing_topology_max_spout_pending", "");
            setProperty("indexing_input_topic", Constants.INDEXING_TOPIC);
            setProperty("indexing_error_topic", ERROR_TOPIC);
            setProperty("indexing_kafka_spout_parallelism", "1");
            setProperty("indexing_writer_parallelism", "1");
        }
    };
    setAdditionalProperties(topologyProperties);
    final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
    final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {

        {
            add(new KafkaComponent.Topic(Constants.INDEXING_TOPIC, 1));
            add(new KafkaComponent.Topic(ERROR_TOPIC, 1));
        }
    });
    List<Map<String, Object>> inputDocs = new ArrayList<>();
    for (byte[] b : inputMessages) {
        Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b), JSONUtils.MAP_SUPPLIER);
        inputDocs.add(m);
    }
    final AtomicBoolean isLoaded = new AtomicBoolean(false);
    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH).withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH).withIndexingConfigsPath(TestConstants.SAMPLE_CONFIG_PATH).withPostStartCallback(component -> {
        try {
            waitForIndex(component.getTopologyProperties().getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY));
        } catch (Exception e) {
            e.printStackTrace();
        }
        isLoaded.set(true);
    });
    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder().withTopologyLocation(new File(getFluxPath())).withTopologyName("test").withTemplateLocation(new File(getTemplatePath())).withTopologyProperties(topologyProperties).build();
    ComponentRunner runner = null;
    InMemoryComponent searchComponent = getSearchComponent(topologyProperties);
    ComponentRunner.Builder componentBuilder = new ComponentRunner.Builder();
    componentBuilder = componentBuilder.withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("storm", fluxComponent).withMillisecondsBetweenAttempts(1500).withNumRetries(NUM_RETRIES).withMaxTimeMS(TOTAL_TIME_MS);
    if (searchComponent != null) {
        componentBuilder = componentBuilder.withComponent("search", getSearchComponent(topologyProperties)).withCustomShutdownOrder(new String[] { "search", "storm", "config", "kafka", "zk" });
    } else {
        componentBuilder = componentBuilder.withCustomShutdownOrder(new String[] { "storm", "config", "kafka", "zk" });
    }
    runner = componentBuilder.build();
    try {
        runner.start();
        while (!isLoaded.get()) {
            Thread.sleep(100);
        }
        fluxComponent.submitTopology();
        kafkaComponent.writeMessages(Constants.INDEXING_TOPIC, inputMessages);
        List<Map<String, Object>> docs = cleanDocs(runner.process(getProcessor(inputMessages)));
        Assert.assertEquals(docs.size(), inputMessages.size());
        // assert that our input docs are equivalent to the output docs, converting the input docs keys based
        // on the field name converter
        assertInputDocsMatchOutputs(inputDocs, docs, getFieldNameConverter());
    } finally {
        if (runner != null) {
            runner.stop();
        }
    }
}

Example 9 with KafkaComponent

use of org.apache.metron.integration.components.KafkaComponent in project metron by apache.

the class KafkaProcessor method process.

@Override
public ReadinessState process(ComponentRunner runner) {
    KafkaComponent kafkaComponent = runner.getComponent(kafkaComponentName, KafkaComponent.class);
    LinkedList<byte[]> outputMessages = new LinkedList<>(kafkaComponent.readMessages(readTopic));
    LinkedList<byte[]> outputErrors = null;
    if (errorTopic != null) {
        outputErrors = new LinkedList<>(kafkaComponent.readMessages(errorTopic));
    }
    Boolean validated = validateReadMessages.apply(new KafkaMessageSet(outputMessages, outputErrors));
    if (validated == null) {
        validated = false;
    }
    if (validated) {
        messages.addAll(outputMessages);
        errors.addAll(outputErrors);
        outputMessages.clear();
        outputErrors.clear();
        return ReadinessState.READY;
    }
    return ReadinessState.NOT_READY;
}