Examples with KafkaWriter org.apache.metron.writer.kafka.KafkaWriter used on opensource projects

Example 1 with KafkaWriter

use of org.apache.metron.writer.kafka.KafkaWriter in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfigWithSensorParserConfigOutputTopic.

@Test
public void shouldCreateWriterConfigWithSensorParserConfigOutputTopic() {
    SensorParserConfig snortConfig = new SensorParserConfig();
    snortConfig.setSensorTopic("snort");
    snortConfig.setOutputTopic("snort_topic");
    when(configs.getSensorParserConfig("snort")).thenReturn(snortConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("snort", snortConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.empty());
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("snort").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals("snort_topic", ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}

Example 2 with KafkaWriter

use of org.apache.metron.writer.kafka.KafkaWriter in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfig.

@Test
public void shouldCreateWriterConfig() {
    SensorParserConfig broConfig = new SensorParserConfig();
    broConfig.setSensorTopic("bro");
    when(configs.getSensorParserConfig("bro")).thenReturn(broConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("bro", broConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.empty());
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("bro").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals(Constants.ENRICHMENT_TOPIC, ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}

Example 3 with KafkaWriter

use of org.apache.metron.writer.kafka.KafkaWriter in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfigWithSuppliedOutputTopic.

@Test
public void shouldCreateWriterConfigWithSuppliedOutputTopic() {
    SensorParserConfig snortConfig = new SensorParserConfig();
    snortConfig.setSensorTopic("snort");
    when(configs.getSensorParserConfig("snort")).thenReturn(snortConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("snort", snortConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.of("supplied_topic"));
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("snort").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals("supplied_topic", ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}

Example 4 with KafkaWriter

use of org.apache.metron.writer.kafka.KafkaWriter in project metron by apache.

the class ParserTopologyBuilder method createKafkaWriter.

/**
 * Create a Kafka writer.
 *
 * @param broker An optional URL to the Kafka brokers.
 * @param zkQuorum The URL to Zookeeper.
 * @param securityProtocol An optional security protocol in use.
 * @return
 */
protected static KafkaWriter createKafkaWriter(Optional<String> broker, String zkQuorum, Optional<String> securityProtocol) {
    KafkaWriter writer = new KafkaWriter();
    // cluster URL; either broker or zookeeper
    if (broker.isPresent()) {
        writer.withBrokerUrl(broker.get());
    } else {
        writer.withZkQuorum(zkQuorum);
    }
    // security protocol
    if (securityProtocol.isPresent()) {
        HashMap<String, Object> config = new HashMap<>();
        config.put("security.protocol", securityProtocol.get());
        writer.withProducerConfigs(config);
    }
    return writer;
}