Examples with BulkMessageWriter org.apache.metron.common.writer.BulkMessageWriter used on opensource projects

Example 1 with BulkMessageWriter

use of org.apache.metron.common.writer.BulkMessageWriter in project metron by apache.

the class BulkMessageWriterBoltTest method testFlushOnBatchTimeout.

@Test
public void testFlushOnBatchTimeout() throws Exception {
    FakeClock clock = new FakeClock();
    bulkMessageWriterBolt = bulkMessageWriterBolt.withBatchTimeoutDivisor(3);
    bulkMessageWriterBolt.setCuratorFramework(client);
    bulkMessageWriterBolt.setZKCache(cache);
    bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType, new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
    {
        bulkMessageWriterBolt.declareOutputFields(declarer);
        verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
    }
    {
        Map stormConf = new HashMap();
        when(bulkMessageWriter.getName()).thenReturn("elasticsearch");
        bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector, clock);
        verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(WriterConfiguration.class));
    }
    {
        int batchTimeout = bulkMessageWriterBolt.getMaxBatchTimeout();
        assertEquals(4, batchTimeout);
        for (int i = 0; i < 4; i++) {
            bulkMessageWriterBolt.execute(tupleList.get(i));
            verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), any(List.class));
        }
        clock.elapseSeconds(5);
        BulkWriterResponse response = new BulkWriterResponse();
        response.addAllSuccesses(messageIdList);
        when(bulkMessageWriter.write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
        bulkMessageWriterBolt.execute(tupleList.get(4));
        verify(bulkMessageWriter, times(1)).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList));
        tupleList.forEach(tuple -> verify(outputCollector, times(1)).ack(tuple));
    }
    Mockito.verifyNoMoreInteractions(outputCollector);
}

Example 2 with BulkMessageWriter

use of org.apache.metron.common.writer.BulkMessageWriter in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfigWithSensorParserConfigOutputTopic.

@Test
public void shouldCreateWriterConfigWithSensorParserConfigOutputTopic() {
    SensorParserConfig snortConfig = new SensorParserConfig();
    snortConfig.setSensorTopic("snort");
    snortConfig.setOutputTopic("snort_topic");
    when(configs.getSensorParserConfig("snort")).thenReturn(snortConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("snort", snortConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.empty());
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("snort").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals("snort_topic", ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}

Example 3 with BulkMessageWriter

use of org.apache.metron.common.writer.BulkMessageWriter in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfig.

@Test
public void shouldCreateWriterConfig() {
    SensorParserConfig broConfig = new SensorParserConfig();
    broConfig.setSensorTopic("bro");
    when(configs.getSensorParserConfig("bro")).thenReturn(broConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("bro", broConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.empty());
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("bro").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals(Constants.ENRICHMENT_TOPIC, ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}

Example 4 with BulkMessageWriter

use of org.apache.metron.common.writer.BulkMessageWriter in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfigWithSuppliedOutputTopic.

@Test
public void shouldCreateWriterConfigWithSuppliedOutputTopic() {
    SensorParserConfig snortConfig = new SensorParserConfig();
    snortConfig.setSensorTopic("snort");
    when(configs.getSensorParserConfig("snort")).thenReturn(snortConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("snort", snortConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.of("supplied_topic"));
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("snort").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals("supplied_topic", ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}

Example 5 with BulkMessageWriter

use of org.apache.metron.common.writer.BulkMessageWriter in project metron by apache.

the class BulkMessageWriterBoltTest method testFlushOnTickTuple.

@Test
public void testFlushOnTickTuple() throws Exception {
    FakeClock clock = new FakeClock();
    bulkMessageWriterBolt.setCuratorFramework(client);
    bulkMessageWriterBolt.setZKCache(cache);
    bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType, new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
    {
        bulkMessageWriterBolt.declareOutputFields(declarer);
        verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
    }
    {
        Map stormConf = new HashMap();
        when(bulkMessageWriter.getName()).thenReturn("elasticsearch");
        bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector, clock);
        verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(WriterConfiguration.class));
    }
    {
        int batchTimeout = bulkMessageWriterBolt.getMaxBatchTimeout();
        assertEquals(14, batchTimeout);
        for (int i = 0; i < 5; i++) {
            bulkMessageWriterBolt.execute(tupleList.get(i));
            verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), any());
        }
        Tuple tickTuple = mock(Tuple.class);
        when(tickTuple.getValueByField("message")).thenReturn(null);
        // mark the tuple as a TickTuple, part 1 of 2
        when(tickTuple.getSourceComponent()).thenReturn("__system");
        // mark the tuple as a TickTuple, part 2 of 2
        when(tickTuple.getSourceStreamId()).thenReturn("__tick");
        BulkWriterResponse response = new BulkWriterResponse();
        response.addAllSuccesses(messageIdList);
        when(bulkMessageWriter.write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
        clock.advanceToSeconds(2);
        bulkMessageWriterBolt.execute(tickTuple);
        verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList));
        // 1 tick
        verify(outputCollector, times(1)).ack(tickTuple);
        clock.advanceToSeconds(9);
        bulkMessageWriterBolt.execute(tickTuple);
        verify(bulkMessageWriter, times(1)).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList));
        assertEquals(5, tupleList.size());
        tupleList.forEach(tuple -> verify(outputCollector, times(1)).ack(tuple));
        verify(outputCollector, times(2)).ack(tickTuple);
    }
    Mockito.verifyNoMoreInteractions(outputCollector);
}