Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 26 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchErrorPathExceptionInWrite.

@Test
public void testBatchErrorPathExceptionInWrite() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
        tuples.add(t);
    }
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
    verify(batchWriter, times(1)).init(any(), any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any(), any());
    }
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
    bolt.execute(goodTuple);
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 27 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchErrorWriteFailure.

@Test
public void testBatchErrorWriteFailure() throws Exception {
    ParserConfigurations configurations = getConfigurations(6);
    String sensorType = "test";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    when(errorTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any(), any());
    }
    // Add both the good and error Tuples. This simulates a seemingly good Tuple that fails on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(tuples);
    writerResponse.addSuccess(goodTuple);
    writerResponse.addError(new IllegalStateException(), errorTuple);
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
    bolt.execute(goodTuple);
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 28 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchErrorPath.

@Test
public void testBatchErrorPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    when(errorTuple.getValueByField(eq("message"))).thenThrow(new IllegalStateException());
    WriterBolt bolt = new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any(), any());
    }
    // Add the good tuples.  Do not add the error tuple, because this is testing an exception on access, not a failure on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(tuples);
    writerResponse.addSuccess(goodTuple);
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    bolt.execute(goodTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 29 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testNonBatchHappyPath.

@Test
public void testNonBatchHappyPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(1);
    String sensorType = "test";
    Tuple t = mock(Tuple.class);
    when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(writer, times(1)).init();
    bolt.execute(t);
    verify(outputCollector, times(1)).ack(t);
    verify(writer, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(0)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 30 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class KafkaWriterTest method createConfiguration.

public WriterConfiguration createConfiguration(final Map<String, Object> parserConfig) {
    ParserConfigurations configurations = new ParserConfigurations();
    configurations.updateSensorParserConfig(SENSOR_TYPE, new SensorParserConfig() {

        {
            setParserConfig(parserConfig);
        }
    });
    return new ParserWriterConfiguration(configurations);
}