Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 41 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccessWithMultipleMessages.

@Test
public void shouldExecuteOnSuccessWithMultipleMessages() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    List<BulkMessage<JSONObject>> messages = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        String messageId = String.format("messageId%s", i + 1);
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("field", String.format("value%s", i + 1));
        messages.add(new BulkMessage<>(messageId, message));
    }
    mockParserRunner.setMessages(messages.stream().map(BulkMessage::getMessage).collect(Collectors.toList()));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        // Verify the correct message is written and ack is handled
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        InOrder inOrder = inOrder(bulkWriterResponseHandler, writerHandler);
        inOrder.verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Arrays.asList("messageId1", "messageId2", "messageId3", "messageId4", "messageId5"));
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(0), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(1), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(2), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(3), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(4), parserConfigurations);
    }
    verifyNoMoreInteractions(writerHandler, bulkWriterResponseHandler, outputCollector);
}

Example 42 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchHappyPath.

@Test
public void testBatchHappyPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 5; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(new MessageId(messageId));
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    // Ensure the batch returns the good Tuples
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(messageIds);
    when(batchWriter.write(any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(tuples.get(4));
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(0)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 43 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchErrorWriteFailure.

@Test
public void testBatchErrorWriteFailure() throws Exception {
    ParserConfigurations configurations = getConfigurations(6);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(new MessageId(messageId));
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    JSONObject goodMessage = new JSONObject();
    goodMessage.put(Constants.GUID, "goodMessageId");
    goodMessage.put("value", "goodMessage");
    JSONObject errorMessage = new JSONObject();
    goodMessage.put(Constants.GUID, "errorMessageId");
    errorMessage.put("value", "errorMessage");
    when(goodTuple.getValueByField(eq("message"))).thenReturn(goodMessage);
    when(errorTuple.getValueByField(eq("message"))).thenReturn(errorMessage);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    // Add both the good and error Tuples. This simulates a seemingly good Tuple that fails on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(messageIds);
    writerResponse.addSuccess(new MessageId("goodMessageId"));
    writerResponse.addError(new IllegalStateException(), new MessageId("errorMessageId"));
    when(batchWriter.write(any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
    bolt.execute(goodTuple);
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 44 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchErrorPath.

@Test
public void testBatchErrorPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(new MessageId(messageId));
        JSONObject message = new JSONObject();
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    when(errorTuple.getValueByField(eq("message"))).thenThrow(new IllegalStateException());
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    // Add the good tuples.  Do not add the error tuple, because this is testing an exception on access, not a failure on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(messageIds);
    writerResponse.addSuccess(new MessageId("goodMessage"));
    when(batchWriter.write(any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    bolt.execute(goodTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 45 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchErrorPathExceptionInWrite.

@Test
public void testBatchErrorPathExceptionInWrite() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<String> messageIds = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(messageId);
        JSONObject message = new JSONObject();
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    doThrow(new Exception()).when(batchWriter).write(any(), any(), any());
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
    bolt.execute(goodTuple);
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}